@korajs/server 1.0.0-beta.0 → 1.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1957 -219
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +962 -183
- package/dist/index.d.ts +962 -183
- package/dist/index.js +1919 -178
- package/dist/index.js.map +1 -1
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -54,11 +54,17 @@ function createDefaultLogger() {
|
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
// src/apply/rejection-taxonomy.ts
|
|
58
|
+
var RETRIABLE_REJECTION_CODES = /* @__PURE__ */ new Set(["RATE_LIMIT"]);
|
|
59
|
+
function isRetriableRejection(code) {
|
|
60
|
+
return RETRIABLE_REJECTION_CODES.has(code);
|
|
61
|
+
}
|
|
62
|
+
|
|
57
63
|
// src/store/memory-server-store.ts
|
|
58
64
|
import { generateUUIDv7 } from "@korajs/core";
|
|
59
65
|
|
|
60
66
|
// src/store/materialization.ts
|
|
61
|
-
import { StorageError } from "@korajs/core";
|
|
67
|
+
import { StorageError, quoteIdent, replayOperationsForRecord } from "@korajs/core";
|
|
62
68
|
function fieldTypeToSql(descriptor, dialect) {
|
|
63
69
|
switch (descriptor.kind) {
|
|
64
70
|
case "string":
|
|
@@ -73,6 +79,14 @@ function fieldTypeToSql(descriptor, dialect) {
|
|
|
73
79
|
return dialect === "postgres" ? "BIGINT" : "INTEGER";
|
|
74
80
|
case "array":
|
|
75
81
|
return dialect === "postgres" ? "JSONB" : "TEXT";
|
|
82
|
+
case "object":
|
|
83
|
+
return dialect === "postgres" ? "JSONB" : "TEXT";
|
|
84
|
+
case "json":
|
|
85
|
+
return dialect === "postgres" ? "JSONB" : "TEXT";
|
|
86
|
+
case "blob":
|
|
87
|
+
return "TEXT";
|
|
88
|
+
case "secret":
|
|
89
|
+
return "TEXT";
|
|
76
90
|
case "richtext":
|
|
77
91
|
return dialect === "postgres" ? "BYTEA" : "BLOB";
|
|
78
92
|
}
|
|
@@ -89,13 +103,13 @@ function generateCollectionDDL(name, collection, dialect) {
|
|
|
89
103
|
const columns = ["id TEXT PRIMARY KEY NOT NULL"];
|
|
90
104
|
for (const [fieldName, descriptor] of Object.entries(collection.fields)) {
|
|
91
105
|
const sqlType = fieldTypeToSql(descriptor, dialect);
|
|
92
|
-
let colDef = `${fieldName} ${sqlType}`;
|
|
106
|
+
let colDef = `${quoteIdent(fieldName)} ${sqlType}`;
|
|
93
107
|
if (descriptor.defaultValue !== void 0) {
|
|
94
108
|
colDef += ` DEFAULT ${sqlDefaultLiteral(descriptor.defaultValue)}`;
|
|
95
109
|
}
|
|
96
110
|
if (descriptor.kind === "enum" && descriptor.enumValues) {
|
|
97
111
|
const values = descriptor.enumValues.map((v) => `'${v}'`).join(", ");
|
|
98
|
-
colDef += ` CHECK (${fieldName} IN (${values}))`;
|
|
112
|
+
colDef += ` CHECK (${quoteIdent(fieldName)} IN (${values}))`;
|
|
99
113
|
}
|
|
100
114
|
columns.push(colDef);
|
|
101
115
|
}
|
|
@@ -103,24 +117,26 @@ function generateCollectionDDL(name, collection, dialect) {
|
|
|
103
117
|
columns.push(`_created_at ${tsType} NOT NULL DEFAULT 0`);
|
|
104
118
|
columns.push(`_updated_at ${tsType} NOT NULL DEFAULT 0`);
|
|
105
119
|
columns.push("_deleted INTEGER NOT NULL DEFAULT 0");
|
|
106
|
-
statements.push(`CREATE TABLE IF NOT EXISTS ${name} (
|
|
120
|
+
statements.push(`CREATE TABLE IF NOT EXISTS ${quoteIdent(name)} (
|
|
107
121
|
${columns.join(",\n ")}
|
|
108
122
|
)`);
|
|
109
123
|
for (const [fieldName, descriptor] of Object.entries(collection.fields)) {
|
|
110
124
|
const sqlType = fieldTypeToSql(descriptor, dialect);
|
|
111
|
-
let colDef = `${fieldName} ${sqlType}`;
|
|
125
|
+
let colDef = `${quoteIdent(fieldName)} ${sqlType}`;
|
|
112
126
|
if (descriptor.defaultValue !== void 0) {
|
|
113
127
|
colDef += ` DEFAULT ${sqlDefaultLiteral(descriptor.defaultValue)}`;
|
|
114
128
|
}
|
|
115
129
|
statements.push(`--kora:safe-alter
|
|
116
|
-
ALTER TABLE ${name} ADD COLUMN ${colDef}`);
|
|
130
|
+
ALTER TABLE ${quoteIdent(name)} ADD COLUMN ${colDef}`);
|
|
117
131
|
}
|
|
118
132
|
for (const indexField of collection.indexes) {
|
|
119
133
|
statements.push(
|
|
120
|
-
`CREATE INDEX IF NOT EXISTS idx_${name}_${indexField} ON ${name} (${indexField})`
|
|
134
|
+
`CREATE INDEX IF NOT EXISTS idx_${name}_${indexField} ON ${quoteIdent(name)} (${quoteIdent(indexField)})`
|
|
121
135
|
);
|
|
122
136
|
}
|
|
123
|
-
statements.push(
|
|
137
|
+
statements.push(
|
|
138
|
+
`CREATE INDEX IF NOT EXISTS idx_${name}__deleted ON ${quoteIdent(name)} (_deleted)`
|
|
139
|
+
);
|
|
124
140
|
return statements;
|
|
125
141
|
}
|
|
126
142
|
function generateAllCollectionDDL(schema, dialect) {
|
|
@@ -130,30 +146,6 @@ function generateAllCollectionDDL(schema, dialect) {
|
|
|
130
146
|
}
|
|
131
147
|
return statements;
|
|
132
148
|
}
|
|
133
|
-
function replayOperationsForRecord(ops) {
|
|
134
|
-
let record = null;
|
|
135
|
-
let deleted = false;
|
|
136
|
-
for (const op of ops) {
|
|
137
|
-
switch (op.type) {
|
|
138
|
-
case "insert":
|
|
139
|
-
if (op.data) {
|
|
140
|
-
record = { ...op.data };
|
|
141
|
-
deleted = false;
|
|
142
|
-
}
|
|
143
|
-
break;
|
|
144
|
-
case "update":
|
|
145
|
-
if (op.data) {
|
|
146
|
-
record = { ...record ?? {}, ...op.data };
|
|
147
|
-
deleted = false;
|
|
148
|
-
}
|
|
149
|
-
break;
|
|
150
|
-
case "delete":
|
|
151
|
-
deleted = true;
|
|
152
|
-
break;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
return deleted ? null : record;
|
|
156
|
-
}
|
|
157
149
|
var BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
158
150
|
var BASE64_REVERSE = new Map(
|
|
159
151
|
Array.from(BASE64_ALPHABET, (char, index3) => [char, index3])
|
|
@@ -204,6 +196,9 @@ function serializeFieldValue(value, descriptor) {
|
|
|
204
196
|
if (value === null || value === void 0) return null;
|
|
205
197
|
switch (descriptor.kind) {
|
|
206
198
|
case "array":
|
|
199
|
+
case "object":
|
|
200
|
+
case "json":
|
|
201
|
+
case "blob":
|
|
207
202
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
208
203
|
case "boolean":
|
|
209
204
|
return value ? 1 : 0;
|
|
@@ -217,6 +212,9 @@ function deserializeFieldValue(value, descriptor) {
|
|
|
217
212
|
if (value === null || value === void 0) return null;
|
|
218
213
|
switch (descriptor.kind) {
|
|
219
214
|
case "array":
|
|
215
|
+
case "object":
|
|
216
|
+
case "json":
|
|
217
|
+
case "blob":
|
|
220
218
|
return typeof value === "string" ? JSON.parse(value) : value;
|
|
221
219
|
case "boolean":
|
|
222
220
|
return value === 1 || value === true;
|
|
@@ -249,6 +247,14 @@ var MemoryServerStore = class {
|
|
|
249
247
|
operations = [];
|
|
250
248
|
operationIndex = /* @__PURE__ */ new Map();
|
|
251
249
|
versionVector = /* @__PURE__ */ new Map();
|
|
250
|
+
/**
|
|
251
|
+
* Server-assigned delivery sequence per operation id. Single-process, so a plain
|
|
252
|
+
* counter incremented at insert time gives commit-order delivery sequence with no
|
|
253
|
+
* gaps. `operations` is kept in insertion (== delivery) order, so a delivery scan
|
|
254
|
+
* is a forward walk.
|
|
255
|
+
*/
|
|
256
|
+
deliverySeqByOpId = /* @__PURE__ */ new Map();
|
|
257
|
+
deliverySeqCounter = 0;
|
|
252
258
|
schema = null;
|
|
253
259
|
/** Materialized records: collection -> recordId -> record data */
|
|
254
260
|
materializedRecords = /* @__PURE__ */ new Map();
|
|
@@ -282,6 +288,8 @@ var MemoryServerStore = class {
|
|
|
282
288
|
}
|
|
283
289
|
this.operations.push(op);
|
|
284
290
|
this.operationIndex.set(op.id, op);
|
|
291
|
+
this.deliverySeqCounter += 1;
|
|
292
|
+
this.deliverySeqByOpId.set(op.id, this.deliverySeqCounter);
|
|
285
293
|
const currentSeq = this.versionVector.get(op.nodeId) ?? 0;
|
|
286
294
|
if (op.sequenceNumber > currentSeq) {
|
|
287
295
|
this.versionVector.set(op.nodeId, op.sequenceNumber);
|
|
@@ -301,6 +309,23 @@ var MemoryServerStore = class {
|
|
|
301
309
|
this.assertOpen();
|
|
302
310
|
return this.operations.length;
|
|
303
311
|
}
|
|
312
|
+
async getMaxDeliverySequence() {
|
|
313
|
+
this.assertOpen();
|
|
314
|
+
return this.deliverySeqCounter;
|
|
315
|
+
}
|
|
316
|
+
async getOperationsAfterDelivery(afterDeliverySequence, limit) {
|
|
317
|
+
this.assertOpen();
|
|
318
|
+
const result = [];
|
|
319
|
+
if (limit <= 0) return result;
|
|
320
|
+
for (const op of this.operations) {
|
|
321
|
+
const deliverySequence = this.deliverySeqByOpId.get(op.id) ?? 0;
|
|
322
|
+
if (deliverySequence > afterDeliverySequence) {
|
|
323
|
+
result.push({ operation: op, deliverySequence });
|
|
324
|
+
if (result.length >= limit) break;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
return result;
|
|
328
|
+
}
|
|
304
329
|
async materializeCollection(collection) {
|
|
305
330
|
this.assertOpen();
|
|
306
331
|
if (this.schema?.collections[collection]) {
|
|
@@ -445,12 +470,16 @@ var MemoryServerStore = class {
|
|
|
445
470
|
this.operations.length = 0;
|
|
446
471
|
this.operationIndex.clear();
|
|
447
472
|
this.versionVector.clear();
|
|
473
|
+
this.deliverySeqByOpId.clear();
|
|
474
|
+
this.deliverySeqCounter = 0;
|
|
448
475
|
for (const [nid, seq] of versionVector) {
|
|
449
476
|
this.versionVector.set(nid, seq);
|
|
450
477
|
}
|
|
451
478
|
for (const op of operations2) {
|
|
452
479
|
this.operations.push(op);
|
|
453
480
|
this.operationIndex.set(op.id, op);
|
|
481
|
+
this.deliverySeqCounter += 1;
|
|
482
|
+
this.deliverySeqByOpId.set(op.id, this.deliverySeqCounter);
|
|
454
483
|
if (this.schema?.collections[op.collection]) {
|
|
455
484
|
this.rebuildMaterializedRecord(op.collection, op.recordId);
|
|
456
485
|
}
|
|
@@ -480,11 +509,12 @@ var MemoryServerStore = class {
|
|
|
480
509
|
return a.timestamp.wallTime - b.timestamp.wallTime;
|
|
481
510
|
if (a.timestamp.logical !== b.timestamp.logical)
|
|
482
511
|
return a.timestamp.logical - b.timestamp.logical;
|
|
483
|
-
return a.
|
|
512
|
+
return a.timestamp.nodeId < b.timestamp.nodeId ? -1 : a.timestamp.nodeId > b.timestamp.nodeId ? 1 : 0;
|
|
484
513
|
});
|
|
485
514
|
const parsedOps = recordOps.map((op) => ({
|
|
486
515
|
type: op.type,
|
|
487
|
-
data: op.data
|
|
516
|
+
data: op.data,
|
|
517
|
+
atomicOps: op.atomicOps ?? null
|
|
488
518
|
}));
|
|
489
519
|
const recordData = replayOperationsForRecord(parsedOps);
|
|
490
520
|
if (recordData) {
|
|
@@ -590,8 +620,8 @@ var MemoryServerStore = class {
|
|
|
590
620
|
};
|
|
591
621
|
|
|
592
622
|
// src/store/postgres-server-store.ts
|
|
593
|
-
import { generateUUIDv7 as generateUUIDv72 } from "@korajs/core";
|
|
594
|
-
import { and, asc, between, count, eq, sql } from "drizzle-orm";
|
|
623
|
+
import { HybridLogicalClock, generateUUIDv7 as generateUUIDv72, quoteIdent as quoteIdent2 } from "@korajs/core";
|
|
624
|
+
import { and, asc, between, count, desc, eq, gt, sql } from "drizzle-orm";
|
|
595
625
|
|
|
596
626
|
// src/store/drizzle-pg-schema.ts
|
|
597
627
|
import { bigint, index, integer, pgTable, text } from "drizzle-orm/pg-core";
|
|
@@ -607,6 +637,8 @@ var pgOperations = pgTable(
|
|
|
607
637
|
// JSON-serialized, null for deletes
|
|
608
638
|
previousData: text("previous_data"),
|
|
609
639
|
// JSON-serialized, null for insert/delete
|
|
640
|
+
atomicOps: text("atomic_ops"),
|
|
641
|
+
// JSON-serialized Record<field, AtomicOp>, null when none
|
|
610
642
|
wallTime: bigint("wall_time", { mode: "number" }).notNull(),
|
|
611
643
|
logical: integer("logical").notNull(),
|
|
612
644
|
timestampNodeId: text("timestamp_node_id").notNull(),
|
|
@@ -614,12 +646,18 @@ var pgOperations = pgTable(
|
|
|
614
646
|
causalDeps: text("causal_deps").notNull().default("[]"),
|
|
615
647
|
// JSON array of op IDs
|
|
616
648
|
schemaVersion: integer("schema_version").notNull(),
|
|
617
|
-
receivedAt: bigint("received_at", { mode: "number" }).notNull()
|
|
649
|
+
receivedAt: bigint("received_at", { mode: "number" }).notNull(),
|
|
650
|
+
// Server-assigned monotonic delivery sequence, ordered by commit (assigned from
|
|
651
|
+
// a counter row locked inside the append transaction, so delivery order equals
|
|
652
|
+
// visibility order even across instances). Drives the gap-free server->client
|
|
653
|
+
// delivery watermark. Nullable only for pre-column rows, backfilled on startup.
|
|
654
|
+
deliverySeq: bigint("delivery_seq", { mode: "number" })
|
|
618
655
|
},
|
|
619
656
|
(table) => ({
|
|
620
657
|
nodeSeqIdx: index("idx_pg_node_seq").on(table.nodeId, table.sequenceNumber),
|
|
621
658
|
collectionIdx: index("idx_pg_collection").on(table.collection),
|
|
622
|
-
receivedIdx: index("idx_pg_received").on(table.receivedAt)
|
|
659
|
+
receivedIdx: index("idx_pg_received").on(table.receivedAt),
|
|
660
|
+
deliveryIdx: index("idx_pg_delivery_seq").on(table.deliverySeq)
|
|
623
661
|
})
|
|
624
662
|
);
|
|
625
663
|
var pgSyncState = pgTable("sync_state", {
|
|
@@ -636,15 +674,43 @@ var PostgresServerStore = class {
|
|
|
636
674
|
ready;
|
|
637
675
|
schema = null;
|
|
638
676
|
closed = false;
|
|
639
|
-
|
|
677
|
+
/**
|
|
678
|
+
* Monotonic counter for this node's server-originated sequence numbers. Lazily
|
|
679
|
+
* seeded from the persisted version vector on first use, then advanced in memory
|
|
680
|
+
* (a synchronous `++`, atomic on the single JS thread) so concurrent conditional
|
|
681
|
+
* applies never receive the same number. Null means "not yet seeded"; reset to
|
|
682
|
+
* null after a backup import so it re-seeds from the restored version vector.
|
|
683
|
+
*/
|
|
684
|
+
sequenceCounter = null;
|
|
685
|
+
/**
|
|
686
|
+
* Time source for HLC timestamps on server-originated operations. Injectable so a
|
|
687
|
+
* test can freeze wall-clock time and prove the conditional-apply ordering holds
|
|
688
|
+
* under same-millisecond commits (the case the advisory lock and clock advance
|
|
689
|
+
* exist to make correct). Defaults to the system clock in production.
|
|
690
|
+
*/
|
|
691
|
+
timeSource;
|
|
692
|
+
constructor(db, nodeId, timeSource) {
|
|
640
693
|
this.db = db;
|
|
641
694
|
this.nodeId = nodeId ?? generateUUIDv72();
|
|
695
|
+
this.timeSource = timeSource ?? { now: () => Date.now() };
|
|
642
696
|
this.ready = this.initialize();
|
|
643
697
|
}
|
|
644
698
|
getVersionVector() {
|
|
645
699
|
this.assertOpen();
|
|
646
700
|
return new Map(this.versionVector);
|
|
647
701
|
}
|
|
702
|
+
/**
|
|
703
|
+
* Atomically reserve this node's next server-operation sequence number. The
|
|
704
|
+
* synchronous increment cannot interleave on the single JS thread, so two
|
|
705
|
+
* concurrent conditional applies are always handed distinct numbers.
|
|
706
|
+
*/
|
|
707
|
+
reserveSequenceNumber() {
|
|
708
|
+
if (this.sequenceCounter === null) {
|
|
709
|
+
this.sequenceCounter = this.versionVector.get(this.nodeId) ?? 0;
|
|
710
|
+
}
|
|
711
|
+
this.sequenceCounter += 1;
|
|
712
|
+
return this.sequenceCounter;
|
|
713
|
+
}
|
|
648
714
|
getNodeId() {
|
|
649
715
|
return this.nodeId;
|
|
650
716
|
}
|
|
@@ -682,8 +748,9 @@ var PostgresServerStore = class {
|
|
|
682
748
|
return "duplicate";
|
|
683
749
|
}
|
|
684
750
|
const now = Date.now();
|
|
685
|
-
const row = this.serializeOperation(op, now);
|
|
686
751
|
await this.db.transaction(async (tx) => {
|
|
752
|
+
const deliverySeq = await this.nextDeliverySeq(tx);
|
|
753
|
+
const row = this.serializeOperation(op, now, deliverySeq);
|
|
687
754
|
await tx.insert(pgOperations).values(row).onConflictDoNothing({ target: pgOperations.id });
|
|
688
755
|
await tx.insert(pgSyncState).values({
|
|
689
756
|
nodeId: op.nodeId,
|
|
@@ -706,6 +773,73 @@ var PostgresServerStore = class {
|
|
|
706
773
|
}
|
|
707
774
|
return "applied";
|
|
708
775
|
}
|
|
776
|
+
/**
|
|
777
|
+
* Conditionally apply operations atomically, serialized across server instances
|
|
778
|
+
* on the target record via a transaction-scoped advisory lock. The lock makes
|
|
779
|
+
* the read-decide-write cycle atomic against concurrent instances, so a cap
|
|
780
|
+
* check like `responseCount < max` cannot be passed by two admissions at once.
|
|
781
|
+
*/
|
|
782
|
+
async applyConditional(input) {
|
|
783
|
+
this.assertOpen();
|
|
784
|
+
await this.ready;
|
|
785
|
+
this.assertSchema();
|
|
786
|
+
this.assertCollection(input.target.collection);
|
|
787
|
+
const schema = this.schema;
|
|
788
|
+
const collectionDef = schema.collections[input.target.collection];
|
|
789
|
+
const lockKey = `kora:${input.target.collection}:${input.target.id}`;
|
|
790
|
+
const result = await this.db.transaction(async (tx) => {
|
|
791
|
+
await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended(${lockKey}, 0))`);
|
|
792
|
+
if (input.idempotencyKey) {
|
|
793
|
+
const existing = await tx.execute(
|
|
794
|
+
sql`SELECT 1 FROM ${sql.raw(quoteIdent2(input.idempotencyKey.collection))} WHERE id = ${input.idempotencyKey.id} AND _deleted = 0 LIMIT 1`
|
|
795
|
+
);
|
|
796
|
+
if (existing.length > 0) {
|
|
797
|
+
return { admitted: true, idempotent: true, applied: [] };
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
const rows = await tx.execute(
|
|
801
|
+
sql`SELECT * FROM ${sql.raw(quoteIdent2(input.target.collection))} WHERE id = ${input.target.id} AND _deleted = 0`
|
|
802
|
+
);
|
|
803
|
+
const current = rows.length > 0 ? this.deserializeRow(rows[0], collectionDef) : null;
|
|
804
|
+
if (!input.admit(current)) {
|
|
805
|
+
return { admitted: false, idempotent: false, applied: [] };
|
|
806
|
+
}
|
|
807
|
+
const clock = new HybridLogicalClock(this.nodeId, this.timeSource);
|
|
808
|
+
const latest = await this.latestOperationTimestamp(
|
|
809
|
+
tx,
|
|
810
|
+
input.target.collection,
|
|
811
|
+
input.target.id
|
|
812
|
+
);
|
|
813
|
+
if (latest) {
|
|
814
|
+
clock.advanceTo(latest);
|
|
815
|
+
}
|
|
816
|
+
const ops = await input.buildOperations(current, { clock });
|
|
817
|
+
const now = Date.now();
|
|
818
|
+
for (const op of ops) {
|
|
819
|
+
const deliverySeq = await this.nextDeliverySeq(tx);
|
|
820
|
+
const row = this.serializeOperation(op, now, deliverySeq);
|
|
821
|
+
await tx.insert(pgOperations).values(row).onConflictDoNothing({ target: pgOperations.id });
|
|
822
|
+
await tx.insert(pgSyncState).values({ nodeId: op.nodeId, maxSequenceNumber: op.sequenceNumber, lastSeenAt: now }).onConflictDoUpdate({
|
|
823
|
+
target: pgSyncState.nodeId,
|
|
824
|
+
set: {
|
|
825
|
+
maxSequenceNumber: sql`GREATEST(${pgSyncState.maxSequenceNumber}, ${op.sequenceNumber})`,
|
|
826
|
+
lastSeenAt: sql`${now}`
|
|
827
|
+
}
|
|
828
|
+
});
|
|
829
|
+
if (this.schema?.collections[op.collection]) {
|
|
830
|
+
await this.rebuildMaterializedRecord(tx, op.collection, op.recordId);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
return { admitted: true, idempotent: false, applied: ops };
|
|
834
|
+
});
|
|
835
|
+
for (const op of result.applied) {
|
|
836
|
+
const currentMax = this.versionVector.get(op.nodeId) ?? 0;
|
|
837
|
+
if (op.sequenceNumber > currentMax) {
|
|
838
|
+
this.versionVector.set(op.nodeId, op.sequenceNumber);
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
return result;
|
|
842
|
+
}
|
|
709
843
|
async getOperationRange(nodeId, fromSeq, toSeq) {
|
|
710
844
|
this.assertOpen();
|
|
711
845
|
await this.ready;
|
|
@@ -720,6 +854,23 @@ var PostgresServerStore = class {
|
|
|
720
854
|
const result = await this.db.select({ value: count() }).from(pgOperations);
|
|
721
855
|
return result[0]?.value ?? 0;
|
|
722
856
|
}
|
|
857
|
+
async getMaxDeliverySequence() {
|
|
858
|
+
this.assertOpen();
|
|
859
|
+
await this.ready;
|
|
860
|
+
const rows = await this.db.execute(
|
|
861
|
+
sql`SELECT COALESCE(MAX(delivery_seq), 0) AS m FROM operations`
|
|
862
|
+
);
|
|
863
|
+
return Number(rows[0]?.m ?? 0);
|
|
864
|
+
}
|
|
865
|
+
async getOperationsAfterDelivery(afterDeliverySequence, limit) {
|
|
866
|
+
this.assertOpen();
|
|
867
|
+
await this.ready;
|
|
868
|
+
const rows = await this.db.select().from(pgOperations).where(gt(pgOperations.deliverySeq, afterDeliverySequence)).orderBy(asc(pgOperations.deliverySeq)).limit(limit);
|
|
869
|
+
return rows.map((row) => ({
|
|
870
|
+
operation: this.deserializeOperation(row),
|
|
871
|
+
deliverySequence: row.deliverySeq ?? 0
|
|
872
|
+
}));
|
|
873
|
+
}
|
|
723
874
|
async materializeCollection(collection) {
|
|
724
875
|
this.assertOpen();
|
|
725
876
|
await this.ready;
|
|
@@ -754,7 +905,7 @@ var PostgresServerStore = class {
|
|
|
754
905
|
this.assertCollection(collection);
|
|
755
906
|
const schema = this.schema;
|
|
756
907
|
const collectionDef = schema.collections[collection];
|
|
757
|
-
const query = sql`SELECT * FROM ${sql.raw(collection)} WHERE id = ${id} AND _deleted = 0`;
|
|
908
|
+
const query = sql`SELECT * FROM ${sql.raw(quoteIdent2(collection))} WHERE id = ${id} AND _deleted = 0`;
|
|
758
909
|
const rows = await this.db.execute(query);
|
|
759
910
|
if (rows.length === 0) return null;
|
|
760
911
|
return this.deserializeRow(rows[0], collectionDef);
|
|
@@ -771,7 +922,7 @@ var PostgresServerStore = class {
|
|
|
771
922
|
}
|
|
772
923
|
}
|
|
773
924
|
const whereClause = this.buildWhereClause(where ?? {}, false);
|
|
774
|
-
const query = sql`SELECT COUNT(*) as cnt FROM ${sql.raw(collection)} WHERE ${whereClause}`;
|
|
925
|
+
const query = sql`SELECT COUNT(*) as cnt FROM ${sql.raw(quoteIdent2(collection))} WHERE ${whereClause}`;
|
|
775
926
|
const rows = await this.db.execute(query);
|
|
776
927
|
const cnt = rows[0]?.cnt;
|
|
777
928
|
return typeof cnt === "string" ? Number.parseInt(cnt, 10) : cnt ?? 0;
|
|
@@ -783,7 +934,7 @@ var PostgresServerStore = class {
|
|
|
783
934
|
this.assertOpen();
|
|
784
935
|
await this.ready;
|
|
785
936
|
const { buildServerBackup } = await import("./server-backup-MOICK6MI.js");
|
|
786
|
-
const rows = await this.db.select().from(pgOperations).orderBy(asc(pgOperations.
|
|
937
|
+
const rows = await this.db.select().from(pgOperations).orderBy(asc(pgOperations.deliverySeq));
|
|
787
938
|
const operations2 = rows.map((row) => this.deserializeOperation(row));
|
|
788
939
|
return buildServerBackup(this.nodeId, operations2, this.versionVector);
|
|
789
940
|
}
|
|
@@ -798,6 +949,7 @@ var PostgresServerStore = class {
|
|
|
798
949
|
const result = await this.applyRemoteOperation(op);
|
|
799
950
|
if (result === "applied") restored++;
|
|
800
951
|
}
|
|
952
|
+
this.sequenceCounter = null;
|
|
801
953
|
return { operationsRestored: restored, success: true };
|
|
802
954
|
}
|
|
803
955
|
const now = Date.now();
|
|
@@ -807,20 +959,49 @@ var PostgresServerStore = class {
|
|
|
807
959
|
for (const [nid, seq] of versionVector) {
|
|
808
960
|
await tx.insert(pgSyncState).values({ nodeId: nid, maxSequenceNumber: seq, lastSeenAt: now }).onConflictDoNothing({ target: pgSyncState.nodeId });
|
|
809
961
|
}
|
|
962
|
+
let deliverySeq = 0;
|
|
810
963
|
for (const op of operations2) {
|
|
811
|
-
|
|
964
|
+
deliverySeq += 1;
|
|
965
|
+
const row = this.serializeOperation(op, now, deliverySeq);
|
|
812
966
|
await tx.insert(pgOperations).values(row).onConflictDoNothing({ target: pgOperations.id });
|
|
813
967
|
}
|
|
968
|
+
await tx.execute(
|
|
969
|
+
sql`INSERT INTO delivery_counter (id, value) VALUES (1, ${deliverySeq})
|
|
970
|
+
ON CONFLICT (id) DO UPDATE SET value = ${deliverySeq}`
|
|
971
|
+
);
|
|
814
972
|
});
|
|
815
973
|
this.versionVector.clear();
|
|
816
974
|
for (const [nid, seq] of versionVector) {
|
|
817
975
|
this.versionVector.set(nid, seq);
|
|
818
976
|
}
|
|
977
|
+
this.sequenceCounter = null;
|
|
819
978
|
return { operationsRestored: operations2.length, success: true };
|
|
820
979
|
}
|
|
821
980
|
// ---------------------------------------------------------------------------
|
|
822
981
|
// Materialization internals
|
|
823
982
|
// ---------------------------------------------------------------------------
|
|
983
|
+
/**
|
|
984
|
+
* The HLC timestamp of the most recent operation on a record, by total order
|
|
985
|
+
* (wallTime, logical, nodeId), or null when the record has no operations. Used
|
|
986
|
+
* by {@link applyConditional} to advance its clock past the record's latest
|
|
987
|
+
* write so newly built operations sort strictly after it.
|
|
988
|
+
*/
|
|
989
|
+
async latestOperationTimestamp(txOrDb, collection, recordId) {
|
|
990
|
+
const rows = await txOrDb.select({
|
|
991
|
+
wallTime: pgOperations.wallTime,
|
|
992
|
+
logical: pgOperations.logical,
|
|
993
|
+
timestampNodeId: pgOperations.timestampNodeId
|
|
994
|
+
}).from(pgOperations).where(and(eq(pgOperations.collection, collection), eq(pgOperations.recordId, recordId))).orderBy(
|
|
995
|
+
desc(pgOperations.wallTime),
|
|
996
|
+
desc(pgOperations.logical),
|
|
997
|
+
desc(pgOperations.timestampNodeId)
|
|
998
|
+
).limit(1);
|
|
999
|
+
const row = rows[0];
|
|
1000
|
+
if (!row) {
|
|
1001
|
+
return null;
|
|
1002
|
+
}
|
|
1003
|
+
return { wallTime: row.wallTime, logical: row.logical, nodeId: row.timestampNodeId };
|
|
1004
|
+
}
|
|
824
1005
|
/**
|
|
825
1006
|
* Rebuild a single record in the materialized collection table by replaying
|
|
826
1007
|
* all operations for that record.
|
|
@@ -831,15 +1012,17 @@ var PostgresServerStore = class {
|
|
|
831
1012
|
const ops = await txOrDb.select({
|
|
832
1013
|
type: pgOperations.type,
|
|
833
1014
|
data: pgOperations.data,
|
|
1015
|
+
atomicOps: pgOperations.atomicOps,
|
|
834
1016
|
wallTime: pgOperations.wallTime
|
|
835
1017
|
}).from(pgOperations).where(and(eq(pgOperations.collection, collection), eq(pgOperations.recordId, recordId))).orderBy(
|
|
836
1018
|
asc(pgOperations.wallTime),
|
|
837
1019
|
asc(pgOperations.logical),
|
|
838
|
-
asc(pgOperations.
|
|
1020
|
+
asc(pgOperations.timestampNodeId)
|
|
839
1021
|
);
|
|
840
1022
|
const parsedOps = ops.map((op) => ({
|
|
841
1023
|
type: op.type,
|
|
842
|
-
data: op.data !== null ? JSON.parse(op.data) : null
|
|
1024
|
+
data: op.data !== null ? JSON.parse(op.data) : null,
|
|
1025
|
+
atomicOps: op.atomicOps != null ? JSON.parse(op.atomicOps) : null
|
|
843
1026
|
}));
|
|
844
1027
|
const recordData = replayOperationsForRecord(parsedOps);
|
|
845
1028
|
const fieldNames = Object.keys(collectionDef.fields);
|
|
@@ -858,7 +1041,7 @@ var PostgresServerStore = class {
|
|
|
858
1041
|
);
|
|
859
1042
|
} else {
|
|
860
1043
|
await txOrDb.execute(
|
|
861
|
-
sql`UPDATE ${sql.raw(collection)} SET _deleted = 1, _updated_at = ${Date.now()} WHERE id = ${recordId}`
|
|
1044
|
+
sql`UPDATE ${sql.raw(quoteIdent2(collection))} SET _deleted = 1, _updated_at = ${Date.now()} WHERE id = ${recordId}`
|
|
862
1045
|
);
|
|
863
1046
|
}
|
|
864
1047
|
}
|
|
@@ -877,16 +1060,16 @@ var PostgresServerStore = class {
|
|
|
877
1060
|
updatedAt,
|
|
878
1061
|
0
|
|
879
1062
|
];
|
|
880
|
-
const columnsSql = sql.raw(allColumns.join(", "));
|
|
1063
|
+
const columnsSql = sql.raw(allColumns.map((c) => quoteIdent2(c)).join(", "));
|
|
881
1064
|
const valuesSql = sql.join(
|
|
882
1065
|
values.map((v) => sql`${v}`),
|
|
883
1066
|
sql.raw(", ")
|
|
884
1067
|
);
|
|
885
1068
|
const updateSet = sql.raw(
|
|
886
|
-
allColumns.slice(1).map((c) => `${c} = excluded.${c}`).join(", ")
|
|
1069
|
+
allColumns.slice(1).map((c) => `${quoteIdent2(c)} = excluded.${quoteIdent2(c)}`).join(", ")
|
|
887
1070
|
);
|
|
888
1071
|
await txOrDb.execute(
|
|
889
|
-
sql`INSERT INTO ${sql.raw(tableName)} (${columnsSql}) VALUES (${valuesSql}) ON CONFLICT (id) DO UPDATE SET ${updateSet}`
|
|
1072
|
+
sql`INSERT INTO ${sql.raw(quoteIdent2(tableName))} (${columnsSql}) VALUES (${valuesSql}) ON CONFLICT (id) DO UPDATE SET ${updateSet}`
|
|
890
1073
|
);
|
|
891
1074
|
}
|
|
892
1075
|
/**
|
|
@@ -908,11 +1091,12 @@ var PostgresServerStore = class {
|
|
|
908
1091
|
recordId: pgOperations.recordId,
|
|
909
1092
|
type: pgOperations.type,
|
|
910
1093
|
data: pgOperations.data,
|
|
1094
|
+
atomicOps: pgOperations.atomicOps,
|
|
911
1095
|
wallTime: pgOperations.wallTime
|
|
912
1096
|
}).from(pgOperations).where(eq(pgOperations.collection, collectionName)).orderBy(
|
|
913
1097
|
asc(pgOperations.wallTime),
|
|
914
1098
|
asc(pgOperations.logical),
|
|
915
|
-
asc(pgOperations.
|
|
1099
|
+
asc(pgOperations.timestampNodeId)
|
|
916
1100
|
);
|
|
917
1101
|
if (allOps.length === 0) return;
|
|
918
1102
|
const grouped = /* @__PURE__ */ new Map();
|
|
@@ -928,7 +1112,8 @@ var PostgresServerStore = class {
|
|
|
928
1112
|
for (const [recordId, recordOps] of grouped) {
|
|
929
1113
|
const parsedOps = recordOps.map((op) => ({
|
|
930
1114
|
type: op.type,
|
|
931
|
-
data: op.data !== null ? JSON.parse(op.data) : null
|
|
1115
|
+
data: op.data !== null ? JSON.parse(op.data) : null,
|
|
1116
|
+
atomicOps: op.atomicOps != null ? JSON.parse(op.atomicOps) : null
|
|
932
1117
|
}));
|
|
933
1118
|
const recordData = replayOperationsForRecord(parsedOps);
|
|
934
1119
|
if (recordData) {
|
|
@@ -946,7 +1131,7 @@ var PostgresServerStore = class {
|
|
|
946
1131
|
);
|
|
947
1132
|
} else {
|
|
948
1133
|
await this.db.execute(
|
|
949
|
-
sql`INSERT INTO ${sql.raw(collectionName)} (id, _deleted, _created_at, _updated_at) VALUES (${recordId}, 1, ${Date.now()}, ${Date.now()}) ON CONFLICT (id) DO UPDATE SET _deleted = 1, _updated_at = ${Date.now()}`
|
|
1134
|
+
sql`INSERT INTO ${sql.raw(quoteIdent2(collectionName))} (id, _deleted, _created_at, _updated_at) VALUES (${recordId}, 1, ${Date.now()}, ${Date.now()}) ON CONFLICT (id) DO UPDATE SET _deleted = 1, _updated_at = ${Date.now()}`
|
|
950
1135
|
);
|
|
951
1136
|
}
|
|
952
1137
|
}
|
|
@@ -959,10 +1144,12 @@ var PostgresServerStore = class {
|
|
|
959
1144
|
options?.where ?? {},
|
|
960
1145
|
options?.includeDeleted ?? false
|
|
961
1146
|
);
|
|
962
|
-
const parts = [
|
|
1147
|
+
const parts = [
|
|
1148
|
+
sql`SELECT * FROM ${sql.raw(quoteIdent2(collection))} WHERE ${whereClause}`
|
|
1149
|
+
];
|
|
963
1150
|
if (options?.orderBy) {
|
|
964
1151
|
const dir = options.orderDirection === "desc" ? "DESC" : "ASC";
|
|
965
|
-
parts.push(sql.raw(` ORDER BY ${options.orderBy} ${dir}`));
|
|
1152
|
+
parts.push(sql.raw(` ORDER BY ${quoteIdent2(options.orderBy)} ${dir}`));
|
|
966
1153
|
}
|
|
967
1154
|
if (options?.limit !== void 0) {
|
|
968
1155
|
parts.push(sql` LIMIT ${options.limit}`);
|
|
@@ -978,7 +1165,7 @@ var PostgresServerStore = class {
|
|
|
978
1165
|
conditions.push(sql.raw("_deleted = 0"));
|
|
979
1166
|
}
|
|
980
1167
|
for (const [key, value] of Object.entries(where)) {
|
|
981
|
-
conditions.push(sql`${sql.raw(key)} = ${value}`);
|
|
1168
|
+
conditions.push(sql`${sql.raw(quoteIdent2(key))} = ${value}`);
|
|
982
1169
|
}
|
|
983
1170
|
if (conditions.length === 0) {
|
|
984
1171
|
return sql.raw("1 = 1");
|
|
@@ -1051,44 +1238,92 @@ var PostgresServerStore = class {
|
|
|
1051
1238
|
}
|
|
1052
1239
|
}
|
|
1053
1240
|
async ensureTables() {
|
|
1054
|
-
await this.db.
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1241
|
+
await this.db.transaction(async (tx) => {
|
|
1242
|
+
await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended('kora:ensure-tables', 0))`);
|
|
1243
|
+
await tx.execute(sql`
|
|
1244
|
+
CREATE TABLE IF NOT EXISTS operations (
|
|
1245
|
+
id TEXT PRIMARY KEY,
|
|
1246
|
+
node_id TEXT NOT NULL,
|
|
1247
|
+
type TEXT NOT NULL,
|
|
1248
|
+
collection TEXT NOT NULL,
|
|
1249
|
+
record_id TEXT NOT NULL,
|
|
1250
|
+
data TEXT,
|
|
1251
|
+
previous_data TEXT,
|
|
1252
|
+
atomic_ops TEXT,
|
|
1253
|
+
wall_time BIGINT NOT NULL,
|
|
1254
|
+
logical INTEGER NOT NULL,
|
|
1255
|
+
timestamp_node_id TEXT NOT NULL,
|
|
1256
|
+
sequence_number INTEGER NOT NULL,
|
|
1257
|
+
causal_deps TEXT NOT NULL DEFAULT '[]',
|
|
1258
|
+
schema_version INTEGER NOT NULL,
|
|
1259
|
+
received_at BIGINT NOT NULL
|
|
1260
|
+
)
|
|
1261
|
+
`);
|
|
1262
|
+
await tx.execute(sql`ALTER TABLE operations ADD COLUMN IF NOT EXISTS atomic_ops TEXT`);
|
|
1263
|
+
await tx.execute(sql`ALTER TABLE operations ADD COLUMN IF NOT EXISTS delivery_seq BIGINT`);
|
|
1264
|
+
await tx.execute(
|
|
1265
|
+
sql`CREATE INDEX IF NOT EXISTS idx_node_seq ON operations (node_id, sequence_number)`
|
|
1266
|
+
);
|
|
1267
|
+
await tx.execute(sql`CREATE INDEX IF NOT EXISTS idx_collection ON operations (collection)`);
|
|
1268
|
+
await tx.execute(sql`CREATE INDEX IF NOT EXISTS idx_received ON operations (received_at)`);
|
|
1269
|
+
await tx.execute(
|
|
1270
|
+
sql`CREATE INDEX IF NOT EXISTS idx_delivery_seq ON operations (delivery_seq)`
|
|
1271
|
+
);
|
|
1272
|
+
await tx.execute(
|
|
1273
|
+
sql`CREATE INDEX IF NOT EXISTS idx_collection_record ON operations (collection, record_id)`
|
|
1274
|
+
);
|
|
1275
|
+
await tx.execute(sql`
|
|
1276
|
+
CREATE TABLE IF NOT EXISTS sync_state (
|
|
1277
|
+
node_id TEXT PRIMARY KEY,
|
|
1278
|
+
max_sequence_number INTEGER NOT NULL,
|
|
1279
|
+
last_seen_at BIGINT NOT NULL
|
|
1280
|
+
)
|
|
1281
|
+
`);
|
|
1282
|
+
await tx.execute(sql`
|
|
1283
|
+
CREATE TABLE IF NOT EXISTS delivery_counter (
|
|
1284
|
+
id INTEGER PRIMARY KEY,
|
|
1285
|
+
value BIGINT NOT NULL
|
|
1286
|
+
)
|
|
1287
|
+
`);
|
|
1288
|
+
await tx.execute(sql`
|
|
1289
|
+
WITH ordered AS (
|
|
1290
|
+
SELECT id, ROW_NUMBER() OVER (
|
|
1291
|
+
ORDER BY received_at ASC, sequence_number ASC, id ASC
|
|
1292
|
+
) + COALESCE((SELECT MAX(delivery_seq) FROM operations), 0) AS rn
|
|
1293
|
+
FROM operations WHERE delivery_seq IS NULL
|
|
1294
|
+
)
|
|
1295
|
+
UPDATE operations o SET delivery_seq = ordered.rn
|
|
1296
|
+
FROM ordered WHERE o.id = ordered.id
|
|
1297
|
+
`);
|
|
1298
|
+
await tx.execute(sql`
|
|
1299
|
+
INSERT INTO delivery_counter (id, value)
|
|
1300
|
+
VALUES (1, COALESCE((SELECT MAX(delivery_seq) FROM operations), 0))
|
|
1301
|
+
ON CONFLICT (id) DO UPDATE
|
|
1302
|
+
SET value = GREATEST(delivery_counter.value, EXCLUDED.value)
|
|
1303
|
+
`);
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* Reserve the next delivery sequence inside an open append transaction. The
|
|
1308
|
+
* `UPDATE ... RETURNING` locks the counter row until this transaction commits,
|
|
1309
|
+
* serializing assignment into commit order (see the counter table comment).
|
|
1310
|
+
*/
|
|
1311
|
+
async nextDeliverySeq(tx) {
|
|
1312
|
+
const rows = await tx.execute(
|
|
1313
|
+
sql`UPDATE delivery_counter SET value = value + 1 WHERE id = 1 RETURNING value`
|
|
1079
1314
|
);
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1315
|
+
const value = rows[0]?.value;
|
|
1316
|
+
if (value === void 0 || value === null) {
|
|
1317
|
+
throw new Error(
|
|
1318
|
+
"delivery_counter row (id=1) is missing; the operations log cannot assign delivery sequences"
|
|
1319
|
+
);
|
|
1320
|
+
}
|
|
1321
|
+
return Number(value);
|
|
1087
1322
|
}
|
|
1088
1323
|
// ---------------------------------------------------------------------------
|
|
1089
1324
|
// Operation serialization
|
|
1090
1325
|
// ---------------------------------------------------------------------------
|
|
1091
|
-
serializeOperation(op, receivedAt) {
|
|
1326
|
+
serializeOperation(op, receivedAt, deliverySeq) {
|
|
1092
1327
|
return {
|
|
1093
1328
|
id: op.id,
|
|
1094
1329
|
nodeId: op.nodeId,
|
|
@@ -1097,16 +1332,19 @@ var PostgresServerStore = class {
|
|
|
1097
1332
|
recordId: op.recordId,
|
|
1098
1333
|
data: op.data !== null ? JSON.stringify(op.data) : null,
|
|
1099
1334
|
previousData: op.previousData !== null ? JSON.stringify(op.previousData) : null,
|
|
1335
|
+
atomicOps: op.atomicOps && Object.keys(op.atomicOps).length > 0 ? JSON.stringify(op.atomicOps) : null,
|
|
1100
1336
|
wallTime: op.timestamp.wallTime,
|
|
1101
1337
|
logical: op.timestamp.logical,
|
|
1102
1338
|
timestampNodeId: op.timestamp.nodeId,
|
|
1103
1339
|
sequenceNumber: op.sequenceNumber,
|
|
1104
1340
|
causalDeps: JSON.stringify(op.causalDeps),
|
|
1105
1341
|
schemaVersion: op.schemaVersion,
|
|
1106
|
-
receivedAt
|
|
1342
|
+
receivedAt,
|
|
1343
|
+
deliverySeq
|
|
1107
1344
|
};
|
|
1108
1345
|
}
|
|
1109
1346
|
deserializeOperation(row) {
|
|
1347
|
+
const atomicOps = row.atomicOps != null ? JSON.parse(row.atomicOps) : void 0;
|
|
1110
1348
|
return {
|
|
1111
1349
|
id: row.id,
|
|
1112
1350
|
nodeId: row.nodeId,
|
|
@@ -1122,7 +1360,8 @@ var PostgresServerStore = class {
|
|
|
1122
1360
|
},
|
|
1123
1361
|
sequenceNumber: row.sequenceNumber,
|
|
1124
1362
|
causalDeps: JSON.parse(row.causalDeps),
|
|
1125
|
-
schemaVersion: row.schemaVersion
|
|
1363
|
+
schemaVersion: row.schemaVersion,
|
|
1364
|
+
...atomicOps ? { atomicOps } : {}
|
|
1126
1365
|
};
|
|
1127
1366
|
}
|
|
1128
1367
|
// ---------------------------------------------------------------------------
|
|
@@ -1173,8 +1412,8 @@ async function loadPostgresDeps() {
|
|
|
1173
1412
|
|
|
1174
1413
|
// src/store/sqlite-server-store.ts
|
|
1175
1414
|
import { createRequire } from "module";
|
|
1176
|
-
import { generateUUIDv7 as generateUUIDv73 } from "@korajs/core";
|
|
1177
|
-
import { and as and2, asc as asc2, between as between2, count as count2, eq as eq2, sql as sql2 } from "drizzle-orm";
|
|
1415
|
+
import { generateUUIDv7 as generateUUIDv73, quoteIdent as quoteIdent3 } from "@korajs/core";
|
|
1416
|
+
import { and as and2, asc as asc2, between as between2, count as count2, eq as eq2, gt as gt2, sql as sql2 } from "drizzle-orm";
|
|
1178
1417
|
|
|
1179
1418
|
// src/store/drizzle-schema.ts
|
|
1180
1419
|
import { index as index2, integer as integer2, sqliteTable, text as text2 } from "drizzle-orm/sqlite-core";
|
|
@@ -1190,6 +1429,8 @@ var operations = sqliteTable(
|
|
|
1190
1429
|
// JSON-serialized, null for deletes
|
|
1191
1430
|
previousData: text2("previous_data"),
|
|
1192
1431
|
// JSON-serialized, null for insert/delete
|
|
1432
|
+
atomicOps: text2("atomic_ops"),
|
|
1433
|
+
// JSON-serialized Record<field, AtomicOp>, null when none
|
|
1193
1434
|
wallTime: integer2("wall_time").notNull(),
|
|
1194
1435
|
logical: integer2("logical").notNull(),
|
|
1195
1436
|
timestampNodeId: text2("timestamp_node_id").notNull(),
|
|
@@ -1197,12 +1438,17 @@ var operations = sqliteTable(
|
|
|
1197
1438
|
causalDeps: text2("causal_deps").notNull().default("[]"),
|
|
1198
1439
|
// JSON array of op IDs
|
|
1199
1440
|
schemaVersion: integer2("schema_version").notNull(),
|
|
1200
|
-
receivedAt: integer2("received_at").notNull()
|
|
1441
|
+
receivedAt: integer2("received_at").notNull(),
|
|
1442
|
+
// Server-assigned monotonic delivery sequence, ordered by commit. Drives the
|
|
1443
|
+
// gap-free server->client delivery watermark. Nullable only for rows written
|
|
1444
|
+
// before the column existed; those are backfilled on startup.
|
|
1445
|
+
deliverySeq: integer2("delivery_seq")
|
|
1201
1446
|
},
|
|
1202
1447
|
(table) => ({
|
|
1203
1448
|
nodeSeqIdx: index2("idx_node_seq").on(table.nodeId, table.sequenceNumber),
|
|
1204
1449
|
collectionIdx: index2("idx_collection").on(table.collection),
|
|
1205
|
-
receivedIdx: index2("idx_received").on(table.receivedAt)
|
|
1450
|
+
receivedIdx: index2("idx_received").on(table.receivedAt),
|
|
1451
|
+
deliveryIdx: index2("idx_delivery_seq").on(table.deliverySeq)
|
|
1206
1452
|
})
|
|
1207
1453
|
);
|
|
1208
1454
|
var syncState = sqliteTable("sync_state", {
|
|
@@ -1210,6 +1456,10 @@ var syncState = sqliteTable("sync_state", {
|
|
|
1210
1456
|
maxSequenceNumber: integer2("max_sequence_number").notNull(),
|
|
1211
1457
|
lastSeenAt: integer2("last_seen_at").notNull()
|
|
1212
1458
|
});
|
|
1459
|
+
var deliveryCounter = sqliteTable("delivery_counter", {
|
|
1460
|
+
id: integer2("id").primaryKey(),
|
|
1461
|
+
value: integer2("value").notNull()
|
|
1462
|
+
});
|
|
1213
1463
|
|
|
1214
1464
|
// src/store/sqlite-server-store.ts
|
|
1215
1465
|
var esmRequire = createRequire(import.meta.url);
|
|
@@ -1263,12 +1513,15 @@ var SqliteServerStore = class {
|
|
|
1263
1513
|
async applyRemoteOperation(op) {
|
|
1264
1514
|
this.assertOpen();
|
|
1265
1515
|
const now = Date.now();
|
|
1266
|
-
const row = this.serializeOperation(op, now);
|
|
1267
1516
|
const result = this.db.transaction((tx) => {
|
|
1268
|
-
const
|
|
1269
|
-
|
|
1517
|
+
const existing = tx.all(
|
|
1518
|
+
sql2`SELECT id FROM operations WHERE id = ${op.id} LIMIT 1`
|
|
1519
|
+
);
|
|
1520
|
+
if (existing.length > 0) {
|
|
1270
1521
|
return "duplicate";
|
|
1271
1522
|
}
|
|
1523
|
+
const row = this.serializeOperation(op, now, this.nextDeliverySeq(tx));
|
|
1524
|
+
tx.insert(operations).values(row).run();
|
|
1272
1525
|
tx.insert(syncState).values({
|
|
1273
1526
|
nodeId: op.nodeId,
|
|
1274
1527
|
maxSequenceNumber: op.sequenceNumber,
|
|
@@ -1297,6 +1550,21 @@ var SqliteServerStore = class {
|
|
|
1297
1550
|
const result = this.db.select({ value: count2() }).from(operations).all();
|
|
1298
1551
|
return result[0]?.value ?? 0;
|
|
1299
1552
|
}
|
|
1553
|
+
async getMaxDeliverySequence() {
|
|
1554
|
+
this.assertOpen();
|
|
1555
|
+
const rows = this.db.all(
|
|
1556
|
+
sql2`SELECT MAX(delivery_seq) AS m FROM operations`
|
|
1557
|
+
);
|
|
1558
|
+
return rows[0]?.m ?? 0;
|
|
1559
|
+
}
|
|
1560
|
+
async getOperationsAfterDelivery(afterDeliverySequence, limit) {
|
|
1561
|
+
this.assertOpen();
|
|
1562
|
+
const rows = this.db.select().from(operations).where(gt2(operations.deliverySeq, afterDeliverySequence)).orderBy(asc2(operations.deliverySeq)).limit(limit).all();
|
|
1563
|
+
return rows.map((row) => ({
|
|
1564
|
+
operation: this.deserializeOperation(row),
|
|
1565
|
+
deliverySequence: row.deliverySeq ?? 0
|
|
1566
|
+
}));
|
|
1567
|
+
}
|
|
1300
1568
|
async materializeCollection(collection) {
|
|
1301
1569
|
this.assertOpen();
|
|
1302
1570
|
if (this.schema?.collections[collection]) {
|
|
@@ -1328,7 +1596,7 @@ var SqliteServerStore = class {
|
|
|
1328
1596
|
this.assertCollection(collection);
|
|
1329
1597
|
const schema = this.schema;
|
|
1330
1598
|
const collectionDef = schema.collections[collection];
|
|
1331
|
-
const query = sql2`SELECT * FROM ${sql2.raw(collection)} WHERE id = ${id} AND _deleted = 0`;
|
|
1599
|
+
const query = sql2`SELECT * FROM ${sql2.raw(quoteIdent3(collection))} WHERE id = ${id} AND _deleted = 0`;
|
|
1332
1600
|
const rows = this.db.all(query);
|
|
1333
1601
|
if (rows.length === 0) return null;
|
|
1334
1602
|
return this.deserializeRow(rows[0], collectionDef);
|
|
@@ -1344,7 +1612,7 @@ var SqliteServerStore = class {
|
|
|
1344
1612
|
}
|
|
1345
1613
|
}
|
|
1346
1614
|
const whereClause = this.buildWhereClause(where ?? {}, false);
|
|
1347
|
-
const query = sql2`SELECT COUNT(*) as cnt FROM ${sql2.raw(collection)} WHERE ${whereClause}`;
|
|
1615
|
+
const query = sql2`SELECT COUNT(*) as cnt FROM ${sql2.raw(quoteIdent3(collection))} WHERE ${whereClause}`;
|
|
1348
1616
|
const rows = this.db.all(query);
|
|
1349
1617
|
return rows[0]?.cnt ?? 0;
|
|
1350
1618
|
}
|
|
@@ -1354,7 +1622,7 @@ var SqliteServerStore = class {
|
|
|
1354
1622
|
async exportBackup() {
|
|
1355
1623
|
this.assertOpen();
|
|
1356
1624
|
const { buildServerBackup } = await import("./server-backup-MOICK6MI.js");
|
|
1357
|
-
const rows = this.db.select().from(operations).all();
|
|
1625
|
+
const rows = this.db.select().from(operations).orderBy(asc2(operations.deliverySeq)).all();
|
|
1358
1626
|
const deserialized = rows.map((row) => this.deserializeOperation(row));
|
|
1359
1627
|
const vv = this.getVersionVector();
|
|
1360
1628
|
return buildServerBackup(this.nodeId, deserialized, vv);
|
|
@@ -1374,13 +1642,17 @@ var SqliteServerStore = class {
|
|
|
1374
1642
|
this.db.transaction((tx) => {
|
|
1375
1643
|
tx.run(sql2.raw("DELETE FROM operations"));
|
|
1376
1644
|
tx.run(sql2.raw("DELETE FROM sync_state"));
|
|
1645
|
+
tx.update(deliveryCounter).set({ value: 0 }).where(eq2(deliveryCounter.id, 1)).run();
|
|
1377
1646
|
for (const [nid, seq] of versionVector) {
|
|
1378
1647
|
tx.insert(syncState).values({ nodeId: nid, maxSequenceNumber: seq, lastSeenAt: Date.now() }).run();
|
|
1379
1648
|
}
|
|
1649
|
+
let deliverySeq = 0;
|
|
1380
1650
|
for (const op of ops) {
|
|
1381
|
-
|
|
1651
|
+
deliverySeq += 1;
|
|
1652
|
+
const row = this.serializeOperation(op, Date.now(), deliverySeq);
|
|
1382
1653
|
tx.insert(operations).values(row).run();
|
|
1383
1654
|
}
|
|
1655
|
+
tx.update(deliveryCounter).set({ value: deliverySeq }).where(eq2(deliveryCounter.id, 1)).run();
|
|
1384
1656
|
});
|
|
1385
1657
|
return { operationsRestored: ops.length, success: true };
|
|
1386
1658
|
}
|
|
@@ -1398,11 +1670,13 @@ var SqliteServerStore = class {
|
|
|
1398
1670
|
const ops = txOrDb.select({
|
|
1399
1671
|
type: operations.type,
|
|
1400
1672
|
data: operations.data,
|
|
1673
|
+
atomicOps: operations.atomicOps,
|
|
1401
1674
|
wallTime: operations.wallTime
|
|
1402
|
-
}).from(operations).where(and2(eq2(operations.collection, collection), eq2(operations.recordId, recordId))).orderBy(asc2(operations.wallTime), asc2(operations.logical), asc2(operations.
|
|
1675
|
+
}).from(operations).where(and2(eq2(operations.collection, collection), eq2(operations.recordId, recordId))).orderBy(asc2(operations.wallTime), asc2(operations.logical), asc2(operations.timestampNodeId)).all();
|
|
1403
1676
|
const parsedOps = ops.map((op) => ({
|
|
1404
1677
|
type: op.type,
|
|
1405
|
-
data: op.data !== null ? JSON.parse(op.data) : null
|
|
1678
|
+
data: op.data !== null ? JSON.parse(op.data) : null,
|
|
1679
|
+
atomicOps: op.atomicOps != null ? JSON.parse(op.atomicOps) : null
|
|
1406
1680
|
}));
|
|
1407
1681
|
const recordData = replayOperationsForRecord(parsedOps);
|
|
1408
1682
|
const fieldNames = Object.keys(collectionDef.fields);
|
|
@@ -1421,7 +1695,7 @@ var SqliteServerStore = class {
|
|
|
1421
1695
|
);
|
|
1422
1696
|
} else {
|
|
1423
1697
|
txOrDb.run(
|
|
1424
|
-
sql2`UPDATE ${sql2.raw(collection)} SET _deleted = 1, _updated_at = ${Date.now()} WHERE id = ${recordId}`
|
|
1698
|
+
sql2`UPDATE ${sql2.raw(quoteIdent3(collection))} SET _deleted = 1, _updated_at = ${Date.now()} WHERE id = ${recordId}`
|
|
1425
1699
|
);
|
|
1426
1700
|
}
|
|
1427
1701
|
}
|
|
@@ -1442,16 +1716,16 @@ var SqliteServerStore = class {
|
|
|
1442
1716
|
0
|
|
1443
1717
|
// _deleted = false
|
|
1444
1718
|
];
|
|
1445
|
-
const columnsSql = sql2.raw(allColumns.join(", "));
|
|
1719
|
+
const columnsSql = sql2.raw(allColumns.map((c) => quoteIdent3(c)).join(", "));
|
|
1446
1720
|
const valuesSql = sql2.join(
|
|
1447
1721
|
values.map((v) => sql2`${v}`),
|
|
1448
1722
|
sql2.raw(", ")
|
|
1449
1723
|
);
|
|
1450
1724
|
const updateSet = sql2.raw(
|
|
1451
|
-
allColumns.slice(1).map((c) => `${c} = excluded.${c}`).join(", ")
|
|
1725
|
+
allColumns.slice(1).map((c) => `${quoteIdent3(c)} = excluded.${quoteIdent3(c)}`).join(", ")
|
|
1452
1726
|
);
|
|
1453
1727
|
txOrDb.run(
|
|
1454
|
-
sql2`INSERT INTO ${sql2.raw(tableName)} (${columnsSql}) VALUES (${valuesSql}) ON CONFLICT (id) DO UPDATE SET ${updateSet}`
|
|
1728
|
+
sql2`INSERT INTO ${sql2.raw(quoteIdent3(tableName))} (${columnsSql}) VALUES (${valuesSql}) ON CONFLICT (id) DO UPDATE SET ${updateSet}`
|
|
1455
1729
|
);
|
|
1456
1730
|
}
|
|
1457
1731
|
/**
|
|
@@ -1474,8 +1748,9 @@ var SqliteServerStore = class {
|
|
|
1474
1748
|
recordId: operations.recordId,
|
|
1475
1749
|
type: operations.type,
|
|
1476
1750
|
data: operations.data,
|
|
1751
|
+
atomicOps: operations.atomicOps,
|
|
1477
1752
|
wallTime: operations.wallTime
|
|
1478
|
-
}).from(operations).where(eq2(operations.collection, collectionName)).orderBy(asc2(operations.wallTime), asc2(operations.logical), asc2(operations.
|
|
1753
|
+
}).from(operations).where(eq2(operations.collection, collectionName)).orderBy(asc2(operations.wallTime), asc2(operations.logical), asc2(operations.timestampNodeId)).all();
|
|
1479
1754
|
if (allOps.length === 0) return;
|
|
1480
1755
|
const grouped = /* @__PURE__ */ new Map();
|
|
1481
1756
|
for (const op of allOps) {
|
|
@@ -1491,7 +1766,8 @@ var SqliteServerStore = class {
|
|
|
1491
1766
|
for (const [recordId, recordOps] of grouped) {
|
|
1492
1767
|
const parsedOps = recordOps.map((op) => ({
|
|
1493
1768
|
type: op.type,
|
|
1494
|
-
data: op.data !== null ? JSON.parse(op.data) : null
|
|
1769
|
+
data: op.data !== null ? JSON.parse(op.data) : null,
|
|
1770
|
+
atomicOps: op.atomicOps != null ? JSON.parse(op.atomicOps) : null
|
|
1495
1771
|
}));
|
|
1496
1772
|
const recordData = replayOperationsForRecord(parsedOps);
|
|
1497
1773
|
if (recordData) {
|
|
@@ -1509,7 +1785,7 @@ var SqliteServerStore = class {
|
|
|
1509
1785
|
);
|
|
1510
1786
|
} else {
|
|
1511
1787
|
tx.run(
|
|
1512
|
-
sql2`INSERT INTO ${sql2.raw(collectionName)} (id, _deleted, _created_at, _updated_at) VALUES (${recordId}, 1, ${Date.now()}, ${Date.now()}) ON CONFLICT (id) DO UPDATE SET _deleted = 1, _updated_at = ${Date.now()}`
|
|
1788
|
+
sql2`INSERT INTO ${sql2.raw(quoteIdent3(collectionName))} (id, _deleted, _created_at, _updated_at) VALUES (${recordId}, 1, ${Date.now()}, ${Date.now()}) ON CONFLICT (id) DO UPDATE SET _deleted = 1, _updated_at = ${Date.now()}`
|
|
1513
1789
|
);
|
|
1514
1790
|
}
|
|
1515
1791
|
}
|
|
@@ -1523,10 +1799,12 @@ var SqliteServerStore = class {
|
|
|
1523
1799
|
options?.where ?? {},
|
|
1524
1800
|
options?.includeDeleted ?? false
|
|
1525
1801
|
);
|
|
1526
|
-
const parts = [
|
|
1802
|
+
const parts = [
|
|
1803
|
+
sql2`SELECT * FROM ${sql2.raw(quoteIdent3(collection))} WHERE ${whereClause}`
|
|
1804
|
+
];
|
|
1527
1805
|
if (options?.orderBy) {
|
|
1528
1806
|
const dir = options.orderDirection === "desc" ? "DESC" : "ASC";
|
|
1529
|
-
parts.push(sql2.raw(` ORDER BY ${options.orderBy} ${dir}`));
|
|
1807
|
+
parts.push(sql2.raw(` ORDER BY ${quoteIdent3(options.orderBy)} ${dir}`));
|
|
1530
1808
|
}
|
|
1531
1809
|
if (options?.limit !== void 0) {
|
|
1532
1810
|
parts.push(sql2` LIMIT ${options.limit}`);
|
|
@@ -1542,7 +1820,7 @@ var SqliteServerStore = class {
|
|
|
1542
1820
|
conditions.push(sql2.raw("_deleted = 0"));
|
|
1543
1821
|
}
|
|
1544
1822
|
for (const [key, value] of Object.entries(where)) {
|
|
1545
|
-
conditions.push(sql2`${sql2.raw(key)} = ${value}`);
|
|
1823
|
+
conditions.push(sql2`${sql2.raw(quoteIdent3(key))} = ${value}`);
|
|
1546
1824
|
}
|
|
1547
1825
|
if (conditions.length === 0) {
|
|
1548
1826
|
return sql2.raw("1 = 1");
|
|
@@ -1613,6 +1891,7 @@ var SqliteServerStore = class {
|
|
|
1613
1891
|
record_id TEXT NOT NULL,
|
|
1614
1892
|
data TEXT,
|
|
1615
1893
|
previous_data TEXT,
|
|
1894
|
+
atomic_ops TEXT,
|
|
1616
1895
|
wall_time INTEGER NOT NULL,
|
|
1617
1896
|
logical INTEGER NOT NULL,
|
|
1618
1897
|
timestamp_node_id TEXT NOT NULL,
|
|
@@ -1622,9 +1901,35 @@ var SqliteServerStore = class {
|
|
|
1622
1901
|
received_at INTEGER NOT NULL
|
|
1623
1902
|
)
|
|
1624
1903
|
`);
|
|
1904
|
+
try {
|
|
1905
|
+
this.db.run(sql2`ALTER TABLE operations ADD COLUMN atomic_ops TEXT`);
|
|
1906
|
+
} catch (e) {
|
|
1907
|
+
const msg = e instanceof Error ? e.message : "";
|
|
1908
|
+
const causeMsg = e instanceof Error && e.cause instanceof Error ? e.cause.message : "";
|
|
1909
|
+
if (!msg.includes("duplicate column") && !causeMsg.includes("duplicate column")) {
|
|
1910
|
+
throw e;
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
try {
|
|
1914
|
+
this.db.run(sql2`ALTER TABLE operations ADD COLUMN delivery_seq INTEGER`);
|
|
1915
|
+
} catch (e) {
|
|
1916
|
+
const msg = e instanceof Error ? e.message : "";
|
|
1917
|
+
const causeMsg = e instanceof Error && e.cause instanceof Error ? e.cause.message : "";
|
|
1918
|
+
if (!msg.includes("duplicate column") && !causeMsg.includes("duplicate column")) {
|
|
1919
|
+
throw e;
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1625
1922
|
this.db.run(sql2`
|
|
1626
1923
|
CREATE INDEX IF NOT EXISTS idx_node_seq ON operations (node_id, sequence_number)
|
|
1627
1924
|
`);
|
|
1925
|
+
this.db.run(sql2`
|
|
1926
|
+
CREATE INDEX IF NOT EXISTS idx_delivery_seq ON operations (delivery_seq)
|
|
1927
|
+
`);
|
|
1928
|
+
this.db.run(sql2`
|
|
1929
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_delivery_seq_unique
|
|
1930
|
+
ON operations (delivery_seq)
|
|
1931
|
+
WHERE delivery_seq IS NOT NULL
|
|
1932
|
+
`);
|
|
1628
1933
|
this.db.run(sql2`
|
|
1629
1934
|
CREATE INDEX IF NOT EXISTS idx_collection ON operations (collection)
|
|
1630
1935
|
`);
|
|
@@ -1641,11 +1946,60 @@ var SqliteServerStore = class {
|
|
|
1641
1946
|
last_seen_at INTEGER NOT NULL
|
|
1642
1947
|
)
|
|
1643
1948
|
`);
|
|
1949
|
+
this.db.run(sql2`
|
|
1950
|
+
CREATE TABLE IF NOT EXISTS delivery_counter (
|
|
1951
|
+
id INTEGER PRIMARY KEY,
|
|
1952
|
+
value INTEGER NOT NULL
|
|
1953
|
+
)
|
|
1954
|
+
`);
|
|
1955
|
+
this.backfillDeliverySequence();
|
|
1956
|
+
}
|
|
1957
|
+
/**
|
|
1958
|
+
* Assign delivery sequences to any operations written before the column existed,
|
|
1959
|
+
* then seed the durable counter from the current max. Ordered by
|
|
1960
|
+
* (received_at, sequence_number, id) so the backfill is deterministic and stable
|
|
1961
|
+
* across restarts. A one-time migration: after it runs, all rows have a value.
|
|
1962
|
+
*/
|
|
1963
|
+
backfillDeliverySequence() {
|
|
1964
|
+
this.db.transaction((tx) => {
|
|
1965
|
+
const maxRow = tx.all(
|
|
1966
|
+
sql2`SELECT MAX(delivery_seq) AS m FROM operations`
|
|
1967
|
+
);
|
|
1968
|
+
let next = maxRow[0]?.m ?? 0;
|
|
1969
|
+
const nullRows = tx.all(
|
|
1970
|
+
sql2`SELECT id FROM operations WHERE delivery_seq IS NULL ORDER BY received_at ASC, sequence_number ASC, id ASC`
|
|
1971
|
+
);
|
|
1972
|
+
for (const row of nullRows) {
|
|
1973
|
+
next += 1;
|
|
1974
|
+
tx.run(sql2`UPDATE operations SET delivery_seq = ${next} WHERE id = ${row.id}`);
|
|
1975
|
+
}
|
|
1976
|
+
tx.insert(deliveryCounter).values({ id: 1, value: next }).onConflictDoUpdate({
|
|
1977
|
+
target: deliveryCounter.id,
|
|
1978
|
+
set: {
|
|
1979
|
+
value: sql2`MAX(${deliveryCounter.value}, ${next})`
|
|
1980
|
+
}
|
|
1981
|
+
}).run();
|
|
1982
|
+
});
|
|
1983
|
+
}
|
|
1984
|
+
/**
|
|
1985
|
+
* Reserve the next delivery sequence inside the append transaction. SQLite locks
|
|
1986
|
+
* the database for the write transaction, and this counter row keeps sequence
|
|
1987
|
+
* allocation shared by every SqliteServerStore instance using the same file.
|
|
1988
|
+
*/
|
|
1989
|
+
nextDeliverySeq(tx) {
|
|
1990
|
+
const rows = tx.update(deliveryCounter).set({ value: sql2`${deliveryCounter.value} + 1` }).where(eq2(deliveryCounter.id, 1)).returning({ value: deliveryCounter.value }).all();
|
|
1991
|
+
const value = rows[0]?.value;
|
|
1992
|
+
if (value === void 0 || value === null) {
|
|
1993
|
+
throw new Error(
|
|
1994
|
+
"delivery_counter row (id=1) is missing; the operations log cannot assign delivery sequences"
|
|
1995
|
+
);
|
|
1996
|
+
}
|
|
1997
|
+
return value;
|
|
1644
1998
|
}
|
|
1645
1999
|
// ---------------------------------------------------------------------------
|
|
1646
2000
|
// Operation serialization
|
|
1647
2001
|
// ---------------------------------------------------------------------------
|
|
1648
|
-
serializeOperation(op, receivedAt) {
|
|
2002
|
+
serializeOperation(op, receivedAt, deliverySeq) {
|
|
1649
2003
|
return {
|
|
1650
2004
|
id: op.id,
|
|
1651
2005
|
nodeId: op.nodeId,
|
|
@@ -1654,16 +2008,19 @@ var SqliteServerStore = class {
|
|
|
1654
2008
|
recordId: op.recordId,
|
|
1655
2009
|
data: op.data !== null ? JSON.stringify(op.data) : null,
|
|
1656
2010
|
previousData: op.previousData !== null ? JSON.stringify(op.previousData) : null,
|
|
2011
|
+
atomicOps: op.atomicOps && Object.keys(op.atomicOps).length > 0 ? JSON.stringify(op.atomicOps) : null,
|
|
1657
2012
|
wallTime: op.timestamp.wallTime,
|
|
1658
2013
|
logical: op.timestamp.logical,
|
|
1659
2014
|
timestampNodeId: op.timestamp.nodeId,
|
|
1660
2015
|
sequenceNumber: op.sequenceNumber,
|
|
1661
2016
|
causalDeps: JSON.stringify(op.causalDeps),
|
|
1662
2017
|
schemaVersion: op.schemaVersion,
|
|
1663
|
-
receivedAt
|
|
2018
|
+
receivedAt,
|
|
2019
|
+
deliverySeq
|
|
1664
2020
|
};
|
|
1665
2021
|
}
|
|
1666
2022
|
deserializeOperation(row) {
|
|
2023
|
+
const atomicOps = row.atomicOps != null ? JSON.parse(row.atomicOps) : void 0;
|
|
1667
2024
|
return {
|
|
1668
2025
|
id: row.id,
|
|
1669
2026
|
nodeId: row.nodeId,
|
|
@@ -1679,7 +2036,8 @@ var SqliteServerStore = class {
|
|
|
1679
2036
|
},
|
|
1680
2037
|
sequenceNumber: row.sequenceNumber,
|
|
1681
2038
|
causalDeps: JSON.parse(row.causalDeps),
|
|
1682
|
-
schemaVersion: row.schemaVersion
|
|
2039
|
+
schemaVersion: row.schemaVersion,
|
|
2040
|
+
...atomicOps ? { atomicOps } : {}
|
|
1683
2041
|
};
|
|
1684
2042
|
}
|
|
1685
2043
|
// ---------------------------------------------------------------------------
|
|
@@ -1861,8 +2219,10 @@ var WsServerTransport = class {
|
|
|
1861
2219
|
};
|
|
1862
2220
|
|
|
1863
2221
|
// src/session/client-session.ts
|
|
1864
|
-
import {
|
|
2222
|
+
import { applyOperationTransforms } from "@korajs/core";
|
|
2223
|
+
import { generateUUIDv7 as generateUUIDv74, hashBlob } from "@korajs/core";
|
|
1865
2224
|
import { topologicalSort } from "@korajs/core/internal";
|
|
2225
|
+
import { decodeBlobChunkBytes } from "@korajs/sync";
|
|
1866
2226
|
import {
|
|
1867
2227
|
NegotiatedMessageSerializer,
|
|
1868
2228
|
SCHEMA_MISMATCH_PREFIX,
|
|
@@ -1960,10 +2320,10 @@ function createServerReferentialContext(store) {
|
|
|
1960
2320
|
}
|
|
1961
2321
|
|
|
1962
2322
|
// src/apply/server-side-effect-operation.ts
|
|
1963
|
-
import { HybridLogicalClock, createOperation } from "@korajs/core";
|
|
2323
|
+
import { HybridLogicalClock as HybridLogicalClock2, createOperation } from "@korajs/core";
|
|
1964
2324
|
async function createServerSideEffectOperation(store, parentOp, effect, schemaVersion, sequenceNumber) {
|
|
1965
2325
|
const nodeId = store.getNodeId();
|
|
1966
|
-
const clock = new
|
|
2326
|
+
const clock = new HybridLogicalClock2(nodeId);
|
|
1967
2327
|
clock.receive(parentOp.timestamp);
|
|
1968
2328
|
return createOperation(
|
|
1969
2329
|
{
|
|
@@ -1981,6 +2341,9 @@ async function createServerSideEffectOperation(store, parentOp, effect, schemaVe
|
|
|
1981
2341
|
);
|
|
1982
2342
|
}
|
|
1983
2343
|
function nextServerSequenceNumber(store) {
|
|
2344
|
+
if (store.reserveSequenceNumber) {
|
|
2345
|
+
return store.reserveSequenceNumber();
|
|
2346
|
+
}
|
|
1984
2347
|
const nodeId = store.getNodeId();
|
|
1985
2348
|
const current = store.getVersionVector().get(nodeId) ?? 0;
|
|
1986
2349
|
return current + 1;
|
|
@@ -1990,14 +2353,28 @@ function nextServerSequenceNumber(store) {
|
|
|
1990
2353
|
async function applyServerOperation(store, op, relationLookup) {
|
|
1991
2354
|
const schema = store.getSchema();
|
|
1992
2355
|
const lookup = relationLookup ?? (schema ? buildMergeRelationLookup(schema) : /* @__PURE__ */ new Map());
|
|
2356
|
+
const shapeCheck = validateOperationShape(op, schema);
|
|
2357
|
+
if (!shapeCheck.valid) {
|
|
2358
|
+
return {
|
|
2359
|
+
result: "skipped",
|
|
2360
|
+
appliedOperations: [],
|
|
2361
|
+
rejection: {
|
|
2362
|
+
code: shapeCheck.code ?? "SCHEMA_VALIDATION_ERROR",
|
|
2363
|
+
message: shapeCheck.message ?? `Operation "${op.id}" does not match the server schema`,
|
|
2364
|
+
retriable: false
|
|
2365
|
+
}
|
|
2366
|
+
};
|
|
2367
|
+
}
|
|
1993
2368
|
const constraintCheck = await validateIncomingOperationConstraints(store, op, schema);
|
|
1994
2369
|
if (!constraintCheck.valid) {
|
|
2370
|
+
const code = constraintCheck.code ?? "CONSTRAINT_VIOLATION";
|
|
1995
2371
|
return {
|
|
1996
2372
|
result: "skipped",
|
|
1997
2373
|
appliedOperations: [],
|
|
1998
2374
|
rejection: {
|
|
1999
|
-
code
|
|
2000
|
-
message: constraintCheck.message ?? `Operation "${op.id}" violates a schema constraint
|
|
2375
|
+
code,
|
|
2376
|
+
message: constraintCheck.message ?? `Operation "${op.id}" violates a schema constraint`,
|
|
2377
|
+
retriable: isRetriableRejection(code)
|
|
2001
2378
|
}
|
|
2002
2379
|
};
|
|
2003
2380
|
}
|
|
@@ -2010,7 +2387,8 @@ async function applyServerOperation(store, op, relationLookup) {
|
|
|
2010
2387
|
appliedOperations: [],
|
|
2011
2388
|
rejection: {
|
|
2012
2389
|
code: "REFERENTIAL_INTEGRITY",
|
|
2013
|
-
message: `Operation "${op.id}" violates referential integrity on "${op.collection}"
|
|
2390
|
+
message: `Operation "${op.id}" violates referential integrity on "${op.collection}"`,
|
|
2391
|
+
retriable: isRetriableRejection("REFERENTIAL_INTEGRITY")
|
|
2014
2392
|
}
|
|
2015
2393
|
};
|
|
2016
2394
|
}
|
|
@@ -2019,16 +2397,14 @@ async function applyServerOperation(store, op, relationLookup) {
|
|
|
2019
2397
|
return { result: primaryResult, appliedOperations: [] };
|
|
2020
2398
|
}
|
|
2021
2399
|
const appliedOperations = [op];
|
|
2022
|
-
let serverSeq = nextServerSequenceNumber(store);
|
|
2023
2400
|
for (const effect of referential.sideEffectOps) {
|
|
2024
2401
|
const sideOp = await createServerSideEffectOperation(
|
|
2025
2402
|
store,
|
|
2026
2403
|
op,
|
|
2027
2404
|
effect,
|
|
2028
2405
|
op.schemaVersion,
|
|
2029
|
-
|
|
2406
|
+
nextServerSequenceNumber(store)
|
|
2030
2407
|
);
|
|
2031
|
-
serverSeq += 1;
|
|
2032
2408
|
const sideResult = await store.applyRemoteOperation(sideOp);
|
|
2033
2409
|
if (sideResult === "applied") {
|
|
2034
2410
|
appliedOperations.push(sideOp);
|
|
@@ -2042,6 +2418,74 @@ async function applyServerOperation(store, op, relationLookup) {
|
|
|
2042
2418
|
appliedOperations: result === "applied" ? [op] : []
|
|
2043
2419
|
};
|
|
2044
2420
|
}
|
|
2421
|
+
function validateOperationShape(op, schema) {
|
|
2422
|
+
if (!schema) {
|
|
2423
|
+
return { valid: true };
|
|
2424
|
+
}
|
|
2425
|
+
const collection = schema.collections[op.collection];
|
|
2426
|
+
if (!collection) {
|
|
2427
|
+
return {
|
|
2428
|
+
valid: false,
|
|
2429
|
+
code: "UNKNOWN_COLLECTION",
|
|
2430
|
+
message: `Operation "${op.id}" targets unknown collection "${op.collection}".`
|
|
2431
|
+
};
|
|
2432
|
+
}
|
|
2433
|
+
const declaredFields = new Set(Object.keys(collection.fields));
|
|
2434
|
+
const systemPreviousFields = /* @__PURE__ */ new Set(["id", "_created_at", "_updated_at", "_deleted"]);
|
|
2435
|
+
const invalidDataField = firstInvalidField(op.data, declaredFields);
|
|
2436
|
+
if (invalidDataField) {
|
|
2437
|
+
return {
|
|
2438
|
+
valid: false,
|
|
2439
|
+
code: "SCHEMA_VALIDATION_ERROR",
|
|
2440
|
+
message: `Operation "${op.id}" contains undeclared field "${invalidDataField}" in data for collection "${op.collection}".`
|
|
2441
|
+
};
|
|
2442
|
+
}
|
|
2443
|
+
const invalidPreviousField = firstInvalidField(
|
|
2444
|
+
op.previousData,
|
|
2445
|
+
declaredFields,
|
|
2446
|
+
systemPreviousFields
|
|
2447
|
+
);
|
|
2448
|
+
if (invalidPreviousField) {
|
|
2449
|
+
return {
|
|
2450
|
+
valid: false,
|
|
2451
|
+
code: "SCHEMA_VALIDATION_ERROR",
|
|
2452
|
+
message: `Operation "${op.id}" contains undeclared field "${invalidPreviousField}" in previousData for collection "${op.collection}".`
|
|
2453
|
+
};
|
|
2454
|
+
}
|
|
2455
|
+
if (op.atomicOps) {
|
|
2456
|
+
for (const field of Object.keys(op.atomicOps)) {
|
|
2457
|
+
if (!declaredFields.has(field)) {
|
|
2458
|
+
return {
|
|
2459
|
+
valid: false,
|
|
2460
|
+
code: "SCHEMA_VALIDATION_ERROR",
|
|
2461
|
+
message: `Operation "${op.id}" contains undeclared atomic field "${field}" for collection "${op.collection}".`
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
return { valid: true };
|
|
2467
|
+
}
|
|
2468
|
+
function firstInvalidField(value, declaredFields, extraAllowedFields) {
|
|
2469
|
+
if (value === null || value === void 0) {
|
|
2470
|
+
return null;
|
|
2471
|
+
}
|
|
2472
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
2473
|
+
return null;
|
|
2474
|
+
}
|
|
2475
|
+
for (const field of Object.keys(value)) {
|
|
2476
|
+
if (!declaredFields.has(field) && !extraAllowedFields?.has(field)) {
|
|
2477
|
+
return field;
|
|
2478
|
+
}
|
|
2479
|
+
}
|
|
2480
|
+
return null;
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
// src/auth/no-auth.ts
|
|
2484
|
+
var NoAuthProvider = class {
|
|
2485
|
+
async authenticate(_token) {
|
|
2486
|
+
return { userId: "anonymous" };
|
|
2487
|
+
}
|
|
2488
|
+
};
|
|
2045
2489
|
|
|
2046
2490
|
// src/scopes/resolve-session-scopes.ts
|
|
2047
2491
|
import { buildScopeMap } from "@korajs/core";
|
|
@@ -2079,11 +2523,12 @@ function mergeScopeMaps(base, overlay, source) {
|
|
|
2079
2523
|
}
|
|
2080
2524
|
|
|
2081
2525
|
// src/scopes/server-scope-filter.ts
|
|
2082
|
-
function operationMatchesScopes(op, scopes) {
|
|
2526
|
+
function operationMatchesScopes(op, scopes, fullRecord) {
|
|
2083
2527
|
if (!scopes) return true;
|
|
2084
2528
|
const collectionScope = scopes[op.collection];
|
|
2085
2529
|
if (!collectionScope) return false;
|
|
2086
|
-
|
|
2530
|
+
if (Object.keys(collectionScope).length === 0) return true;
|
|
2531
|
+
const snapshot = buildSnapshot(op, fullRecord);
|
|
2087
2532
|
if (!snapshot) return false;
|
|
2088
2533
|
for (const [field, expected] of Object.entries(collectionScope)) {
|
|
2089
2534
|
if (snapshot[field] !== expected) {
|
|
@@ -2092,11 +2537,20 @@ function operationMatchesScopes(op, scopes) {
|
|
|
2092
2537
|
}
|
|
2093
2538
|
return true;
|
|
2094
2539
|
}
|
|
2095
|
-
function
|
|
2540
|
+
function missingScopeFields(op, scopes) {
|
|
2541
|
+
if (!scopes) return [];
|
|
2542
|
+
const collectionScope = scopes[op.collection];
|
|
2543
|
+
if (!collectionScope) return [];
|
|
2544
|
+
if (Object.keys(collectionScope).length === 0) return [];
|
|
2545
|
+
const snapshot = buildSnapshot(op);
|
|
2546
|
+
return Object.keys(collectionScope).filter((field) => !snapshot || !(field in snapshot));
|
|
2547
|
+
}
|
|
2548
|
+
function buildSnapshot(op, fullRecord) {
|
|
2096
2549
|
const previous = asRecord(op.previousData);
|
|
2097
2550
|
const next = asRecord(op.data);
|
|
2098
|
-
if (!previous && !next) return null;
|
|
2551
|
+
if (!previous && !next && !fullRecord) return null;
|
|
2099
2552
|
return {
|
|
2553
|
+
...fullRecord ?? {},
|
|
2100
2554
|
...previous ?? {},
|
|
2101
2555
|
...next ?? {}
|
|
2102
2556
|
};
|
|
@@ -2168,28 +2622,106 @@ var SessionRateLimiter = class {
|
|
|
2168
2622
|
// src/session/client-session.ts
|
|
2169
2623
|
var DEFAULT_BATCH_SIZE = 100;
|
|
2170
2624
|
var DEFAULT_SCHEMA_VERSION = 1;
|
|
2171
|
-
var
|
|
2625
|
+
var warnedUnscopedProviders = /* @__PURE__ */ new WeakSet();
|
|
2626
|
+
function warnIfMultiTenantWithoutScopes(auth, resolvedScopes, schema) {
|
|
2627
|
+
if (!auth || auth instanceof NoAuthProvider) {
|
|
2628
|
+
return;
|
|
2629
|
+
}
|
|
2630
|
+
if (resolvedScopes) {
|
|
2631
|
+
return;
|
|
2632
|
+
}
|
|
2633
|
+
if (!schema || Object.keys(schema.collections).length === 0) {
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
if (warnedUnscopedProviders.has(auth)) {
|
|
2637
|
+
return;
|
|
2638
|
+
}
|
|
2639
|
+
warnedUnscopedProviders.add(auth);
|
|
2640
|
+
console.warn(
|
|
2641
|
+
"[kora] An authenticated session resolved to no sync scopes, so every user will sync every other user's data. Return per-user sync scopes from your auth provider (for example KoraAuthProvider's resolveScopes) to isolate tenants. Note: declaring sync rules in your schema is not enough on its own \u2014 the per-user values come from the auth provider. (This warning is expected for single-tenant apps where all authenticated users are meant to share the same data.)"
|
|
2642
|
+
);
|
|
2643
|
+
}
|
|
2644
|
+
function stableStringify(value) {
|
|
2645
|
+
if (Array.isArray(value)) {
|
|
2646
|
+
return `[${value.map((item) => stableStringify(item)).join(",")}]`;
|
|
2647
|
+
}
|
|
2648
|
+
if (value && typeof value === "object") {
|
|
2649
|
+
const record = value;
|
|
2650
|
+
const entries = Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`);
|
|
2651
|
+
return `{${entries.join(",")}}`;
|
|
2652
|
+
}
|
|
2653
|
+
return JSON.stringify(value);
|
|
2654
|
+
}
|
|
2655
|
+
function sameScopeMap(a, b) {
|
|
2656
|
+
return stableStringify(a ?? null) === stableStringify(b ?? null);
|
|
2657
|
+
}
|
|
2658
|
+
var ClientSession = class _ClientSession {
|
|
2172
2659
|
state = "connected";
|
|
2173
2660
|
clientNodeId = null;
|
|
2174
2661
|
authContext = null;
|
|
2175
2662
|
syncQuerySubsets = [];
|
|
2176
2663
|
resumeDeltaCursor = null;
|
|
2664
|
+
/**
|
|
2665
|
+
* The delivery watermark the client reported at handshake, or null when the client
|
|
2666
|
+
* does not use the delivery watermark (old client, or no watermark yet). When set,
|
|
2667
|
+
* the server drives the gap-free server->client stream from delivery sequences
|
|
2668
|
+
* instead of the version-vector delta.
|
|
2669
|
+
*/
|
|
2670
|
+
clientDeliveryWatermark = null;
|
|
2671
|
+
/**
|
|
2672
|
+
* The highest delivery sequence this client has ACKNOWLEDGED (its confirmed
|
|
2673
|
+
* watermark, as reported in acks). Incremental streaming pushes resume from here, not
|
|
2674
|
+
* from the highest sequence sent, so a dropped or unapplied batch is always
|
|
2675
|
+
* re-included by the next push (which re-scans from the acknowledged position). This
|
|
2676
|
+
* is what makes streaming recovery independent of any bounded retransmit buffer: there
|
|
2677
|
+
* is no sent-but-unacked window that a buffer eviction could strand. Seeded from the
|
|
2678
|
+
* client's reported watermark at handshake.
|
|
2679
|
+
*/
|
|
2680
|
+
lastAckedDeliverySeq = 0;
|
|
2681
|
+
/**
|
|
2682
|
+
* Timestamp of the last delivery-stream send attempt. Periodic retransmission uses
|
|
2683
|
+
* this as its stale window so a slow client does not receive duplicate full-stream
|
|
2684
|
+
* batches every server tick while the first batch is still being applied.
|
|
2685
|
+
*/
|
|
2686
|
+
lastDeliveryPushAttemptAtMs = 0;
|
|
2687
|
+
deliveryStallRepeatCount = 0;
|
|
2688
|
+
/** Serializes incremental delivery pushes so their batches never interleave. */
|
|
2689
|
+
deliveryPushChain = Promise.resolve();
|
|
2690
|
+
/**
|
|
2691
|
+
* Relay batches sent to this client but not yet acknowledged, keyed by messageId.
|
|
2692
|
+
* The client acks every operation-batch it applies; on ack the entry is cleared.
|
|
2693
|
+
* {@link retransmitPendingRelays} re-sends anything still unacked, so a relay
|
|
2694
|
+
* dropped by a lossy transport is redelivered instead of leaving the client with a
|
|
2695
|
+
* permanent version-vector gap (a lost operation) that delta sync cannot recover.
|
|
2696
|
+
* Bounded so a silent client cannot grow it without limit.
|
|
2697
|
+
*/
|
|
2698
|
+
pendingRelays = /* @__PURE__ */ new Map();
|
|
2699
|
+
static MAX_PENDING_RELAYS = 1e3;
|
|
2177
2700
|
sessionId;
|
|
2178
2701
|
transport;
|
|
2179
2702
|
store;
|
|
2180
2703
|
auth;
|
|
2181
2704
|
serializer;
|
|
2182
2705
|
emitter;
|
|
2706
|
+
logger;
|
|
2183
2707
|
batchSize;
|
|
2184
2708
|
schemaVersion;
|
|
2185
2709
|
supportedSchemaVersions;
|
|
2710
|
+
operationTransforms;
|
|
2186
2711
|
onRelay;
|
|
2187
2712
|
onAwarenessUpdate;
|
|
2188
2713
|
onYjsDocUpdate;
|
|
2714
|
+
onBlobChunkRequest;
|
|
2715
|
+
onBlobChunkResponse;
|
|
2716
|
+
persistBlobChunk;
|
|
2189
2717
|
onClose;
|
|
2718
|
+
onOrphanedRelays;
|
|
2719
|
+
takeOrphanedRelays;
|
|
2190
2720
|
maxOperationBytes;
|
|
2191
2721
|
maxOpsPerMinute;
|
|
2192
2722
|
rateLimiter;
|
|
2723
|
+
validateOperation;
|
|
2724
|
+
koraContext;
|
|
2193
2725
|
constructor(options) {
|
|
2194
2726
|
this.sessionId = options.sessionId;
|
|
2195
2727
|
this.transport = options.transport;
|
|
@@ -2197,6 +2729,7 @@ var ClientSession = class {
|
|
|
2197
2729
|
this.auth = options.auth ?? null;
|
|
2198
2730
|
this.serializer = options.serializer ?? new NegotiatedMessageSerializer("json");
|
|
2199
2731
|
this.emitter = options.emitter ?? null;
|
|
2732
|
+
this.logger = options.logger ?? null;
|
|
2200
2733
|
this.batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
2201
2734
|
this.schemaVersion = options.schemaVersion ?? DEFAULT_SCHEMA_VERSION;
|
|
2202
2735
|
const supported = options.supportedSchemaVersions;
|
|
@@ -2204,13 +2737,21 @@ var ClientSession = class {
|
|
|
2204
2737
|
min: this.schemaVersion,
|
|
2205
2738
|
max: this.schemaVersion
|
|
2206
2739
|
};
|
|
2740
|
+
this.operationTransforms = options.operationTransforms ?? [];
|
|
2207
2741
|
this.onRelay = options.onRelay ?? null;
|
|
2208
2742
|
this.onAwarenessUpdate = options.onAwarenessUpdate ?? null;
|
|
2209
2743
|
this.onYjsDocUpdate = options.onYjsDocUpdate ?? null;
|
|
2744
|
+
this.onBlobChunkRequest = options.onBlobChunkRequest ?? null;
|
|
2745
|
+
this.onBlobChunkResponse = options.onBlobChunkResponse ?? null;
|
|
2746
|
+
this.persistBlobChunk = options.persistBlobChunk ?? null;
|
|
2210
2747
|
this.onClose = options.onClose ?? null;
|
|
2748
|
+
this.onOrphanedRelays = options.onOrphanedRelays ?? null;
|
|
2749
|
+
this.takeOrphanedRelays = options.takeOrphanedRelays ?? null;
|
|
2211
2750
|
this.maxOperationBytes = options.maxOperationBytes ?? DEFAULT_MAX_OPERATION_BYTES;
|
|
2212
2751
|
this.maxOpsPerMinute = options.maxOpsPerMinute ?? DEFAULT_MAX_OPS_PER_MINUTE;
|
|
2213
2752
|
this.rateLimiter = new SessionRateLimiter(this.maxOpsPerMinute);
|
|
2753
|
+
this.validateOperation = options.validateOperation ?? null;
|
|
2754
|
+
this.koraContext = options.koraContext ?? null;
|
|
2214
2755
|
}
|
|
2215
2756
|
/**
|
|
2216
2757
|
* Start handling messages from the client transport.
|
|
@@ -2230,9 +2771,41 @@ var ClientSession = class {
|
|
|
2230
2771
|
*/
|
|
2231
2772
|
relayOperations(operations2) {
|
|
2232
2773
|
if (this.state !== "streaming" || !this.transport.isConnected()) return;
|
|
2774
|
+
if (this.clientDeliveryWatermark !== null) {
|
|
2775
|
+
this.pushDeliveryStream();
|
|
2776
|
+
return;
|
|
2777
|
+
}
|
|
2233
2778
|
if (operations2.length === 0) return;
|
|
2234
|
-
|
|
2779
|
+
void this.relayVisibleOperations(operations2);
|
|
2780
|
+
}
|
|
2781
|
+
/**
|
|
2782
|
+
* Push newly-available in-scope operations to a delivery-watermark client, resuming
|
|
2783
|
+
* from the highest delivery sequence already sent. Pushes are serialized so their
|
|
2784
|
+
* batches never interleave (which would break the base/max chain).
|
|
2785
|
+
*/
|
|
2786
|
+
pushDeliveryStream() {
|
|
2787
|
+
this.deliveryPushChain = this.deliveryPushChain.then(async () => {
|
|
2788
|
+
if (this.state !== "streaming" || !this.transport.isConnected()) return;
|
|
2789
|
+
try {
|
|
2790
|
+
this.lastDeliveryPushAttemptAtMs = Date.now();
|
|
2791
|
+
await this.sendDeliveryStream(
|
|
2792
|
+
this.lastAckedDeliverySeq,
|
|
2793
|
+
false,
|
|
2794
|
+
this.clientNodeId ?? void 0
|
|
2795
|
+
);
|
|
2796
|
+
} catch {
|
|
2797
|
+
}
|
|
2798
|
+
});
|
|
2799
|
+
}
|
|
2800
|
+
async relayVisibleOperations(operations2) {
|
|
2801
|
+
const visibleOperations = [];
|
|
2802
|
+
for (const op of operations2) {
|
|
2803
|
+
if (await this.operationVisibleToClient(op)) {
|
|
2804
|
+
visibleOperations.push(op);
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2235
2807
|
if (visibleOperations.length === 0) return;
|
|
2808
|
+
if (this.state !== "streaming" || !this.transport.isConnected()) return;
|
|
2236
2809
|
const serializedOps = visibleOperations.map((op) => this.serializer.encodeOperation(op));
|
|
2237
2810
|
const msg = {
|
|
2238
2811
|
type: "operation-batch",
|
|
@@ -2241,14 +2814,86 @@ var ClientSession = class {
|
|
|
2241
2814
|
isFinal: true,
|
|
2242
2815
|
batchIndex: 0
|
|
2243
2816
|
};
|
|
2817
|
+
this.trackPendingRelay(msg, visibleOperations);
|
|
2244
2818
|
this.sendToClient(msg);
|
|
2245
2819
|
}
|
|
2820
|
+
/** Record a relay batch as awaiting acknowledgment, evicting the oldest if full. */
|
|
2821
|
+
trackPendingRelay(msg, ops) {
|
|
2822
|
+
if (this.pendingRelays.size >= _ClientSession.MAX_PENDING_RELAYS) {
|
|
2823
|
+
const oldest = this.pendingRelays.keys().next().value;
|
|
2824
|
+
if (oldest !== void 0) {
|
|
2825
|
+
this.pendingRelays.delete(oldest);
|
|
2826
|
+
}
|
|
2827
|
+
}
|
|
2828
|
+
this.pendingRelays.set(msg.messageId, { message: msg, ops, sentAtMs: Date.now() });
|
|
2829
|
+
}
|
|
2830
|
+
/**
|
|
2831
|
+
* Retransmit relay batches this client has not acknowledged within `staleMs`.
|
|
2832
|
+
* Called on a periodic tick by the server (and directly by tests). Redelivering an
|
|
2833
|
+
* already-applied op is harmless: the client dedups by content-addressed id.
|
|
2834
|
+
*/
|
|
2835
|
+
retransmitPendingRelays(staleMs = 0) {
|
|
2836
|
+
if (this.state !== "streaming" || !this.transport.isConnected()) return;
|
|
2837
|
+
if (this.clientDeliveryWatermark !== null) {
|
|
2838
|
+
return;
|
|
2839
|
+
}
|
|
2840
|
+
if (this.pendingRelays.size === 0) return;
|
|
2841
|
+
const cutoff = Date.now() - staleMs;
|
|
2842
|
+
for (const { message, sentAtMs } of this.pendingRelays.values()) {
|
|
2843
|
+
if (sentAtMs <= cutoff) {
|
|
2844
|
+
this.sendToClient(message);
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2848
|
+
/**
|
|
2849
|
+
* Wake the durable delivery stream for clients that negotiated delivery
|
|
2850
|
+
* watermarks. This is used both for dropped watermark batches and for operations
|
|
2851
|
+
* appended by another server/store instance: the session always scans from its
|
|
2852
|
+
* own acknowledged cursor and applies its visibility filter before sending.
|
|
2853
|
+
*/
|
|
2854
|
+
pushDeliveryStreamIfSupported(staleMs = 0, options = {}) {
|
|
2855
|
+
if (this.state !== "streaming" || !this.transport.isConnected()) return;
|
|
2856
|
+
if (this.clientDeliveryWatermark === null) return;
|
|
2857
|
+
if (staleMs > 0 && Date.now() - this.lastDeliveryPushAttemptAtMs < staleMs) {
|
|
2858
|
+
return;
|
|
2859
|
+
}
|
|
2860
|
+
if (options.trackStall) {
|
|
2861
|
+
this.deliveryStallRepeatCount += 1;
|
|
2862
|
+
if (this.deliveryStallRepeatCount >= 3) {
|
|
2863
|
+
this.emitter?.emit({
|
|
2864
|
+
type: "sync:delivery-stalled",
|
|
2865
|
+
sessionId: this.sessionId,
|
|
2866
|
+
watermark: this.lastAckedDeliverySeq,
|
|
2867
|
+
repeatCount: this.deliveryStallRepeatCount
|
|
2868
|
+
});
|
|
2869
|
+
}
|
|
2870
|
+
}
|
|
2871
|
+
this.pushDeliveryStream();
|
|
2872
|
+
}
|
|
2246
2873
|
/**
|
|
2247
2874
|
* Close this session.
|
|
2248
2875
|
*/
|
|
2876
|
+
/**
|
|
2877
|
+
* Hand any unacknowledged relays to the server so they survive this session and are
|
|
2878
|
+
* redelivered when this client reconnects (per-node buffer), then clear them. Called
|
|
2879
|
+
* from every session-teardown path (explicit close and transport close).
|
|
2880
|
+
*/
|
|
2881
|
+
flushOrphanedRelays() {
|
|
2882
|
+
if (this.onOrphanedRelays && this.clientNodeId && this.pendingRelays.size > 0) {
|
|
2883
|
+
const ops = [];
|
|
2884
|
+
for (const entry of this.pendingRelays.values()) {
|
|
2885
|
+
ops.push(...entry.ops);
|
|
2886
|
+
}
|
|
2887
|
+
if (ops.length > 0) {
|
|
2888
|
+
this.onOrphanedRelays(this.clientNodeId, ops);
|
|
2889
|
+
}
|
|
2890
|
+
}
|
|
2891
|
+
this.pendingRelays.clear();
|
|
2892
|
+
}
|
|
2249
2893
|
close(reason) {
|
|
2250
2894
|
if (this.state === "closed") return;
|
|
2251
2895
|
this.state = "closed";
|
|
2896
|
+
this.flushOrphanedRelays();
|
|
2252
2897
|
if (this.transport.isConnected()) {
|
|
2253
2898
|
this.transport.close(1e3, reason ?? "session closed");
|
|
2254
2899
|
}
|
|
@@ -2303,6 +2948,11 @@ var ClientSession = class {
|
|
|
2303
2948
|
await this.handleOperationBatch(message);
|
|
2304
2949
|
break;
|
|
2305
2950
|
case "acknowledgment":
|
|
2951
|
+
this.pendingRelays.delete(message.acknowledgedMessageId);
|
|
2952
|
+
if (message.deliverySequence !== void 0) {
|
|
2953
|
+
this.lastAckedDeliverySeq = Math.max(this.lastAckedDeliverySeq, message.deliverySequence);
|
|
2954
|
+
this.deliveryStallRepeatCount = 0;
|
|
2955
|
+
}
|
|
2306
2956
|
break;
|
|
2307
2957
|
case "error":
|
|
2308
2958
|
break;
|
|
@@ -2312,10 +2962,47 @@ var ClientSession = class {
|
|
|
2312
2962
|
case "yjs-doc-update":
|
|
2313
2963
|
this.handleYjsDocUpdate(message);
|
|
2314
2964
|
break;
|
|
2315
|
-
|
|
2316
|
-
|
|
2965
|
+
case "blob-chunk-request":
|
|
2966
|
+
this.onBlobChunkRequest?.(this.sessionId, message);
|
|
2967
|
+
break;
|
|
2968
|
+
case "blob-chunk-response":
|
|
2969
|
+
this.onBlobChunkResponse?.(this.sessionId, message);
|
|
2970
|
+
break;
|
|
2971
|
+
case "blob-chunk-push":
|
|
2972
|
+
await this.handleBlobChunkPush(message);
|
|
2973
|
+
break;
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
/**
|
|
2977
|
+
* Persist a client-uploaded blob chunk (or manifest) centrally. The bytes are
|
|
2978
|
+
* verified to hash to the declared hash before storing, so a corrupt or
|
|
2979
|
+
* mislabeled upload is rejected rather than served later as trusted content.
|
|
2980
|
+
*/
|
|
2981
|
+
async handleBlobChunkPush(message) {
|
|
2982
|
+
if (!this.persistBlobChunk) {
|
|
2983
|
+
return;
|
|
2984
|
+
}
|
|
2985
|
+
const bytes = decodeBlobChunkBytes(message.bytes);
|
|
2986
|
+
const actual = await hashBlob(bytes);
|
|
2987
|
+
if (actual !== message.hash) {
|
|
2988
|
+
return;
|
|
2989
|
+
}
|
|
2990
|
+
await this.persistBlobChunk(message.hash, bytes);
|
|
2991
|
+
}
|
|
2317
2992
|
handleMessageFailure(error) {
|
|
2318
2993
|
const reason = error instanceof Error ? error.message : "Message handling failed";
|
|
2994
|
+
this.logger?.log({
|
|
2995
|
+
timestamp: Date.now(),
|
|
2996
|
+
level: "error",
|
|
2997
|
+
event: "session.message_failed",
|
|
2998
|
+
sessionId: this.sessionId,
|
|
2999
|
+
nodeId: this.clientNodeId ?? void 0,
|
|
3000
|
+
error: reason,
|
|
3001
|
+
details: {
|
|
3002
|
+
state: this.state,
|
|
3003
|
+
errorName: error instanceof Error ? error.name : typeof error
|
|
3004
|
+
}
|
|
3005
|
+
});
|
|
2319
3006
|
this.sendError("SYNC_ERROR", reason, true);
|
|
2320
3007
|
this.close(reason);
|
|
2321
3008
|
}
|
|
@@ -2347,17 +3034,38 @@ var ClientSession = class {
|
|
|
2347
3034
|
this.authContext = { userId: msg.nodeId, scopes: resolvedScopes };
|
|
2348
3035
|
}
|
|
2349
3036
|
}
|
|
3037
|
+
warnIfMultiTenantWithoutScopes(this.auth, resolvedScopes, this.store.getSchema());
|
|
2350
3038
|
if (msg.syncQueries && msg.syncQueries.length > 0) {
|
|
2351
3039
|
this.syncQuerySubsets = dedupeQuerySubsets(msg.syncQueries);
|
|
2352
3040
|
} else {
|
|
2353
3041
|
this.syncQuerySubsets = [];
|
|
2354
3042
|
}
|
|
2355
3043
|
this.resumeDeltaCursor = msg.deltaCursor ? decodeDeltaCursor(msg.deltaCursor) : null;
|
|
3044
|
+
this.clientDeliveryWatermark = msg.lastDeliverySequence ?? null;
|
|
3045
|
+
if (this.clientDeliveryWatermark !== null && !sameScopeMap(msg.syncScope, this.authContext?.scopes)) {
|
|
3046
|
+
this.clientDeliveryWatermark = 0;
|
|
3047
|
+
}
|
|
3048
|
+
const clientUsesWatermark = this.clientDeliveryWatermark !== null;
|
|
3049
|
+
let serverMaxDelivery = 0;
|
|
3050
|
+
if (clientUsesWatermark) {
|
|
3051
|
+
serverMaxDelivery = await this.store.getMaxDeliverySequence();
|
|
3052
|
+
if (this.clientDeliveryWatermark !== null && this.clientDeliveryWatermark > serverMaxDelivery) {
|
|
3053
|
+
this.clientDeliveryWatermark = 0;
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
2356
3056
|
const serverVector = this.store.getVersionVector();
|
|
2357
3057
|
const selectedWireFormat = selectWireFormat(msg.supportedWireFormats);
|
|
2358
3058
|
this.setSerializerWireFormat(selectedWireFormat);
|
|
2359
3059
|
if (!isClientSchemaVersionSupported(msg.schemaVersion, this.supportedSchemaVersions)) {
|
|
2360
3060
|
const { min, max } = this.supportedSchemaVersions;
|
|
3061
|
+
this.emitter?.emit({
|
|
3062
|
+
type: "sync:schema-mismatch",
|
|
3063
|
+
clientSchemaVersion: msg.schemaVersion,
|
|
3064
|
+
serverSchemaVersion: this.schemaVersion,
|
|
3065
|
+
supportedMin: min,
|
|
3066
|
+
supportedMax: max,
|
|
3067
|
+
reason: `${SCHEMA_MISMATCH_PREFIX}: client schema version ${msg.schemaVersion} not in supported range [${min}, ${max}]`
|
|
3068
|
+
});
|
|
2361
3069
|
const response2 = {
|
|
2362
3070
|
type: "handshake-response",
|
|
2363
3071
|
messageId: generateUUIDv74(),
|
|
@@ -2383,6 +3091,13 @@ var ClientSession = class {
|
|
|
2383
3091
|
accepted: true,
|
|
2384
3092
|
selectedWireFormat,
|
|
2385
3093
|
serverTime: Date.now(),
|
|
3094
|
+
// The server's highest delivery sequence, so a client whose persisted watermark
|
|
3095
|
+
// is ahead of it (server rolled back) can reset to a full resync. Sent only to
|
|
3096
|
+
// watermark-using clients.
|
|
3097
|
+
...clientUsesWatermark ? { serverMaxDeliverySequence: serverMaxDelivery } : {},
|
|
3098
|
+
// Advertise central blob storage so the client uploads the bytes behind
|
|
3099
|
+
// its blob fields, keeping them available after the author goes offline.
|
|
3100
|
+
...this.persistBlobChunk ? { blobStorageEnabled: true } : {},
|
|
2386
3101
|
// Confirm the accepted scope so the client knows what data will be synced.
|
|
2387
3102
|
// This may differ from what the client requested if auth scopes are narrower.
|
|
2388
3103
|
...this.authContext?.scopes ? { acceptedScope: this.authContext.scopes } : {}
|
|
@@ -2390,17 +3105,40 @@ var ClientSession = class {
|
|
|
2390
3105
|
this.sendToClient(response);
|
|
2391
3106
|
this.emitter?.emit({ type: "sync:connected", nodeId: msg.nodeId });
|
|
2392
3107
|
this.state = "syncing";
|
|
2393
|
-
|
|
2394
|
-
|
|
3108
|
+
if (this.clientDeliveryWatermark !== null) {
|
|
3109
|
+
this.lastAckedDeliverySeq = this.clientDeliveryWatermark;
|
|
3110
|
+
const excludeOwn = this.clientDeliveryWatermark > 0 ? this.clientNodeId ?? void 0 : void 0;
|
|
3111
|
+
this.lastDeliveryPushAttemptAtMs = Date.now();
|
|
3112
|
+
await this.sendDeliveryStream(this.clientDeliveryWatermark, true, excludeOwn);
|
|
3113
|
+
} else {
|
|
3114
|
+
const clientVector = wireToVersionVector(msg.versionVector);
|
|
3115
|
+
await this.sendDelta(clientVector);
|
|
3116
|
+
}
|
|
2395
3117
|
this.state = "streaming";
|
|
3118
|
+
if (this.takeOrphanedRelays && this.clientNodeId) {
|
|
3119
|
+
const buffered = this.takeOrphanedRelays(this.clientNodeId);
|
|
3120
|
+
if (buffered.length > 0) {
|
|
3121
|
+
this.relayOperations(buffered);
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
2396
3124
|
}
|
|
2397
3125
|
async handleOperationBatch(msg) {
|
|
2398
3126
|
const operations2 = msg.operations.map((s) => this.serializer.decodeOperation(s));
|
|
2399
3127
|
const applied = [];
|
|
2400
|
-
|
|
3128
|
+
let acknowledgedThrough = 0;
|
|
3129
|
+
let canAdvanceAck = true;
|
|
2401
3130
|
for (const op of operations2) {
|
|
2402
|
-
if (!
|
|
2403
|
-
|
|
3131
|
+
if (!canAdvanceAck) {
|
|
3132
|
+
continue;
|
|
3133
|
+
}
|
|
3134
|
+
if (!await this.operationVisibleToClient(op)) {
|
|
3135
|
+
this.sendOperationRejected(
|
|
3136
|
+
op,
|
|
3137
|
+
"SCOPE_VIOLATION",
|
|
3138
|
+
`Operation "${op.id}" in collection "${op.collection}" is outside the client's sync scope. Refresh sync scopes and retry.`,
|
|
3139
|
+
true
|
|
3140
|
+
);
|
|
3141
|
+
canAdvanceAck = false;
|
|
2404
3142
|
continue;
|
|
2405
3143
|
}
|
|
2406
3144
|
if (!isOperationTimestampValid(op)) {
|
|
@@ -2409,6 +3147,7 @@ var ClientSession = class {
|
|
|
2409
3147
|
`Operation "${op.id}" timestamp is too far in the future`,
|
|
2410
3148
|
false
|
|
2411
3149
|
);
|
|
3150
|
+
canAdvanceAck = false;
|
|
2412
3151
|
continue;
|
|
2413
3152
|
}
|
|
2414
3153
|
if (!this.rateLimiter.allow(1)) {
|
|
@@ -2417,6 +3156,7 @@ var ClientSession = class {
|
|
|
2417
3156
|
`Session exceeded operation rate limit (${String(this.maxOpsPerMinute)} ops/min)`,
|
|
2418
3157
|
true
|
|
2419
3158
|
);
|
|
3159
|
+
canAdvanceAck = false;
|
|
2420
3160
|
continue;
|
|
2421
3161
|
}
|
|
2422
3162
|
const sizeCheck = validateOperationSize(op, this.maxOperationBytes);
|
|
@@ -2426,27 +3166,73 @@ var ClientSession = class {
|
|
|
2426
3166
|
sizeCheck.message ?? `Operation "${op.id}" is too large`,
|
|
2427
3167
|
false
|
|
2428
3168
|
);
|
|
3169
|
+
canAdvanceAck = false;
|
|
2429
3170
|
continue;
|
|
2430
3171
|
}
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
3172
|
+
const serverOp = this.transformForServerSchema(op);
|
|
3173
|
+
if (serverOp === null) {
|
|
3174
|
+
this.sendOperationRejected(
|
|
3175
|
+
op,
|
|
3176
|
+
"SCHEMA_TRANSFORM_UNAVAILABLE",
|
|
3177
|
+
`Operation "${op.id}" cannot be transformed from schema v${op.schemaVersion} to server schema v${this.schemaVersion}.`,
|
|
3178
|
+
false
|
|
3179
|
+
);
|
|
3180
|
+
acknowledgedThrough = op.sequenceNumber;
|
|
3181
|
+
continue;
|
|
3182
|
+
}
|
|
3183
|
+
if (this.validateOperation && this.koraContext) {
|
|
3184
|
+
let decision;
|
|
3185
|
+
try {
|
|
3186
|
+
decision = await this.validateOperation(serverOp, {
|
|
3187
|
+
auth: this.authContext,
|
|
3188
|
+
kora: this.koraContext
|
|
3189
|
+
});
|
|
3190
|
+
} catch (error) {
|
|
3191
|
+
const message = error instanceof Error ? error.message : "validator error";
|
|
3192
|
+
this.sendOperationRejected(
|
|
3193
|
+
serverOp,
|
|
3194
|
+
"VALIDATION_ERROR",
|
|
3195
|
+
`Validator threw: ${message}`,
|
|
3196
|
+
true
|
|
3197
|
+
);
|
|
3198
|
+
canAdvanceAck = false;
|
|
3199
|
+
continue;
|
|
3200
|
+
}
|
|
3201
|
+
if (decision.action === "reject") {
|
|
3202
|
+
const retriable = decision.retriable ?? isRetriableRejection(decision.code);
|
|
3203
|
+
this.sendOperationRejected(serverOp, decision.code, decision.message, retriable);
|
|
3204
|
+
if (retriable) {
|
|
3205
|
+
canAdvanceAck = false;
|
|
3206
|
+
} else {
|
|
3207
|
+
acknowledgedThrough = op.sequenceNumber;
|
|
3208
|
+
}
|
|
2435
3209
|
continue;
|
|
2436
3210
|
}
|
|
2437
|
-
if (
|
|
2438
|
-
|
|
3211
|
+
if (decision.action === "ignore") {
|
|
3212
|
+
acknowledgedThrough = op.sequenceNumber;
|
|
3213
|
+
continue;
|
|
2439
3214
|
}
|
|
2440
|
-
} catch {
|
|
2441
3215
|
}
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
3216
|
+
const applyResult = await applyServerOperation(this.store, serverOp);
|
|
3217
|
+
if (applyResult.rejection) {
|
|
3218
|
+
this.sendOperationRejected(
|
|
3219
|
+
serverOp,
|
|
3220
|
+
applyResult.rejection.code,
|
|
3221
|
+
applyResult.rejection.message,
|
|
3222
|
+
applyResult.rejection.retriable
|
|
2449
3223
|
);
|
|
3224
|
+
if (applyResult.rejection.retriable) {
|
|
3225
|
+
canAdvanceAck = false;
|
|
3226
|
+
} else {
|
|
3227
|
+
acknowledgedThrough = op.sequenceNumber;
|
|
3228
|
+
}
|
|
3229
|
+
continue;
|
|
3230
|
+
}
|
|
3231
|
+
if (applyResult.result === "applied") {
|
|
3232
|
+
applied.push(...applyResult.appliedOperations);
|
|
3233
|
+
acknowledgedThrough = op.sequenceNumber;
|
|
3234
|
+
} else {
|
|
3235
|
+
acknowledgedThrough = op.sequenceNumber;
|
|
2450
3236
|
}
|
|
2451
3237
|
}
|
|
2452
3238
|
if (operations2.length > 0) {
|
|
@@ -2461,13 +3247,19 @@ var ClientSession = class {
|
|
|
2461
3247
|
type: "acknowledgment",
|
|
2462
3248
|
messageId: generateUUIDv74(),
|
|
2463
3249
|
acknowledgedMessageId: msg.messageId,
|
|
2464
|
-
lastSequenceNumber: lastOp ? lastOp.sequenceNumber : 0
|
|
3250
|
+
lastSequenceNumber: lastOp ? Math.min(acknowledgedThrough, lastOp.sequenceNumber) : 0
|
|
2465
3251
|
};
|
|
2466
3252
|
this.sendToClient(ack);
|
|
2467
3253
|
if (applied.length > 0) {
|
|
2468
3254
|
this.onRelay?.(this.sessionId, applied);
|
|
2469
3255
|
}
|
|
2470
3256
|
}
|
|
3257
|
+
transformForServerSchema(op) {
|
|
3258
|
+
if (op.schemaVersion === this.schemaVersion) {
|
|
3259
|
+
return op;
|
|
3260
|
+
}
|
|
3261
|
+
return applyOperationTransforms(op, this.schemaVersion, this.operationTransforms);
|
|
3262
|
+
}
|
|
2471
3263
|
async sendDelta(clientVector) {
|
|
2472
3264
|
const serverVector = this.store.getVersionVector();
|
|
2473
3265
|
const missing = [];
|
|
@@ -2475,8 +3267,11 @@ var ClientSession = class {
|
|
|
2475
3267
|
const clientSeq = clientVector.get(nodeId) ?? 0;
|
|
2476
3268
|
if (serverSeq > clientSeq) {
|
|
2477
3269
|
const ops = await this.store.getOperationRange(nodeId, clientSeq + 1, serverSeq);
|
|
2478
|
-
const
|
|
2479
|
-
|
|
3270
|
+
for (const op of ops) {
|
|
3271
|
+
if (await this.operationVisibleToClient(op)) {
|
|
3272
|
+
missing.push(op);
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
2480
3275
|
}
|
|
2481
3276
|
}
|
|
2482
3277
|
if (missing.length === 0) {
|
|
@@ -2528,11 +3323,111 @@ var ClientSession = class {
|
|
|
2528
3323
|
});
|
|
2529
3324
|
}
|
|
2530
3325
|
}
|
|
2531
|
-
|
|
2532
|
-
|
|
3326
|
+
/**
|
|
3327
|
+
* Send the gap-free server->client delivery stream, resuming just after
|
|
3328
|
+
* `fromDeliverySeq`. Operations are scanned in server delivery-sequence order
|
|
3329
|
+
* (commit order), scope-filtered for this session, and sent in batches that chain:
|
|
3330
|
+
* each batch's `baseDeliverySequence` equals the previous batch's
|
|
3331
|
+
* `maxDeliverySequence` (the first batch bases on `fromDeliverySeq`). The client
|
|
3332
|
+
* applies a batch only when its watermark equals the base and advances the
|
|
3333
|
+
* watermark to the max, so a dropped batch stalls the watermark and is recovered by
|
|
3334
|
+
* the next handshake resend. Because delivery-sequence order respects causal order
|
|
3335
|
+
* (a dependency is always committed, and thus sequenced, before its dependent), no
|
|
3336
|
+
* topological sort is needed and the client never defers on a missing dependency
|
|
3337
|
+
* that is itself in this stream.
|
|
3338
|
+
*
|
|
3339
|
+
* The final batch advances the watermark to the highest delivery sequence scanned,
|
|
3340
|
+
* not merely the last in-scope one, so an out-of-scope tail is not re-scanned on the
|
|
3341
|
+
* next reconnect. Returns the number of operations actually sent.
|
|
3342
|
+
*/
|
|
3343
|
+
async sendDeliveryStream(fromDeliverySeq, finalizeWhenEmpty, excludeNodeId) {
|
|
3344
|
+
const scanChunk = Math.max(this.batchSize, 1) * 5;
|
|
3345
|
+
let scanCursor = fromDeliverySeq;
|
|
3346
|
+
let maxScanned = fromDeliverySeq;
|
|
3347
|
+
const visible = [];
|
|
3348
|
+
while (true) {
|
|
3349
|
+
const chunk = await this.store.getOperationsAfterDelivery(scanCursor, scanChunk);
|
|
3350
|
+
const last = chunk[chunk.length - 1];
|
|
3351
|
+
if (last === void 0) break;
|
|
3352
|
+
scanCursor = last.deliverySequence;
|
|
3353
|
+
maxScanned = scanCursor;
|
|
3354
|
+
for (const delivered of chunk) {
|
|
3355
|
+
if (excludeNodeId !== void 0 && delivered.operation.nodeId === excludeNodeId) {
|
|
3356
|
+
continue;
|
|
3357
|
+
}
|
|
3358
|
+
if (await this.operationVisibleToClient(delivered.operation)) {
|
|
3359
|
+
visible.push(delivered);
|
|
3360
|
+
}
|
|
3361
|
+
}
|
|
3362
|
+
if (chunk.length < scanChunk) break;
|
|
3363
|
+
}
|
|
3364
|
+
if (visible.length === 0) {
|
|
3365
|
+
if (finalizeWhenEmpty) {
|
|
3366
|
+
this.sendDeliveryBatch([], fromDeliverySeq, maxScanned, 0, true);
|
|
3367
|
+
}
|
|
3368
|
+
return 0;
|
|
3369
|
+
}
|
|
3370
|
+
const totalBatches = Math.ceil(visible.length / this.batchSize);
|
|
3371
|
+
let base = fromDeliverySeq;
|
|
3372
|
+
for (let i = 0; i < totalBatches; i++) {
|
|
3373
|
+
const slice = visible.slice(i * this.batchSize, (i + 1) * this.batchSize);
|
|
3374
|
+
const lastInSlice = slice[slice.length - 1];
|
|
3375
|
+
if (lastInSlice === void 0) continue;
|
|
3376
|
+
const isFinal = i === totalBatches - 1;
|
|
3377
|
+
const lastSeq = lastInSlice.deliverySequence;
|
|
3378
|
+
const max = isFinal ? Math.max(maxScanned, lastSeq) : lastSeq;
|
|
3379
|
+
this.sendDeliveryBatch(slice, base, max, i, isFinal);
|
|
3380
|
+
base = max;
|
|
3381
|
+
}
|
|
3382
|
+
return visible.length;
|
|
3383
|
+
}
|
|
3384
|
+
/** Build, track, and send one chained delivery-stream batch. */
|
|
3385
|
+
sendDeliveryBatch(slice, base, max, batchIndex, isFinal) {
|
|
3386
|
+
const batchMsg = {
|
|
3387
|
+
type: "operation-batch",
|
|
3388
|
+
messageId: generateUUIDv74(),
|
|
3389
|
+
operations: slice.map((delivered) => this.serializer.encodeOperation(delivered.operation)),
|
|
3390
|
+
isFinal,
|
|
3391
|
+
batchIndex,
|
|
3392
|
+
baseDeliverySequence: base,
|
|
3393
|
+
maxDeliverySequence: max
|
|
3394
|
+
};
|
|
3395
|
+
this.sendToClient(batchMsg);
|
|
3396
|
+
if (slice.length > 0) {
|
|
3397
|
+
this.emitter?.emit({
|
|
3398
|
+
type: "sync:sent",
|
|
3399
|
+
operations: slice.map((delivered) => delivered.operation),
|
|
3400
|
+
batchSize: slice.length
|
|
3401
|
+
});
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
async operationVisibleToClient(op) {
|
|
3405
|
+
const scopes = this.authContext?.scopes;
|
|
3406
|
+
const subsets = this.syncQuerySubsets;
|
|
3407
|
+
const needsBackfill = missingScopeFields(op, scopes).length > 0 || subsets !== void 0 && subsets.length > 0;
|
|
3408
|
+
const fullRecord = needsBackfill ? await this.lookupRecordFields(op.collection, op.recordId) : void 0;
|
|
3409
|
+
if (!operationMatchesScopes(op, scopes, fullRecord)) {
|
|
2533
3410
|
return false;
|
|
2534
3411
|
}
|
|
2535
|
-
return operationMatchesQuerySubsets(op,
|
|
3412
|
+
return operationMatchesQuerySubsets(op, subsets, fullRecord);
|
|
3413
|
+
}
|
|
3414
|
+
/**
|
|
3415
|
+
* Read a record's current field values from the materialized store for scope /
|
|
3416
|
+
* query-subset backfill. Includes soft-deleted rows so a relayed delete (whose op
|
|
3417
|
+
* carries no fields) is still judged against the record's actual scope. Returns
|
|
3418
|
+
* undefined when the record cannot be read (never materialized, or no schema).
|
|
3419
|
+
*/
|
|
3420
|
+
async lookupRecordFields(collection, recordId) {
|
|
3421
|
+
try {
|
|
3422
|
+
const rows = await this.store.queryCollection(collection, {
|
|
3423
|
+
where: { id: recordId },
|
|
3424
|
+
includeDeleted: true,
|
|
3425
|
+
limit: 1
|
|
3426
|
+
});
|
|
3427
|
+
return rows[0];
|
|
3428
|
+
} catch {
|
|
3429
|
+
return void 0;
|
|
3430
|
+
}
|
|
2536
3431
|
}
|
|
2537
3432
|
handleAwarenessUpdate(msg) {
|
|
2538
3433
|
this.onAwarenessUpdate?.(this.sessionId, msg);
|
|
@@ -2550,6 +3445,25 @@ var ClientSession = class {
|
|
|
2550
3445
|
};
|
|
2551
3446
|
this.sendToClient(errorMsg);
|
|
2552
3447
|
}
|
|
3448
|
+
/**
|
|
3449
|
+
* Reject one specific client operation, tied to its id, so the submitter can
|
|
3450
|
+
* divert it out of its pending queue into a durable rejected store rather than
|
|
3451
|
+
* losing it or retrying forever. The op is NOT materialized, so no other
|
|
3452
|
+
* replica ever sees it.
|
|
3453
|
+
*/
|
|
3454
|
+
sendOperationRejected(op, code, message, retriable) {
|
|
3455
|
+
const rejectedMsg = {
|
|
3456
|
+
type: "operation-rejected",
|
|
3457
|
+
messageId: generateUUIDv74(),
|
|
3458
|
+
operationId: op.id,
|
|
3459
|
+
collection: op.collection,
|
|
3460
|
+
recordId: op.recordId,
|
|
3461
|
+
code,
|
|
3462
|
+
message,
|
|
3463
|
+
retriable
|
|
3464
|
+
};
|
|
3465
|
+
this.sendToClient(rejectedMsg);
|
|
3466
|
+
}
|
|
2553
3467
|
setSerializerWireFormat(format) {
|
|
2554
3468
|
if (typeof this.serializer.setWireFormat === "function") {
|
|
2555
3469
|
this.serializer.setWireFormat(format);
|
|
@@ -2558,6 +3472,7 @@ var ClientSession = class {
|
|
|
2558
3472
|
handleTransportClose() {
|
|
2559
3473
|
if (this.state === "closed") return;
|
|
2560
3474
|
this.state = "closed";
|
|
3475
|
+
this.flushOrphanedRelays();
|
|
2561
3476
|
this.emitter?.emit({ type: "sync:disconnected", reason: "transport closed" });
|
|
2562
3477
|
this.onClose?.(this.sessionId);
|
|
2563
3478
|
}
|
|
@@ -2570,7 +3485,7 @@ function selectWireFormat(supportedWireFormats) {
|
|
|
2570
3485
|
}
|
|
2571
3486
|
|
|
2572
3487
|
// src/server/kora-sync-server.ts
|
|
2573
|
-
import { SyncError as SyncError4, generateUUIDv7 as
|
|
3488
|
+
import { SyncError as SyncError4, generateUUIDv7 as generateUUIDv77, isBlobRef } from "@korajs/core";
|
|
2574
3489
|
import { SimpleEventEmitter } from "@korajs/core/internal";
|
|
2575
3490
|
import { JsonMessageSerializer as JsonMessageSerializer2 } from "@korajs/sync";
|
|
2576
3491
|
|
|
@@ -2787,6 +3702,115 @@ var ServerMetricsCollector = class {
|
|
|
2787
3702
|
}
|
|
2788
3703
|
};
|
|
2789
3704
|
|
|
3705
|
+
// src/richtext/blob-chunk-relay.ts
|
|
3706
|
+
import {
|
|
3707
|
+
encodeBlobChunkBytes
|
|
3708
|
+
} from "@korajs/sync";
|
|
3709
|
+
var BlobChunkRelay = class {
|
|
3710
|
+
clients = /* @__PURE__ */ new Map();
|
|
3711
|
+
/** requestId -> the session that originated the request (to route the answer back). */
|
|
3712
|
+
pending = /* @__PURE__ */ new Map();
|
|
3713
|
+
resolveBlobChunk;
|
|
3714
|
+
constructor(resolveBlobChunk) {
|
|
3715
|
+
this.resolveBlobChunk = resolveBlobChunk ?? null;
|
|
3716
|
+
}
|
|
3717
|
+
addClient(sessionId, transport) {
|
|
3718
|
+
this.clients.set(sessionId, { sessionId, transport });
|
|
3719
|
+
}
|
|
3720
|
+
removeClient(sessionId) {
|
|
3721
|
+
this.clients.delete(sessionId);
|
|
3722
|
+
for (const [requestId, originSessionId] of this.pending) {
|
|
3723
|
+
if (originSessionId === sessionId) {
|
|
3724
|
+
this.pending.delete(requestId);
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
}
|
|
3728
|
+
getClientCount() {
|
|
3729
|
+
return this.clients.size;
|
|
3730
|
+
}
|
|
3731
|
+
getPendingCount() {
|
|
3732
|
+
return this.pending.size;
|
|
3733
|
+
}
|
|
3734
|
+
clear() {
|
|
3735
|
+
this.clients.clear();
|
|
3736
|
+
this.pending.clear();
|
|
3737
|
+
}
|
|
3738
|
+
/**
|
|
3739
|
+
* Handle an inbound chunk request from a session. Tries the server's own
|
|
3740
|
+
* store first (if configured), otherwise forwards the request to peer
|
|
3741
|
+
* sessions and remembers who to route the answer back to.
|
|
3742
|
+
*/
|
|
3743
|
+
handleRequest(sourceSessionId, message) {
|
|
3744
|
+
if (!this.clients.has(sourceSessionId)) {
|
|
3745
|
+
return;
|
|
3746
|
+
}
|
|
3747
|
+
if (this.resolveBlobChunk) {
|
|
3748
|
+
void this.resolveBlobChunk(message.hash).then(
|
|
3749
|
+
(bytes) => {
|
|
3750
|
+
if (bytes !== null) {
|
|
3751
|
+
this.sendResponseTo(sourceSessionId, message.requestId, encodeBlobChunkBytes(bytes));
|
|
3752
|
+
return;
|
|
3753
|
+
}
|
|
3754
|
+
this.forwardRequestToPeers(sourceSessionId, message);
|
|
3755
|
+
},
|
|
3756
|
+
() => {
|
|
3757
|
+
this.forwardRequestToPeers(sourceSessionId, message);
|
|
3758
|
+
}
|
|
3759
|
+
);
|
|
3760
|
+
return;
|
|
3761
|
+
}
|
|
3762
|
+
this.forwardRequestToPeers(sourceSessionId, message);
|
|
3763
|
+
}
|
|
3764
|
+
/**
|
|
3765
|
+
* Handle an inbound chunk response from a peer. Routes it back to the session
|
|
3766
|
+
* that originally requested it. A "not held" answer (bytes === null) is
|
|
3767
|
+
* ignored so a peer without the chunk does not preempt one that has it; the
|
|
3768
|
+
* requester's own per-request timeout bounds the wait.
|
|
3769
|
+
*/
|
|
3770
|
+
handleResponse(sourceSessionId, message) {
|
|
3771
|
+
if (!this.clients.has(sourceSessionId)) {
|
|
3772
|
+
return;
|
|
3773
|
+
}
|
|
3774
|
+
if (message.bytes === null) {
|
|
3775
|
+
return;
|
|
3776
|
+
}
|
|
3777
|
+
const originSessionId = this.pending.get(message.requestId);
|
|
3778
|
+
if (!originSessionId) {
|
|
3779
|
+
return;
|
|
3780
|
+
}
|
|
3781
|
+
this.pending.delete(message.requestId);
|
|
3782
|
+
this.sendResponseTo(originSessionId, message.requestId, message.bytes);
|
|
3783
|
+
}
|
|
3784
|
+
forwardRequestToPeers(sourceSessionId, message) {
|
|
3785
|
+
this.pending.set(message.requestId, sourceSessionId);
|
|
3786
|
+
for (const [, client] of this.clients) {
|
|
3787
|
+
if (client.sessionId === sourceSessionId) {
|
|
3788
|
+
continue;
|
|
3789
|
+
}
|
|
3790
|
+
if (!client.transport.isConnected()) {
|
|
3791
|
+
continue;
|
|
3792
|
+
}
|
|
3793
|
+
client.transport.send(message);
|
|
3794
|
+
}
|
|
3795
|
+
}
|
|
3796
|
+
sendResponseTo(sessionId, requestId, bytes) {
|
|
3797
|
+
const client = this.clients.get(sessionId);
|
|
3798
|
+
if (!client || !client.transport.isConnected()) {
|
|
3799
|
+
return;
|
|
3800
|
+
}
|
|
3801
|
+
const response = {
|
|
3802
|
+
type: "blob-chunk-response",
|
|
3803
|
+
messageId: generateResponseId(requestId),
|
|
3804
|
+
requestId,
|
|
3805
|
+
bytes
|
|
3806
|
+
};
|
|
3807
|
+
client.transport.send(response);
|
|
3808
|
+
}
|
|
3809
|
+
};
|
|
3810
|
+
function generateResponseId(requestId) {
|
|
3811
|
+
return `blob-resp-${requestId}`;
|
|
3812
|
+
}
|
|
3813
|
+
|
|
2790
3814
|
// src/richtext/yjs-doc-relay.ts
|
|
2791
3815
|
var YjsDocRelay = class {
|
|
2792
3816
|
clients = /* @__PURE__ */ new Map();
|
|
@@ -2821,21 +3845,421 @@ var YjsDocRelay = class {
|
|
|
2821
3845
|
}
|
|
2822
3846
|
};
|
|
2823
3847
|
|
|
3848
|
+
// src/server/route-context.ts
|
|
3849
|
+
import {
|
|
3850
|
+
HybridLogicalClock as HybridLogicalClock3,
|
|
3851
|
+
createOperation as createOperation2,
|
|
3852
|
+
generateUUIDv7 as generateUUIDv76,
|
|
3853
|
+
isAtomicOp,
|
|
3854
|
+
resolveAtomicOp,
|
|
3855
|
+
toAtomicOp
|
|
3856
|
+
} from "@korajs/core";
|
|
3857
|
+
|
|
3858
|
+
// src/apply/route-predicate.ts
|
|
3859
|
+
function evaluateField(actual, operators) {
|
|
3860
|
+
if ("$eq" in operators && !Object.is(actual, operators.$eq)) {
|
|
3861
|
+
return false;
|
|
3862
|
+
}
|
|
3863
|
+
if ("$ne" in operators && Object.is(actual, operators.$ne)) {
|
|
3864
|
+
return false;
|
|
3865
|
+
}
|
|
3866
|
+
if (operators.$in !== void 0 && !operators.$in.some((candidate) => Object.is(actual, candidate))) {
|
|
3867
|
+
return false;
|
|
3868
|
+
}
|
|
3869
|
+
const numericChecks = [
|
|
3870
|
+
["$lt", (a, b) => a < b],
|
|
3871
|
+
["$lte", (a, b) => a <= b],
|
|
3872
|
+
["$gt", (a, b) => a > b],
|
|
3873
|
+
["$gte", (a, b) => a >= b]
|
|
3874
|
+
];
|
|
3875
|
+
for (const [operator, compare] of numericChecks) {
|
|
3876
|
+
const bound = operators[operator];
|
|
3877
|
+
if (bound === void 0) {
|
|
3878
|
+
continue;
|
|
3879
|
+
}
|
|
3880
|
+
if (typeof actual !== "number" || typeof bound !== "number") {
|
|
3881
|
+
return false;
|
|
3882
|
+
}
|
|
3883
|
+
if (!compare(actual, bound)) {
|
|
3884
|
+
return false;
|
|
3885
|
+
}
|
|
3886
|
+
}
|
|
3887
|
+
return true;
|
|
3888
|
+
}
|
|
3889
|
+
function evaluateRoutePredicate(record, predicate) {
|
|
3890
|
+
const fields = Object.keys(predicate);
|
|
3891
|
+
if (fields.length === 0) {
|
|
3892
|
+
return true;
|
|
3893
|
+
}
|
|
3894
|
+
if (record === null) {
|
|
3895
|
+
return false;
|
|
3896
|
+
}
|
|
3897
|
+
for (const field of fields) {
|
|
3898
|
+
const operators = predicate[field];
|
|
3899
|
+
if (operators && !evaluateField(record[field], operators)) {
|
|
3900
|
+
return false;
|
|
3901
|
+
}
|
|
3902
|
+
}
|
|
3903
|
+
return true;
|
|
3904
|
+
}
|
|
3905
|
+
|
|
3906
|
+
// src/server/route-context.ts
|
|
3907
|
+
function recordMatchesScope(collection, record, scope) {
|
|
3908
|
+
const collectionScope = scope[collection];
|
|
3909
|
+
if (!collectionScope) {
|
|
3910
|
+
return false;
|
|
3911
|
+
}
|
|
3912
|
+
for (const [field, expected] of Object.entries(collectionScope)) {
|
|
3913
|
+
if (record[field] !== expected) {
|
|
3914
|
+
return false;
|
|
3915
|
+
}
|
|
3916
|
+
}
|
|
3917
|
+
return true;
|
|
3918
|
+
}
|
|
3919
|
+
function pickFields(record, keys) {
|
|
3920
|
+
const picked = {};
|
|
3921
|
+
for (const key of keys) {
|
|
3922
|
+
if (key in record) {
|
|
3923
|
+
picked[key] = record[key];
|
|
3924
|
+
}
|
|
3925
|
+
}
|
|
3926
|
+
return picked;
|
|
3927
|
+
}
|
|
3928
|
+
async function buildRouteOperation(store, mutation, currentOverride, clockOverride) {
|
|
3929
|
+
const nodeId = store.getNodeId();
|
|
3930
|
+
const clock = clockOverride ?? new HybridLogicalClock3(nodeId);
|
|
3931
|
+
const schemaVersion = store.getSchema()?.version ?? 1;
|
|
3932
|
+
const sequenceNumber = nextServerSequenceNumber(store);
|
|
3933
|
+
if (mutation.type === "insert") {
|
|
3934
|
+
return createOperation2(
|
|
3935
|
+
{
|
|
3936
|
+
nodeId,
|
|
3937
|
+
type: "insert",
|
|
3938
|
+
collection: mutation.collection,
|
|
3939
|
+
recordId: mutation.recordId ?? generateUUIDv76(),
|
|
3940
|
+
data: mutation.data ?? {},
|
|
3941
|
+
previousData: null,
|
|
3942
|
+
sequenceNumber,
|
|
3943
|
+
causalDeps: [],
|
|
3944
|
+
schemaVersion
|
|
3945
|
+
},
|
|
3946
|
+
clock
|
|
3947
|
+
);
|
|
3948
|
+
}
|
|
3949
|
+
if (!mutation.recordId) {
|
|
3950
|
+
throw new RouteMutationError(
|
|
3951
|
+
"MISSING_RECORD_ID",
|
|
3952
|
+
`A ${mutation.type} mutation on "${mutation.collection}" requires a recordId.`
|
|
3953
|
+
);
|
|
3954
|
+
}
|
|
3955
|
+
const current = currentOverride ? currentOverride.record : await store.findRecord(mutation.collection, mutation.recordId);
|
|
3956
|
+
if (mutation.type === "update") {
|
|
3957
|
+
const rawData = mutation.data ?? {};
|
|
3958
|
+
const data = {};
|
|
3959
|
+
const atomicOps = {};
|
|
3960
|
+
for (const key of Object.keys(rawData)) {
|
|
3961
|
+
const value = rawData[key];
|
|
3962
|
+
if (isAtomicOp(value)) {
|
|
3963
|
+
data[key] = resolveAtomicOp(current ? current[key] : void 0, value);
|
|
3964
|
+
atomicOps[key] = toAtomicOp(value);
|
|
3965
|
+
} else {
|
|
3966
|
+
data[key] = value;
|
|
3967
|
+
}
|
|
3968
|
+
}
|
|
3969
|
+
return createOperation2(
|
|
3970
|
+
{
|
|
3971
|
+
nodeId,
|
|
3972
|
+
type: "update",
|
|
3973
|
+
collection: mutation.collection,
|
|
3974
|
+
recordId: mutation.recordId,
|
|
3975
|
+
data,
|
|
3976
|
+
previousData: current ? pickFields(current, Object.keys(rawData)) : {},
|
|
3977
|
+
sequenceNumber,
|
|
3978
|
+
causalDeps: [],
|
|
3979
|
+
schemaVersion,
|
|
3980
|
+
...Object.keys(atomicOps).length > 0 ? { atomicOps } : {}
|
|
3981
|
+
},
|
|
3982
|
+
clock
|
|
3983
|
+
);
|
|
3984
|
+
}
|
|
3985
|
+
return createOperation2(
|
|
3986
|
+
{
|
|
3987
|
+
nodeId,
|
|
3988
|
+
type: "delete",
|
|
3989
|
+
collection: mutation.collection,
|
|
3990
|
+
recordId: mutation.recordId,
|
|
3991
|
+
data: null,
|
|
3992
|
+
previousData: current ? { ...current } : null,
|
|
3993
|
+
sequenceNumber,
|
|
3994
|
+
causalDeps: [],
|
|
3995
|
+
schemaVersion
|
|
3996
|
+
},
|
|
3997
|
+
clock
|
|
3998
|
+
);
|
|
3999
|
+
}
|
|
4000
|
+
var RouteMutationError = class extends Error {
|
|
4001
|
+
constructor(code, message) {
|
|
4002
|
+
super(message);
|
|
4003
|
+
this.code = code;
|
|
4004
|
+
this.name = "RouteMutationError";
|
|
4005
|
+
}
|
|
4006
|
+
code;
|
|
4007
|
+
};
|
|
4008
|
+
function createRouteContext(server, store) {
|
|
4009
|
+
let mutationTail = Promise.resolve();
|
|
4010
|
+
function serialize(work) {
|
|
4011
|
+
const run = mutationTail.then(work, work);
|
|
4012
|
+
mutationTail = run.then(
|
|
4013
|
+
() => void 0,
|
|
4014
|
+
() => void 0
|
|
4015
|
+
);
|
|
4016
|
+
return run;
|
|
4017
|
+
}
|
|
4018
|
+
async function prepareMutation(mutation, scope) {
|
|
4019
|
+
let op;
|
|
4020
|
+
try {
|
|
4021
|
+
op = await buildRouteOperation(store, mutation);
|
|
4022
|
+
} catch (error) {
|
|
4023
|
+
if (error instanceof RouteMutationError) {
|
|
4024
|
+
return {
|
|
4025
|
+
ok: false,
|
|
4026
|
+
rejection: {
|
|
4027
|
+
ok: false,
|
|
4028
|
+
code: error.code,
|
|
4029
|
+
message: error.message,
|
|
4030
|
+
retriable: isRetriableRejection(error.code)
|
|
4031
|
+
}
|
|
4032
|
+
};
|
|
4033
|
+
}
|
|
4034
|
+
throw error;
|
|
4035
|
+
}
|
|
4036
|
+
if (scope && !operationMatchesScopes(op, scope)) {
|
|
4037
|
+
return {
|
|
4038
|
+
ok: false,
|
|
4039
|
+
rejection: {
|
|
4040
|
+
ok: false,
|
|
4041
|
+
code: "SCOPE_VIOLATION",
|
|
4042
|
+
message: `Mutation on "${mutation.collection}" is outside the provided scope.`,
|
|
4043
|
+
retriable: isRetriableRejection("SCOPE_VIOLATION")
|
|
4044
|
+
}
|
|
4045
|
+
};
|
|
4046
|
+
}
|
|
4047
|
+
return { ok: true, op };
|
|
4048
|
+
}
|
|
4049
|
+
async function commitOperation(op, collection) {
|
|
4050
|
+
const result = await server.applyLocalOperation(op);
|
|
4051
|
+
if (result.result !== "applied") {
|
|
4052
|
+
const code = result.rejection?.code ?? "NOT_APPLIED";
|
|
4053
|
+
return {
|
|
4054
|
+
ok: false,
|
|
4055
|
+
code,
|
|
4056
|
+
message: result.rejection?.message ?? `Operation on "${collection}" was not applied (${result.result}).`,
|
|
4057
|
+
// Prefer the pipeline's own classification when present; fall back
|
|
4058
|
+
// to classifying the code so this stays correct if new codes appear.
|
|
4059
|
+
retriable: result.rejection?.retriable ?? isRetriableRejection(code)
|
|
4060
|
+
};
|
|
4061
|
+
}
|
|
4062
|
+
const record = await store.findRecord(collection, op.recordId);
|
|
4063
|
+
return { ok: true, operation: op, record };
|
|
4064
|
+
}
|
|
4065
|
+
async function rawApply(mutation, scope) {
|
|
4066
|
+
const prepared = await prepareMutation(mutation, scope);
|
|
4067
|
+
if (!prepared.ok) {
|
|
4068
|
+
return prepared.rejection;
|
|
4069
|
+
}
|
|
4070
|
+
return commitOperation(prepared.op, mutation.collection);
|
|
4071
|
+
}
|
|
4072
|
+
async function runConditionalViaStore(storeApplyConditional, spec, options) {
|
|
4073
|
+
const scope = options?.scope;
|
|
4074
|
+
const mutations = [];
|
|
4075
|
+
const hasTargetUpdate = spec.update !== void 0 && spec.update !== null;
|
|
4076
|
+
if (hasTargetUpdate) {
|
|
4077
|
+
mutations.push({
|
|
4078
|
+
collection: spec.collection,
|
|
4079
|
+
type: "update",
|
|
4080
|
+
recordId: spec.id,
|
|
4081
|
+
data: spec.update
|
|
4082
|
+
});
|
|
4083
|
+
}
|
|
4084
|
+
if (spec.also) {
|
|
4085
|
+
mutations.push(...spec.also);
|
|
4086
|
+
}
|
|
4087
|
+
let result;
|
|
4088
|
+
try {
|
|
4089
|
+
result = await storeApplyConditional({
|
|
4090
|
+
target: { collection: spec.collection, id: spec.id },
|
|
4091
|
+
...spec.idempotencyKey ? { idempotencyKey: spec.idempotencyKey } : {},
|
|
4092
|
+
admit: (current) => spec.if && Object.keys(spec.if).length > 0 ? evaluateRoutePredicate(current, spec.if) : true,
|
|
4093
|
+
buildOperations: async (current, context) => {
|
|
4094
|
+
const ops = [];
|
|
4095
|
+
for (let i = 0; i < mutations.length; i++) {
|
|
4096
|
+
const mutation = mutations[i];
|
|
4097
|
+
const useLockedCurrent = i === 0 && hasTargetUpdate;
|
|
4098
|
+
const op = await buildRouteOperation(
|
|
4099
|
+
store,
|
|
4100
|
+
mutation,
|
|
4101
|
+
useLockedCurrent ? { record: current } : void 0,
|
|
4102
|
+
context.clock
|
|
4103
|
+
);
|
|
4104
|
+
if (scope && !operationMatchesScopes(op, scope)) {
|
|
4105
|
+
throw new RouteMutationError(
|
|
4106
|
+
"SCOPE_VIOLATION",
|
|
4107
|
+
`Mutation on "${mutation.collection}" is outside the provided scope.`
|
|
4108
|
+
);
|
|
4109
|
+
}
|
|
4110
|
+
ops.push(op);
|
|
4111
|
+
}
|
|
4112
|
+
return ops;
|
|
4113
|
+
}
|
|
4114
|
+
});
|
|
4115
|
+
} catch (error) {
|
|
4116
|
+
if (error instanceof RouteMutationError) {
|
|
4117
|
+
return {
|
|
4118
|
+
ok: false,
|
|
4119
|
+
code: error.code,
|
|
4120
|
+
message: error.message,
|
|
4121
|
+
retriable: isRetriableRejection(error.code)
|
|
4122
|
+
};
|
|
4123
|
+
}
|
|
4124
|
+
throw error;
|
|
4125
|
+
}
|
|
4126
|
+
if (!result.admitted) {
|
|
4127
|
+
return {
|
|
4128
|
+
ok: false,
|
|
4129
|
+
code: spec.reject?.code ?? "CONDITION_NOT_MET",
|
|
4130
|
+
message: spec.reject?.message ?? `Admission condition not met on "${spec.collection}" record "${spec.id}".`,
|
|
4131
|
+
retriable: false
|
|
4132
|
+
};
|
|
4133
|
+
}
|
|
4134
|
+
if (result.idempotent) {
|
|
4135
|
+
const records2 = spec.idempotencyKey ? [await store.findRecord(spec.idempotencyKey.collection, spec.idempotencyKey.id)] : [];
|
|
4136
|
+
return { ok: true, idempotent: true, operations: [], records: records2 };
|
|
4137
|
+
}
|
|
4138
|
+
server.relayServerOperations(result.applied);
|
|
4139
|
+
const records = [];
|
|
4140
|
+
for (const op of result.applied) {
|
|
4141
|
+
records.push(await store.findRecord(op.collection, op.recordId));
|
|
4142
|
+
}
|
|
4143
|
+
return { ok: true, idempotent: false, operations: result.applied, records };
|
|
4144
|
+
}
|
|
4145
|
+
return {
|
|
4146
|
+
apply(mutation, options) {
|
|
4147
|
+
return serialize(() => rawApply(mutation, options?.scope));
|
|
4148
|
+
},
|
|
4149
|
+
applyConditional(spec, options) {
|
|
4150
|
+
const storeApplyConditional = store.applyConditional;
|
|
4151
|
+
if (storeApplyConditional) {
|
|
4152
|
+
return runConditionalViaStore(storeApplyConditional.bind(store), spec, options);
|
|
4153
|
+
}
|
|
4154
|
+
return serialize(async () => {
|
|
4155
|
+
if (spec.idempotencyKey) {
|
|
4156
|
+
const existing = await store.findRecord(
|
|
4157
|
+
spec.idempotencyKey.collection,
|
|
4158
|
+
spec.idempotencyKey.id
|
|
4159
|
+
);
|
|
4160
|
+
if (existing) {
|
|
4161
|
+
return { ok: true, idempotent: true, operations: [], records: [existing] };
|
|
4162
|
+
}
|
|
4163
|
+
}
|
|
4164
|
+
if (spec.if && Object.keys(spec.if).length > 0) {
|
|
4165
|
+
const target = await store.findRecord(spec.collection, spec.id);
|
|
4166
|
+
if (!evaluateRoutePredicate(target, spec.if)) {
|
|
4167
|
+
const code = spec.reject?.code ?? "CONDITION_NOT_MET";
|
|
4168
|
+
return {
|
|
4169
|
+
ok: false,
|
|
4170
|
+
code,
|
|
4171
|
+
message: spec.reject?.message ?? `Admission condition not met on "${spec.collection}" record "${spec.id}".`,
|
|
4172
|
+
retriable: false
|
|
4173
|
+
};
|
|
4174
|
+
}
|
|
4175
|
+
}
|
|
4176
|
+
const mutations = [];
|
|
4177
|
+
if (spec.update !== void 0 && spec.update !== null) {
|
|
4178
|
+
mutations.push({
|
|
4179
|
+
collection: spec.collection,
|
|
4180
|
+
type: "update",
|
|
4181
|
+
recordId: spec.id,
|
|
4182
|
+
data: spec.update
|
|
4183
|
+
});
|
|
4184
|
+
}
|
|
4185
|
+
if (spec.also) {
|
|
4186
|
+
mutations.push(...spec.also);
|
|
4187
|
+
}
|
|
4188
|
+
const prepared = [];
|
|
4189
|
+
for (const mutation of mutations) {
|
|
4190
|
+
const p = await prepareMutation(mutation, options?.scope);
|
|
4191
|
+
if (!p.ok) {
|
|
4192
|
+
return p.rejection;
|
|
4193
|
+
}
|
|
4194
|
+
prepared.push({ op: p.op, collection: mutation.collection });
|
|
4195
|
+
}
|
|
4196
|
+
const operations2 = [];
|
|
4197
|
+
const records = [];
|
|
4198
|
+
for (const { op, collection } of prepared) {
|
|
4199
|
+
const result = await commitOperation(op, collection);
|
|
4200
|
+
if (!result.ok) {
|
|
4201
|
+
return {
|
|
4202
|
+
ok: false,
|
|
4203
|
+
code: result.code,
|
|
4204
|
+
message: result.message,
|
|
4205
|
+
retriable: result.retriable
|
|
4206
|
+
};
|
|
4207
|
+
}
|
|
4208
|
+
operations2.push(result.operation);
|
|
4209
|
+
records.push(result.record);
|
|
4210
|
+
}
|
|
4211
|
+
return { ok: true, idempotent: false, operations: operations2, records };
|
|
4212
|
+
});
|
|
4213
|
+
},
|
|
4214
|
+
async query(collection, options) {
|
|
4215
|
+
const { scope, ...queryOptions } = options ?? {};
|
|
4216
|
+
const records = await store.queryCollection(collection, queryOptions);
|
|
4217
|
+
if (!scope) {
|
|
4218
|
+
return records;
|
|
4219
|
+
}
|
|
4220
|
+
return records.filter((record) => recordMatchesScope(collection, record, scope));
|
|
4221
|
+
},
|
|
4222
|
+
async findById(collection, id, options) {
|
|
4223
|
+
const record = await store.findRecord(collection, id);
|
|
4224
|
+
if (!record) {
|
|
4225
|
+
return null;
|
|
4226
|
+
}
|
|
4227
|
+
if (options?.scope && !recordMatchesScope(collection, record, options.scope)) {
|
|
4228
|
+
return null;
|
|
4229
|
+
}
|
|
4230
|
+
return record;
|
|
4231
|
+
}
|
|
4232
|
+
};
|
|
4233
|
+
}
|
|
4234
|
+
|
|
2824
4235
|
// src/server/kora-sync-server.ts
|
|
2825
4236
|
var DEFAULT_MAX_CONNECTIONS = 0;
|
|
2826
4237
|
var DEFAULT_BATCH_SIZE2 = 100;
|
|
2827
4238
|
var DEFAULT_SCHEMA_VERSION2 = 1;
|
|
2828
4239
|
var DEFAULT_HOST = "0.0.0.0";
|
|
2829
4240
|
var DEFAULT_PATH = "/";
|
|
2830
|
-
var
|
|
4241
|
+
var DEFAULT_RELAY_RETRANSMIT_INTERVAL_MS = 2e3;
|
|
4242
|
+
var DEFAULT_DELIVERY_POLL_INTERVAL_MS = 2e3;
|
|
4243
|
+
function validateIntervalOption(name, value) {
|
|
4244
|
+
if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0) {
|
|
4245
|
+
throw new SyncError4(`${name} must be a non-negative integer number of milliseconds`, {
|
|
4246
|
+
[name]: value
|
|
4247
|
+
});
|
|
4248
|
+
}
|
|
4249
|
+
return value;
|
|
4250
|
+
}
|
|
4251
|
+
var KoraSyncServer = class _KoraSyncServer {
|
|
2831
4252
|
store;
|
|
2832
4253
|
auth;
|
|
2833
4254
|
serializer;
|
|
2834
4255
|
emitter;
|
|
2835
4256
|
maxConnections;
|
|
2836
4257
|
batchSize;
|
|
4258
|
+
relayRetransmitIntervalMs;
|
|
4259
|
+
deliveryPollIntervalMs;
|
|
2837
4260
|
schemaVersion;
|
|
2838
4261
|
supportedSchemaVersions;
|
|
4262
|
+
operationTransforms;
|
|
2839
4263
|
port;
|
|
2840
4264
|
host;
|
|
2841
4265
|
path;
|
|
@@ -2843,34 +4267,183 @@ var KoraSyncServer = class {
|
|
|
2843
4267
|
metrics;
|
|
2844
4268
|
awarenessRelay = new AwarenessRelay();
|
|
2845
4269
|
yjsDocRelay = new YjsDocRelay();
|
|
4270
|
+
blobChunkRelay;
|
|
4271
|
+
persistBlobChunk;
|
|
4272
|
+
maxOperationBytes;
|
|
4273
|
+
maxOpsPerMinute;
|
|
4274
|
+
validateOperation;
|
|
4275
|
+
koraContext;
|
|
2846
4276
|
sessions = /* @__PURE__ */ new Map();
|
|
2847
4277
|
httpClients = /* @__PURE__ */ new Map();
|
|
2848
4278
|
httpSessionToClient = /* @__PURE__ */ new Map();
|
|
2849
|
-
|
|
4279
|
+
// Informational value reported by getStatus(). Keep in sync with the
|
|
4280
|
+
// @korajs/server package version on release; it is not used for protocol negotiation.
|
|
4281
|
+
serverVersion = "1.0.0-beta.0";
|
|
2850
4282
|
wsServer = null;
|
|
2851
4283
|
running = false;
|
|
4284
|
+
/**
|
|
4285
|
+
* Periodic tick that retransmits relay batches connected clients have not
|
|
4286
|
+
* acknowledged, so a relay dropped by a lossy transport is redelivered instead of
|
|
4287
|
+
* leaving that client with a permanent version-vector gap (a lost operation).
|
|
4288
|
+
*/
|
|
4289
|
+
relayRetransmitTimer = null;
|
|
4290
|
+
deliveryPollTimer = null;
|
|
4291
|
+
lastObservedDeliverySequence = 0;
|
|
4292
|
+
deliveryPollInFlight = false;
|
|
4293
|
+
/**
|
|
4294
|
+
* Relay operations a client never acknowledged before it disconnected, buffered by
|
|
4295
|
+
* client node id so they can be redelivered on its next connection. Bounded per
|
|
4296
|
+
* node and expired by age so a client that never returns cannot grow this forever.
|
|
4297
|
+
*/
|
|
4298
|
+
orphanedRelaysByNode = /* @__PURE__ */ new Map();
|
|
4299
|
+
static MAX_ORPHANED_RELAY_OPS_PER_NODE = 5e3;
|
|
4300
|
+
static ORPHANED_RELAY_TTL_MS = 5 * 6e4;
|
|
2852
4301
|
constructor(config) {
|
|
2853
4302
|
this.store = config.store;
|
|
4303
|
+
const storeSchemaVersion = this.store.getSchema()?.version;
|
|
4304
|
+
if (config.schemaVersion !== void 0 && storeSchemaVersion !== void 0 && config.schemaVersion !== storeSchemaVersion) {
|
|
4305
|
+
throw new SyncError4("Sync schema version does not match the configured store schema.", {
|
|
4306
|
+
syncSchemaVersion: config.schemaVersion,
|
|
4307
|
+
storeSchemaVersion
|
|
4308
|
+
});
|
|
4309
|
+
}
|
|
2854
4310
|
this.auth = config.auth ?? null;
|
|
2855
4311
|
this.serializer = config.serializer ?? new JsonMessageSerializer2();
|
|
2856
4312
|
this.emitter = config.emitter ?? null;
|
|
2857
4313
|
this.maxConnections = config.maxConnections ?? DEFAULT_MAX_CONNECTIONS;
|
|
2858
4314
|
this.batchSize = config.batchSize ?? DEFAULT_BATCH_SIZE2;
|
|
2859
|
-
this.
|
|
4315
|
+
this.relayRetransmitIntervalMs = validateIntervalOption(
|
|
4316
|
+
"relayRetransmitIntervalMs",
|
|
4317
|
+
config.relayRetransmitIntervalMs ?? DEFAULT_RELAY_RETRANSMIT_INTERVAL_MS
|
|
4318
|
+
);
|
|
4319
|
+
this.deliveryPollIntervalMs = validateIntervalOption(
|
|
4320
|
+
"deliveryPollIntervalMs",
|
|
4321
|
+
config.deliveryPollIntervalMs ?? DEFAULT_DELIVERY_POLL_INTERVAL_MS
|
|
4322
|
+
);
|
|
4323
|
+
this.schemaVersion = config.schemaVersion ?? storeSchemaVersion ?? DEFAULT_SCHEMA_VERSION2;
|
|
2860
4324
|
this.supportedSchemaVersions = config.supportedSchemaVersions ?? {
|
|
2861
4325
|
min: this.schemaVersion,
|
|
2862
4326
|
max: this.schemaVersion
|
|
2863
4327
|
};
|
|
4328
|
+
this.operationTransforms = config.operationTransforms ?? [];
|
|
2864
4329
|
this.port = config.port;
|
|
2865
4330
|
this.host = config.host ?? DEFAULT_HOST;
|
|
2866
4331
|
this.path = config.path ?? DEFAULT_PATH;
|
|
2867
4332
|
this.logger = config.logger ?? createDefaultLogger();
|
|
2868
4333
|
this.metrics = config.metricsCollector ?? new ServerMetricsCollector();
|
|
2869
4334
|
this.metrics.setSchemaVersion(this.schemaVersion);
|
|
4335
|
+
this.blobChunkRelay = new BlobChunkRelay(config.resolveBlobChunk);
|
|
4336
|
+
this.persistBlobChunk = config.persistBlobChunk ?? null;
|
|
4337
|
+
this.maxOperationBytes = config.maxOperationBytes;
|
|
4338
|
+
this.maxOpsPerMinute = config.maxOpsPerMinute;
|
|
4339
|
+
this.validateOperation = config.validateOperation;
|
|
4340
|
+
this.koraContext = createRouteContext(this, this.store);
|
|
2870
4341
|
if (!this.emitter) {
|
|
2871
4342
|
this.emitter = new SimpleEventEmitter();
|
|
2872
4343
|
}
|
|
2873
4344
|
}
|
|
4345
|
+
ensureBackgroundTimersStarted() {
|
|
4346
|
+
if (this.relayRetransmitIntervalMs > 0 && !this.relayRetransmitTimer) {
|
|
4347
|
+
this.relayRetransmitTimer = setInterval(() => {
|
|
4348
|
+
this.retransmitPendingRelays(this.relayRetransmitIntervalMs);
|
|
4349
|
+
}, this.relayRetransmitIntervalMs);
|
|
4350
|
+
this.relayRetransmitTimer.unref?.();
|
|
4351
|
+
}
|
|
4352
|
+
if (this.deliveryPollIntervalMs > 0 && !this.deliveryPollTimer) {
|
|
4353
|
+
this.deliveryPollTimer = setInterval(() => {
|
|
4354
|
+
void this.pollDeliveryLog();
|
|
4355
|
+
}, this.deliveryPollIntervalMs);
|
|
4356
|
+
this.deliveryPollTimer.unref?.();
|
|
4357
|
+
}
|
|
4358
|
+
}
|
|
4359
|
+
/**
|
|
4360
|
+
* Retransmit relay batches that connected clients have not acknowledged within
|
|
4361
|
+
* `staleMs`. Called on a periodic tick and directly by tests. Redelivering an
|
|
4362
|
+
* already-applied operation is harmless (clients dedup by content-addressed id).
|
|
4363
|
+
*/
|
|
4364
|
+
retransmitPendingRelays(staleMs = 0) {
|
|
4365
|
+
for (const session of this.sessions.values()) {
|
|
4366
|
+
session.retransmitPendingRelays(staleMs);
|
|
4367
|
+
}
|
|
4368
|
+
this.expireOrphanedRelays();
|
|
4369
|
+
}
|
|
4370
|
+
/**
|
|
4371
|
+
* Check whether the authoritative delivery log advanced without passing through
|
|
4372
|
+
* this server instance, then wake delivery-watermark sessions to scan from their
|
|
4373
|
+
* own acknowledged cursors. Session-level filtering remains the only fan-out path.
|
|
4374
|
+
*/
|
|
4375
|
+
async pollDeliveryLog() {
|
|
4376
|
+
if (this.deliveryPollInFlight) return;
|
|
4377
|
+
this.deliveryPollInFlight = true;
|
|
4378
|
+
try {
|
|
4379
|
+
const maxDeliverySequence = await this.store.getMaxDeliverySequence();
|
|
4380
|
+
if (maxDeliverySequence <= this.lastObservedDeliverySequence) {
|
|
4381
|
+
for (const session of this.sessions.values()) {
|
|
4382
|
+
session.pushDeliveryStreamIfSupported(this.deliveryPollIntervalMs, {
|
|
4383
|
+
trackStall: true
|
|
4384
|
+
});
|
|
4385
|
+
}
|
|
4386
|
+
return;
|
|
4387
|
+
}
|
|
4388
|
+
this.lastObservedDeliverySequence = maxDeliverySequence;
|
|
4389
|
+
for (const session of this.sessions.values()) {
|
|
4390
|
+
session.pushDeliveryStreamIfSupported(0);
|
|
4391
|
+
}
|
|
4392
|
+
} catch (error) {
|
|
4393
|
+
this.logger.log({
|
|
4394
|
+
timestamp: Date.now(),
|
|
4395
|
+
level: "warn",
|
|
4396
|
+
event: "delivery_poll.failed",
|
|
4397
|
+
details: { error: error instanceof Error ? error.message : String(error) }
|
|
4398
|
+
});
|
|
4399
|
+
} finally {
|
|
4400
|
+
this.deliveryPollInFlight = false;
|
|
4401
|
+
}
|
|
4402
|
+
}
|
|
4403
|
+
/**
|
|
4404
|
+
* Buffer relay operations a disconnected client never acknowledged, keyed by its
|
|
4405
|
+
* node id, deduped by operation id and bounded by count. Redelivered when the
|
|
4406
|
+
* client reconnects (see {@link takeOrphanedRelays}).
|
|
4407
|
+
*/
|
|
4408
|
+
bufferOrphanedRelays(nodeId, ops) {
|
|
4409
|
+
if (ops.length === 0) return;
|
|
4410
|
+
const existing = this.orphanedRelaysByNode.get(nodeId);
|
|
4411
|
+
const merged = existing ? existing.ops : [];
|
|
4412
|
+
const seen = new Set(merged.map((op) => op.id));
|
|
4413
|
+
for (const op of ops) {
|
|
4414
|
+
if (!seen.has(op.id)) {
|
|
4415
|
+
merged.push(op);
|
|
4416
|
+
seen.add(op.id);
|
|
4417
|
+
}
|
|
4418
|
+
}
|
|
4419
|
+
const bounded = merged.length > _KoraSyncServer.MAX_ORPHANED_RELAY_OPS_PER_NODE ? merged.slice(merged.length - _KoraSyncServer.MAX_ORPHANED_RELAY_OPS_PER_NODE) : merged;
|
|
4420
|
+
this.orphanedRelaysByNode.set(nodeId, { ops: bounded, bufferedAtMs: Date.now() });
|
|
4421
|
+
}
|
|
4422
|
+
/** Remove and return a node's buffered orphaned relay operations, if any. */
|
|
4423
|
+
takeOrphanedRelays(nodeId) {
|
|
4424
|
+
const entry = this.orphanedRelaysByNode.get(nodeId);
|
|
4425
|
+
if (!entry) return [];
|
|
4426
|
+
this.orphanedRelaysByNode.delete(nodeId);
|
|
4427
|
+
return entry.ops;
|
|
4428
|
+
}
|
|
4429
|
+
/** Drop buffered orphaned relays older than the TTL (client never returned). */
|
|
4430
|
+
expireOrphanedRelays() {
|
|
4431
|
+
if (this.orphanedRelaysByNode.size === 0) return;
|
|
4432
|
+
const cutoff = Date.now() - _KoraSyncServer.ORPHANED_RELAY_TTL_MS;
|
|
4433
|
+
for (const [nodeId, entry] of this.orphanedRelaysByNode) {
|
|
4434
|
+
if (entry.bufferedAtMs <= cutoff) {
|
|
4435
|
+
this.orphanedRelaysByNode.delete(nodeId);
|
|
4436
|
+
}
|
|
4437
|
+
}
|
|
4438
|
+
}
|
|
4439
|
+
/**
|
|
4440
|
+
* The trusted, scoped data-plane context (`apply` / `query` / `findById`).
|
|
4441
|
+
* Shared with the production server so HTTP routes, the operation validator,
|
|
4442
|
+
* and `server.kora` are all the same object over the same pipeline.
|
|
4443
|
+
*/
|
|
4444
|
+
getKoraContext() {
|
|
4445
|
+
return this.koraContext;
|
|
4446
|
+
}
|
|
2874
4447
|
/**
|
|
2875
4448
|
* Subscribe to session-level events for metrics collection and logging.
|
|
2876
4449
|
* Called when a new session is created.
|
|
@@ -2919,6 +4492,21 @@ var KoraSyncServer = class {
|
|
|
2919
4492
|
details: { reason: event.reason }
|
|
2920
4493
|
});
|
|
2921
4494
|
});
|
|
4495
|
+
sessionEmitter.on("sync:schema-mismatch", (event) => {
|
|
4496
|
+
this.logger.log({
|
|
4497
|
+
timestamp: Date.now(),
|
|
4498
|
+
level: "warn",
|
|
4499
|
+
event: "session.schema_mismatch",
|
|
4500
|
+
sessionId,
|
|
4501
|
+
details: {
|
|
4502
|
+
clientSchemaVersion: event.clientSchemaVersion,
|
|
4503
|
+
serverSchemaVersion: event.serverSchemaVersion,
|
|
4504
|
+
supportedMin: event.supportedMin,
|
|
4505
|
+
supportedMax: event.supportedMax,
|
|
4506
|
+
reason: event.reason
|
|
4507
|
+
}
|
|
4508
|
+
});
|
|
4509
|
+
});
|
|
2922
4510
|
}
|
|
2923
4511
|
/**
|
|
2924
4512
|
* Get the metrics collector for external access (e.g., HTTP endpoints).
|
|
@@ -2988,8 +4576,18 @@ var KoraSyncServer = class {
|
|
|
2988
4576
|
event: "server.stopping",
|
|
2989
4577
|
details: { connectedClients: this.sessions.size }
|
|
2990
4578
|
});
|
|
4579
|
+
if (this.relayRetransmitTimer) {
|
|
4580
|
+
clearInterval(this.relayRetransmitTimer);
|
|
4581
|
+
this.relayRetransmitTimer = null;
|
|
4582
|
+
}
|
|
4583
|
+
if (this.deliveryPollTimer) {
|
|
4584
|
+
clearInterval(this.deliveryPollTimer);
|
|
4585
|
+
this.deliveryPollTimer = null;
|
|
4586
|
+
}
|
|
4587
|
+
this.orphanedRelaysByNode.clear();
|
|
2991
4588
|
this.awarenessRelay.clear();
|
|
2992
4589
|
this.yjsDocRelay.clear();
|
|
4590
|
+
this.blobChunkRelay.clear();
|
|
2993
4591
|
for (const session of this.sessions.values()) {
|
|
2994
4592
|
session.close("server shutting down");
|
|
2995
4593
|
}
|
|
@@ -3051,7 +4649,7 @@ var KoraSyncServer = class {
|
|
|
3051
4649
|
if (this.maxConnections > 0 && this.sessions.size >= this.maxConnections) {
|
|
3052
4650
|
transport.send({
|
|
3053
4651
|
type: "error",
|
|
3054
|
-
messageId:
|
|
4652
|
+
messageId: generateUUIDv77(),
|
|
3055
4653
|
code: "MAX_CONNECTIONS",
|
|
3056
4654
|
message: `Server has reached maximum connections (${this.maxConnections})`,
|
|
3057
4655
|
retriable: true
|
|
@@ -3069,7 +4667,8 @@ var KoraSyncServer = class {
|
|
|
3069
4667
|
max: this.maxConnections
|
|
3070
4668
|
});
|
|
3071
4669
|
}
|
|
3072
|
-
|
|
4670
|
+
this.ensureBackgroundTimersStarted();
|
|
4671
|
+
const sessionId = generateUUIDv77();
|
|
3073
4672
|
this.metrics.recordConnection(sessionId);
|
|
3074
4673
|
const sessionEmitter = new SimpleEventEmitter();
|
|
3075
4674
|
sessionEmitter.on("sync:connected", (event) => {
|
|
@@ -3115,6 +4714,33 @@ var KoraSyncServer = class {
|
|
|
3115
4714
|
sessionId
|
|
3116
4715
|
});
|
|
3117
4716
|
});
|
|
4717
|
+
sessionEmitter.on("sync:schema-mismatch", (event) => {
|
|
4718
|
+
this.logger.log({
|
|
4719
|
+
timestamp: Date.now(),
|
|
4720
|
+
level: "warn",
|
|
4721
|
+
event: "session.schema_mismatch",
|
|
4722
|
+
sessionId,
|
|
4723
|
+
details: {
|
|
4724
|
+
clientSchemaVersion: event.clientSchemaVersion,
|
|
4725
|
+
serverSchemaVersion: event.serverSchemaVersion,
|
|
4726
|
+
supportedMin: event.supportedMin,
|
|
4727
|
+
supportedMax: event.supportedMax,
|
|
4728
|
+
reason: event.reason
|
|
4729
|
+
}
|
|
4730
|
+
});
|
|
4731
|
+
});
|
|
4732
|
+
sessionEmitter.on("sync:delivery-stalled", (event) => {
|
|
4733
|
+
this.logger.log({
|
|
4734
|
+
timestamp: Date.now(),
|
|
4735
|
+
level: "warn",
|
|
4736
|
+
event: "session.delivery_stalled",
|
|
4737
|
+
sessionId,
|
|
4738
|
+
details: {
|
|
4739
|
+
watermark: event.watermark,
|
|
4740
|
+
repeatCount: event.repeatCount
|
|
4741
|
+
}
|
|
4742
|
+
});
|
|
4743
|
+
});
|
|
3118
4744
|
const session = new ClientSession({
|
|
3119
4745
|
sessionId,
|
|
3120
4746
|
transport,
|
|
@@ -3122,9 +4748,11 @@ var KoraSyncServer = class {
|
|
|
3122
4748
|
auth: this.auth ?? void 0,
|
|
3123
4749
|
serializer: this.serializer,
|
|
3124
4750
|
emitter: sessionEmitter,
|
|
4751
|
+
logger: this.logger,
|
|
3125
4752
|
batchSize: this.batchSize,
|
|
3126
4753
|
schemaVersion: this.schemaVersion,
|
|
3127
4754
|
supportedSchemaVersions: this.supportedSchemaVersions,
|
|
4755
|
+
operationTransforms: this.operationTransforms,
|
|
3128
4756
|
onRelay: (sourceSessionId, operations2) => {
|
|
3129
4757
|
this.handleRelay(sourceSessionId, operations2);
|
|
3130
4758
|
},
|
|
@@ -3134,12 +4762,29 @@ var KoraSyncServer = class {
|
|
|
3134
4762
|
onYjsDocUpdate: (sourceSessionId, message) => {
|
|
3135
4763
|
this.handleYjsDocRelay(sourceSessionId, message);
|
|
3136
4764
|
},
|
|
4765
|
+
onBlobChunkRequest: (sourceSessionId, message) => {
|
|
4766
|
+
this.blobChunkRelay.handleRequest(sourceSessionId, message);
|
|
4767
|
+
},
|
|
4768
|
+
onBlobChunkResponse: (sourceSessionId, message) => {
|
|
4769
|
+
this.blobChunkRelay.handleResponse(sourceSessionId, message);
|
|
4770
|
+
},
|
|
4771
|
+
...this.persistBlobChunk ? { persistBlobChunk: this.persistBlobChunk } : {},
|
|
4772
|
+
// Only forward when configured, so an unset server value leaves the
|
|
4773
|
+
// session on its own documented default rather than `undefined`.
|
|
4774
|
+
...this.maxOperationBytes !== void 0 ? { maxOperationBytes: this.maxOperationBytes } : {},
|
|
4775
|
+
...this.maxOpsPerMinute !== void 0 ? { maxOpsPerMinute: this.maxOpsPerMinute } : {},
|
|
4776
|
+
...this.validateOperation ? { validateOperation: this.validateOperation, koraContext: this.koraContext } : {},
|
|
3137
4777
|
onClose: (sid) => {
|
|
3138
4778
|
this.handleSessionClose(sid);
|
|
3139
|
-
}
|
|
4779
|
+
},
|
|
4780
|
+
onOrphanedRelays: (nodeId, ops) => {
|
|
4781
|
+
this.bufferOrphanedRelays(nodeId, ops);
|
|
4782
|
+
},
|
|
4783
|
+
takeOrphanedRelays: (nodeId) => this.takeOrphanedRelays(nodeId)
|
|
3140
4784
|
});
|
|
3141
4785
|
this.sessions.set(sessionId, session);
|
|
3142
4786
|
this.yjsDocRelay.addClient(sessionId, transport);
|
|
4787
|
+
this.blobChunkRelay.addClient(sessionId, transport);
|
|
3143
4788
|
session.start();
|
|
3144
4789
|
this.logger.log({
|
|
3145
4790
|
timestamp: Date.now(),
|
|
@@ -3172,6 +4817,77 @@ var KoraSyncServer = class {
|
|
|
3172
4817
|
errorCount: snapshot.errorCount
|
|
3173
4818
|
};
|
|
3174
4819
|
}
|
|
4820
|
+
/**
|
|
4821
|
+
* Apply a server-originated operation (for example one created by a custom
|
|
4822
|
+
* HTTP route) through the same validated pipeline that incoming client
|
|
4823
|
+
* operations use — Tier 2 constraints, referential integrity, and cascade
|
|
4824
|
+
* side effects — then relay every applied operation to connected clients.
|
|
4825
|
+
*
|
|
4826
|
+
* Because the operation did not come from a client session, it is relayed to
|
|
4827
|
+
* ALL sessions (there is no source session to exclude). Each session still
|
|
4828
|
+
* applies its own per-scope visibility filter in `relayOperations`, so a
|
|
4829
|
+
* client only receives the operation if it falls within that client's scope.
|
|
4830
|
+
*
|
|
4831
|
+
* @param op - A fully-formed, server-originated operation to apply
|
|
4832
|
+
* @returns The apply result, including any server-generated side-effect ops
|
|
4833
|
+
*/
|
|
4834
|
+
async applyLocalOperation(op) {
|
|
4835
|
+
const result = await applyServerOperation(this.store, op);
|
|
4836
|
+
if (result.result === "applied" && result.appliedOperations.length > 0) {
|
|
4837
|
+
for (const session of this.sessions.values()) {
|
|
4838
|
+
session.relayOperations(result.appliedOperations);
|
|
4839
|
+
}
|
|
4840
|
+
}
|
|
4841
|
+
return result;
|
|
4842
|
+
}
|
|
4843
|
+
/**
|
|
4844
|
+
* Relay already-applied, server-originated operations to every connected client.
|
|
4845
|
+
* Used by the conditional route apply, which commits its operations atomically
|
|
4846
|
+
* through the store (bypassing {@link applyLocalOperation}) and then fans them
|
|
4847
|
+
* out. Each session still enforces its own per-scope visibility filter.
|
|
4848
|
+
*
|
|
4849
|
+
* @param operations - Operations that have already been committed to the store
|
|
4850
|
+
*/
|
|
4851
|
+
relayServerOperations(operations2) {
|
|
4852
|
+
if (operations2.length === 0) {
|
|
4853
|
+
return;
|
|
4854
|
+
}
|
|
4855
|
+
for (const session of this.sessions.values()) {
|
|
4856
|
+
session.relayOperations(operations2);
|
|
4857
|
+
}
|
|
4858
|
+
}
|
|
4859
|
+
/**
|
|
4860
|
+
* Collect every blob reference still reachable from live records on the server.
|
|
4861
|
+
*
|
|
4862
|
+
* This is the live set for garbage-collecting the server's central blob store:
|
|
4863
|
+
* pass the result to `collectBlobGarbage(blobStore, liveRefs)` from
|
|
4864
|
+
* `@korajs/store` to reclaim bytes no record references any more. Only
|
|
4865
|
+
* collections that declare a `blob` field are scanned.
|
|
4866
|
+
*
|
|
4867
|
+
* @returns Every live blob reference across all collections
|
|
4868
|
+
*/
|
|
4869
|
+
async getLiveBlobRefs() {
|
|
4870
|
+
const schema = this.store.getSchema();
|
|
4871
|
+
if (!schema) {
|
|
4872
|
+
return [];
|
|
4873
|
+
}
|
|
4874
|
+
const refs = [];
|
|
4875
|
+
for (const [name, collection] of Object.entries(schema.collections)) {
|
|
4876
|
+
const hasBlobField = Object.values(collection.fields).some((field) => field.kind === "blob");
|
|
4877
|
+
if (!hasBlobField) {
|
|
4878
|
+
continue;
|
|
4879
|
+
}
|
|
4880
|
+
const records = await this.store.materializeCollection(name);
|
|
4881
|
+
for (const record of records) {
|
|
4882
|
+
for (const value of Object.values(record)) {
|
|
4883
|
+
if (isBlobRef(value)) {
|
|
4884
|
+
refs.push(value);
|
|
4885
|
+
}
|
|
4886
|
+
}
|
|
4887
|
+
}
|
|
4888
|
+
}
|
|
4889
|
+
return refs;
|
|
4890
|
+
}
|
|
3175
4891
|
/**
|
|
3176
4892
|
* Get the number of currently connected clients.
|
|
3177
4893
|
*/
|
|
@@ -3205,6 +4921,7 @@ var KoraSyncServer = class {
|
|
|
3205
4921
|
this.metrics.recordDisconnection(sessionId);
|
|
3206
4922
|
this.awarenessRelay.removeClient(sessionId);
|
|
3207
4923
|
this.yjsDocRelay.removeClient(sessionId);
|
|
4924
|
+
this.blobChunkRelay.removeClient(sessionId);
|
|
3208
4925
|
this.sessions.delete(sessionId);
|
|
3209
4926
|
const clientId = this.httpSessionToClient.get(sessionId);
|
|
3210
4927
|
if (clientId) {
|
|
@@ -3257,13 +4974,6 @@ function normalizeHttpBody(body, contentType) {
|
|
|
3257
4974
|
return body;
|
|
3258
4975
|
}
|
|
3259
4976
|
|
|
3260
|
-
// src/auth/no-auth.ts
|
|
3261
|
-
var NoAuthProvider = class {
|
|
3262
|
-
async authenticate(_token) {
|
|
3263
|
-
return { userId: "anonymous" };
|
|
3264
|
-
}
|
|
3265
|
-
};
|
|
3266
|
-
|
|
3267
4977
|
// src/auth/token-auth.ts
|
|
3268
4978
|
var TokenAuthProvider = class {
|
|
3269
4979
|
validate;
|
|
@@ -3363,11 +5073,13 @@ function createProductionServer(config) {
|
|
|
3363
5073
|
const port = config.port ?? (Number(process.env.PORT) || 3001);
|
|
3364
5074
|
const staticDir = config.staticDir ?? "./dist";
|
|
3365
5075
|
const syncPath = config.syncPath ?? "/kora-sync";
|
|
5076
|
+
const crossOriginEmbedderPolicy = config.crossOriginEmbedderPolicy ?? "credentialless";
|
|
3366
5077
|
const syncServer = new KoraSyncServer({
|
|
3367
5078
|
store: config.store,
|
|
3368
5079
|
enableDashboard: true,
|
|
3369
5080
|
...config.syncOptions
|
|
3370
5081
|
});
|
|
5082
|
+
const routeContext = syncServer.getKoraContext();
|
|
3371
5083
|
let httpServer = null;
|
|
3372
5084
|
function getOperationalToken(kind) {
|
|
3373
5085
|
const auth = config.operationalAuth;
|
|
@@ -3454,6 +5166,10 @@ function createProductionServer(config) {
|
|
|
3454
5166
|
const chunks = [];
|
|
3455
5167
|
req.on("data", (chunk) => chunks.push(chunk));
|
|
3456
5168
|
req.on("end", () => resolve(Buffer.concat(chunks)));
|
|
5169
|
+
if (!req.readableFlowing) {
|
|
5170
|
+
req.resume();
|
|
5171
|
+
}
|
|
5172
|
+
req.on("error", () => resolve(Buffer.concat(chunks)));
|
|
3457
5173
|
});
|
|
3458
5174
|
}
|
|
3459
5175
|
async function readJsonBody(req) {
|
|
@@ -3503,6 +5219,10 @@ function createProductionServer(config) {
|
|
|
3503
5219
|
res.end(JSON.stringify(result.body ?? null));
|
|
3504
5220
|
}
|
|
3505
5221
|
return {
|
|
5222
|
+
kora: routeContext,
|
|
5223
|
+
getLiveBlobRefs() {
|
|
5224
|
+
return syncServer.getLiveBlobRefs();
|
|
5225
|
+
},
|
|
3506
5226
|
async start() {
|
|
3507
5227
|
const { createServer } = await import("http");
|
|
3508
5228
|
const { createReadStream, existsSync, statSync } = await import("fs");
|
|
@@ -3510,8 +5230,21 @@ function createProductionServer(config) {
|
|
|
3510
5230
|
const { WebSocketServer } = await import("ws");
|
|
3511
5231
|
const distDir = resolve(staticDir);
|
|
3512
5232
|
httpServer = createServer(async (req, res) => {
|
|
5233
|
+
try {
|
|
5234
|
+
await handleRequest(req, res);
|
|
5235
|
+
} catch (error) {
|
|
5236
|
+
console.error("[kora] Unhandled error in HTTP request handler:", error);
|
|
5237
|
+
if (!res.headersSent) {
|
|
5238
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
5239
|
+
res.end(JSON.stringify({ error: "Internal server error" }));
|
|
5240
|
+
} else if (!res.writableEnded) {
|
|
5241
|
+
res.end();
|
|
5242
|
+
}
|
|
5243
|
+
}
|
|
5244
|
+
});
|
|
5245
|
+
async function handleRequest(req, res) {
|
|
3513
5246
|
res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
|
|
3514
|
-
res.setHeader("Cross-Origin-Embedder-Policy",
|
|
5247
|
+
res.setHeader("Cross-Origin-Embedder-Policy", crossOriginEmbedderPolicy);
|
|
3515
5248
|
const url = new URL(req.url || "/", `http://${req.headers.host}`);
|
|
3516
5249
|
if (url.pathname === "/health") {
|
|
3517
5250
|
const status = await syncServer.getStatus();
|
|
@@ -3637,7 +5370,8 @@ data: ${JSON.stringify(s)}
|
|
|
3637
5370
|
body: await readJsonBody(req),
|
|
3638
5371
|
headers: req.headers,
|
|
3639
5372
|
query: getQuery(url),
|
|
3640
|
-
ip: getClientIp(req)
|
|
5373
|
+
ip: getClientIp(req),
|
|
5374
|
+
kora: routeContext
|
|
3641
5375
|
});
|
|
3642
5376
|
writeJsonResponse(res, result);
|
|
3643
5377
|
return;
|
|
@@ -3671,7 +5405,7 @@ data: ${JSON.stringify(s)}
|
|
|
3671
5405
|
const contentType = MIME_TYPES[ext] || "application/octet-stream";
|
|
3672
5406
|
res.writeHead(200, { "Content-Type": contentType });
|
|
3673
5407
|
createReadStream(filePath).pipe(res);
|
|
3674
|
-
}
|
|
5408
|
+
}
|
|
3675
5409
|
const wss = new WebSocketServer({ noServer: true });
|
|
3676
5410
|
httpServer.on("upgrade", (req, socket, head) => {
|
|
3677
5411
|
const url = new URL(req.url || "/", `http://${req.headers.host}`);
|
|
@@ -3684,8 +5418,13 @@ data: ${JSON.stringify(s)}
|
|
|
3684
5418
|
socket.destroy();
|
|
3685
5419
|
}
|
|
3686
5420
|
});
|
|
3687
|
-
return new Promise((resolve2) => {
|
|
5421
|
+
return new Promise((resolve2, reject) => {
|
|
5422
|
+
const onError = (error) => {
|
|
5423
|
+
reject(error);
|
|
5424
|
+
};
|
|
5425
|
+
httpServer?.once("error", onError);
|
|
3688
5426
|
httpServer?.listen(port, "0.0.0.0", () => {
|
|
5427
|
+
httpServer?.off("error", onError);
|
|
3689
5428
|
resolve2(`http://localhost:${port}`);
|
|
3690
5429
|
});
|
|
3691
5430
|
});
|
|
@@ -3787,6 +5526,7 @@ export {
|
|
|
3787
5526
|
MixedAuthProvider,
|
|
3788
5527
|
NoAuthProvider,
|
|
3789
5528
|
PostgresServerStore,
|
|
5529
|
+
RETRIABLE_REJECTION_CODES,
|
|
3790
5530
|
SqliteServerStore,
|
|
3791
5531
|
TokenAuthProvider,
|
|
3792
5532
|
WsServerTransport,
|
|
@@ -3797,6 +5537,7 @@ export {
|
|
|
3797
5537
|
createPrettyLogger,
|
|
3798
5538
|
createProductionServer,
|
|
3799
5539
|
createSilentLogger,
|
|
3800
|
-
createSqliteServerStore
|
|
5540
|
+
createSqliteServerStore,
|
|
5541
|
+
isRetriableRejection
|
|
3801
5542
|
};
|
|
3802
5543
|
//# sourceMappingURL=index.js.map
|