@syncular/server 0.26.4 → 0.26.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-encoding.d.ts +17 -0
- package/dist/content-encoding.js +22 -0
- package/dist/context.d.ts +9 -0
- package/dist/d1-storage.d.ts +46 -0
- package/dist/d1-storage.js +12 -1
- package/dist/errors.d.ts +19 -0
- package/dist/errors.js +29 -0
- package/dist/handler.js +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/operations-realtime.js +4 -1
- package/dist/operations.js +6 -1
- package/dist/postgres-storage.d.ts +42 -0
- package/dist/postgres-storage.js +110 -45
- package/dist/pull.d.ts +46 -6
- package/dist/pull.js +129 -34
- package/dist/push.js +19 -2
- package/dist/realtime.d.ts +3 -0
- package/dist/realtime.js +32 -26
- package/dist/relational-rows.d.ts +19 -0
- package/dist/relational-rows.js +38 -0
- package/dist/s3-segment-store.d.ts +3 -1
- package/dist/s3-segment-store.js +16 -0
- package/dist/segment-download.d.ts +16 -0
- package/dist/segment-download.js +71 -21
- package/dist/segment-store.d.ts +16 -0
- package/dist/segment-store.js +19 -0
- package/dist/sqlite-storage.d.ts +5 -0
- package/dist/sqlite-storage.js +18 -1
- package/dist/storage.d.ts +20 -0
- package/package.json +2 -2
- package/src/content-encoding.ts +35 -0
- package/src/context.ts +9 -0
- package/src/d1-storage.ts +20 -1
- package/src/errors.ts +56 -0
- package/src/handler.ts +2 -0
- package/src/index.ts +1 -0
- package/src/operations-realtime.ts +4 -1
- package/src/operations.ts +6 -1
- package/src/postgres-storage.ts +145 -60
- package/src/pull.ts +212 -39
- package/src/push.ts +18 -2
- package/src/realtime.ts +45 -32
- package/src/relational-rows.ts +43 -0
- package/src/s3-segment-store.ts +15 -0
- package/src/segment-download.ts +129 -21
- package/src/segment-store.ts +36 -0
- package/src/sqlite-storage.ts +27 -0
- package/src/storage.ts +21 -0
package/src/postgres-storage.ts
CHANGED
|
@@ -40,7 +40,7 @@ import { serveNotReadyError } from './readiness';
|
|
|
40
40
|
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
41
41
|
* does not tolerate). Cross-partition pushes never contend.
|
|
42
42
|
*/
|
|
43
|
-
import type { PushOperationResult } from '@syncular/core';
|
|
43
|
+
import type { PushOperationResult, ScopeMap } from '@syncular/core';
|
|
44
44
|
import {
|
|
45
45
|
bindAuthoritativePartition,
|
|
46
46
|
type BoundAuthoritativeQuery,
|
|
@@ -59,6 +59,7 @@ import {
|
|
|
59
59
|
assertPhysicalColumns,
|
|
60
60
|
assertStoredLayouts,
|
|
61
61
|
COMMIT_WINDOW_BATCH_SQL,
|
|
62
|
+
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
62
63
|
deleteRowSql,
|
|
63
64
|
dropTableDdl,
|
|
64
65
|
indexRowPageStatement,
|
|
@@ -76,6 +77,7 @@ import {
|
|
|
76
77
|
schemaDdl,
|
|
77
78
|
selectRowScopesSql,
|
|
78
79
|
selectRowSql,
|
|
80
|
+
prefetchRowsSql,
|
|
79
81
|
selectRowsForRewriteSql,
|
|
80
82
|
upsertSql,
|
|
81
83
|
upsertValues,
|
|
@@ -697,21 +699,31 @@ async function writeRowOn(
|
|
|
697
699
|
partition: string,
|
|
698
700
|
row: StoredRow,
|
|
699
701
|
): Promise<void> {
|
|
702
|
+
// One statement per row write: the upsert, and the scope index synced to
|
|
703
|
+
// the row's scope map (entries outside it deleted, missing ones inserted).
|
|
704
|
+
// No index entry is touched twice, so the writable CTEs cannot conflict.
|
|
705
|
+
// The scopes bind as text before the JSONB cast (see `appendCommit`).
|
|
706
|
+
const params = upsertValues(compiled, partition, row, 'postgres');
|
|
707
|
+
const n = params.length;
|
|
700
708
|
await q.query(
|
|
701
|
-
upsertSql(compiled, 'postgres'),
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
709
|
+
`WITH upserted AS (${upsertSql(compiled, 'postgres')}),
|
|
710
|
+
scope AS (SELECT key, value FROM jsonb_each_text($${n + 1}::text::jsonb)),
|
|
711
|
+
removed AS (
|
|
712
|
+
DELETE FROM sync_row_scopes
|
|
713
|
+
WHERE partition=$${n + 2} AND tbl=$${n + 3} AND row_id=$${n + 4}
|
|
714
|
+
AND (var, value) NOT IN (SELECT key, value FROM scope)
|
|
715
|
+
)
|
|
716
|
+
INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
717
|
+
SELECT $${n + 2}, $${n + 3}, key, value, $${n + 4} FROM scope
|
|
718
|
+
ON CONFLICT DO NOTHING`,
|
|
719
|
+
[
|
|
720
|
+
...params,
|
|
721
|
+
JSON.stringify(row.scopes),
|
|
722
|
+
partition,
|
|
723
|
+
compiled.name,
|
|
724
|
+
row.rowId,
|
|
725
|
+
],
|
|
707
726
|
);
|
|
708
|
-
for (const [variable, value] of Object.entries(row.scopes)) {
|
|
709
|
-
await q.query(
|
|
710
|
-
`INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
711
|
-
VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`,
|
|
712
|
-
[partition, compiled.name, variable, value, row.rowId],
|
|
713
|
-
);
|
|
714
|
-
}
|
|
715
727
|
}
|
|
716
728
|
|
|
717
729
|
async function getPushResultOn(
|
|
@@ -770,6 +782,13 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
770
782
|
#resolveTable: (name: string) => CompiledTable;
|
|
771
783
|
#open = true;
|
|
772
784
|
#pushApplySavepoint = false;
|
|
785
|
+
/**
|
|
786
|
+
* `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
|
|
787
|
+
* (or tombstone) read as absent. A write in this transaction drops the
|
|
788
|
+
* key, so the next read goes to the database again.
|
|
789
|
+
*/
|
|
790
|
+
readonly #prefetchedRows = new Map<string, StoredRow | null>();
|
|
791
|
+
readonly #prefetchedTombstones = new Map<string, number | null>();
|
|
773
792
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
774
793
|
#resolve: () => void;
|
|
775
794
|
#reject: (error: unknown) => void;
|
|
@@ -814,6 +833,10 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
814
833
|
|
|
815
834
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
|
|
816
835
|
this.#assertOpen();
|
|
836
|
+
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
837
|
+
if (prefetched !== undefined) {
|
|
838
|
+
return Promise.resolve(prefetched ?? undefined);
|
|
839
|
+
}
|
|
817
840
|
return getRowOn(
|
|
818
841
|
this.#client,
|
|
819
842
|
this.#resolveTable(table),
|
|
@@ -822,11 +845,47 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
822
845
|
);
|
|
823
846
|
}
|
|
824
847
|
|
|
848
|
+
async prefetchRows(table: string, rowIds: readonly string[]): Promise<void> {
|
|
849
|
+
this.#assertOpen();
|
|
850
|
+
const ids = [...new Set(rowIds)].filter(
|
|
851
|
+
(rowId) => !this.#prefetchedRows.has(`${table}\u0000${rowId}`),
|
|
852
|
+
);
|
|
853
|
+
if (ids.length === 0) return;
|
|
854
|
+
const { rows } = await this.#client.query<
|
|
855
|
+
Omit<RowRecord, 'row_id'> & {
|
|
856
|
+
lookup_id: string;
|
|
857
|
+
row_id: string | null;
|
|
858
|
+
tombstone_seq: unknown;
|
|
859
|
+
}
|
|
860
|
+
>(prefetchRowsSql(this.#resolveTable(table)), [
|
|
861
|
+
this.#partition,
|
|
862
|
+
JSON.stringify(ids),
|
|
863
|
+
table,
|
|
864
|
+
]);
|
|
865
|
+
for (const record of rows) {
|
|
866
|
+
const key = `${table}\u0000${record.lookup_id}`;
|
|
867
|
+
this.#prefetchedRows.set(
|
|
868
|
+
key,
|
|
869
|
+
record.row_id === null
|
|
870
|
+
? null
|
|
871
|
+
: toStoredRow({ ...record, row_id: record.row_id }),
|
|
872
|
+
);
|
|
873
|
+
this.#prefetchedTombstones.set(
|
|
874
|
+
key,
|
|
875
|
+
record.tombstone_seq === null || record.tombstone_seq === undefined
|
|
876
|
+
? null
|
|
877
|
+
: asNumber(record.tombstone_seq),
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
825
882
|
async getTombstoneSeq(
|
|
826
883
|
table: string,
|
|
827
884
|
rowId: string,
|
|
828
885
|
): Promise<number | undefined> {
|
|
829
886
|
this.#assertOpen();
|
|
887
|
+
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
888
|
+
if (prefetched !== undefined) return prefetched ?? undefined;
|
|
830
889
|
const { rows } = await this.#client.query<{ commit_seq: unknown }>(
|
|
831
890
|
'SELECT commit_seq FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
832
891
|
[this.#partition, table, rowId],
|
|
@@ -837,6 +896,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
837
896
|
|
|
838
897
|
async clearTombstone(table: string, rowId: string): Promise<void> {
|
|
839
898
|
this.#assertOpen();
|
|
899
|
+
this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
|
|
840
900
|
await this.#client.query(
|
|
841
901
|
'DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
842
902
|
[this.#partition, table, rowId],
|
|
@@ -976,6 +1036,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
976
1036
|
context?: { readonly opIndex: number },
|
|
977
1037
|
): Promise<void> {
|
|
978
1038
|
this.#assertOpen();
|
|
1039
|
+
this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
|
|
979
1040
|
try {
|
|
980
1041
|
await writeRowOn(
|
|
981
1042
|
this.#client,
|
|
@@ -993,18 +1054,16 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
993
1054
|
|
|
994
1055
|
async deleteRow(table: string, rowId: string): Promise<void> {
|
|
995
1056
|
this.#assertOpen();
|
|
1057
|
+
this.#prefetchedRows.delete(`${table}\u0000${rowId}`);
|
|
1058
|
+
// One statement: the row, its scope-index entries, and (§5.9.4: a
|
|
1059
|
+
// deleted row references no blobs) its blob references.
|
|
996
1060
|
await this.#client.query(
|
|
997
|
-
deleteRowSql(this.#resolveTable(table), 'postgres'),
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
[this.#partition,
|
|
1003
|
-
);
|
|
1004
|
-
// §5.9.4: a deleted row references no blobs.
|
|
1005
|
-
await this.#client.query(
|
|
1006
|
-
'DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
1007
|
-
[this.#partition, table, rowId],
|
|
1061
|
+
`WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
|
|
1062
|
+
unscoped AS (
|
|
1063
|
+
DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
|
|
1064
|
+
)
|
|
1065
|
+
DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`,
|
|
1066
|
+
[this.#partition, rowId, table],
|
|
1008
1067
|
);
|
|
1009
1068
|
}
|
|
1010
1069
|
|
|
@@ -1030,6 +1089,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1030
1089
|
|
|
1031
1090
|
async appendCommit(commit: NewCommit): Promise<number> {
|
|
1032
1091
|
this.#assertOpen();
|
|
1092
|
+
// The commit writes its delete tombstones below.
|
|
1093
|
+
this.#prefetchedTombstones.clear();
|
|
1033
1094
|
const q = this.#client;
|
|
1034
1095
|
const p = this.#partition;
|
|
1035
1096
|
// Allocate the next dense commitSeq under a per-partition row lock: the
|
|
@@ -1055,42 +1116,46 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1055
1116
|
],
|
|
1056
1117
|
);
|
|
1057
1118
|
const commitSeq = asNumber(rows[0]?.commit_seq);
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
)
|
|
1077
|
-
INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1078
|
-
SELECT partition, tbl, row_id, commit_seq FROM inserted WHERE op = 2
|
|
1079
|
-
ON CONFLICT (partition, tbl, row_id)
|
|
1080
|
-
DO UPDATE SET commit_seq = excluded.commit_seq`,
|
|
1081
|
-
[
|
|
1082
|
-
p,
|
|
1083
|
-
commitSeq,
|
|
1084
|
-
idx,
|
|
1085
|
-
change.table,
|
|
1086
|
-
change.rowId,
|
|
1087
|
-
change.op === 'upsert' ? 1 : 2,
|
|
1088
|
-
change.rowVersion ?? null,
|
|
1089
|
-
JSON.stringify(change.scopes),
|
|
1090
|
-
change.payload ?? null,
|
|
1091
|
-
],
|
|
1119
|
+
if (commit.changes.length === 0) return commitSeq;
|
|
1120
|
+
// One statement for the whole commit: its changes, their inverted scope
|
|
1121
|
+
// entries, and the §5 delete tombstones ride one writable-CTE chain.
|
|
1122
|
+
// Serialized scopes bind as text before parsing JSONB; drivers that
|
|
1123
|
+
// encode JSONB parameters would otherwise store the string as a scalar.
|
|
1124
|
+
// A commit can delete one row twice, so the tombstone insert keeps one
|
|
1125
|
+
// row per key (every candidate carries the same commit_seq).
|
|
1126
|
+
const params: unknown[] = [p, commitSeq];
|
|
1127
|
+
const values = commit.changes.map((change, idx) => {
|
|
1128
|
+
const base = params.length;
|
|
1129
|
+
params.push(
|
|
1130
|
+
idx,
|
|
1131
|
+
change.table,
|
|
1132
|
+
change.rowId,
|
|
1133
|
+
change.op === 'upsert' ? 1 : 2,
|
|
1134
|
+
change.rowVersion ?? null,
|
|
1135
|
+
JSON.stringify(change.scopes),
|
|
1136
|
+
change.payload ?? null,
|
|
1092
1137
|
);
|
|
1093
|
-
|
|
1138
|
+
return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
|
|
1139
|
+
});
|
|
1140
|
+
await q.query(
|
|
1141
|
+
`WITH inserted AS (
|
|
1142
|
+
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1143
|
+
VALUES ${values.join(',')}
|
|
1144
|
+
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1145
|
+
),
|
|
1146
|
+
scoped AS (
|
|
1147
|
+
INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
1148
|
+
SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
|
|
1149
|
+
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
1150
|
+
ON CONFLICT DO NOTHING
|
|
1151
|
+
)
|
|
1152
|
+
INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1153
|
+
SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
|
|
1154
|
+
FROM inserted WHERE op = 2
|
|
1155
|
+
ON CONFLICT (partition, tbl, row_id)
|
|
1156
|
+
DO UPDATE SET commit_seq = excluded.commit_seq`,
|
|
1157
|
+
params,
|
|
1158
|
+
);
|
|
1094
1159
|
return commitSeq;
|
|
1095
1160
|
}
|
|
1096
1161
|
|
|
@@ -2142,6 +2207,26 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
2142
2207
|
};
|
|
2143
2208
|
}
|
|
2144
2209
|
|
|
2210
|
+
async latestScopedChangeSeq(
|
|
2211
|
+
partition: string,
|
|
2212
|
+
query: { readonly table: string; readonly scopeFilter: ScopeMap },
|
|
2213
|
+
): Promise<number> {
|
|
2214
|
+
const firstVariable = Object.keys(query.scopeFilter).sort()[0];
|
|
2215
|
+
const values =
|
|
2216
|
+
firstVariable === undefined
|
|
2217
|
+
? []
|
|
2218
|
+
: (query.scopeFilter[firstVariable] ?? []);
|
|
2219
|
+
if (firstVariable === undefined || values.length === 0) return 0;
|
|
2220
|
+
// Concurrent reads of one table in a pull share one statement.
|
|
2221
|
+
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
2222
|
+
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
2223
|
+
partition,
|
|
2224
|
+
query.table,
|
|
2225
|
+
{ var: firstVariable, vals: values },
|
|
2226
|
+
);
|
|
2227
|
+
return row === undefined ? 0 : asNumber(row.seq);
|
|
2228
|
+
}
|
|
2229
|
+
|
|
2145
2230
|
async readCommitWindow(
|
|
2146
2231
|
partition: string,
|
|
2147
2232
|
query: CommitWindowQuery,
|
package/src/pull.ts
CHANGED
|
@@ -9,13 +9,18 @@ import {
|
|
|
9
9
|
type PullHeaderFrame,
|
|
10
10
|
type ResponseFrame,
|
|
11
11
|
type ScopeMap,
|
|
12
|
+
type SegmentRefFrame,
|
|
12
13
|
type SegmentRow,
|
|
13
14
|
type SubscriptionFrame,
|
|
14
15
|
} from '@syncular/core';
|
|
15
|
-
import type { SyncRequestContext } from './context';
|
|
16
|
+
import type { SyncRequestContext, SyncServerConfig } from './context';
|
|
16
17
|
import { clockOf, limitsOf } from './context';
|
|
17
18
|
import type { PullSegmentSummary } from './events';
|
|
18
|
-
import
|
|
19
|
+
import {
|
|
20
|
+
type CompiledSchema,
|
|
21
|
+
type CompiledTable,
|
|
22
|
+
compileSchema,
|
|
23
|
+
} from './schema';
|
|
19
24
|
import { scopeDigest } from './scopes';
|
|
20
25
|
import type { SegmentRecord, SegmentStore } from './segment-store';
|
|
21
26
|
import { issueSegmentUrl } from './signed-url';
|
|
@@ -244,21 +249,11 @@ async function* sqliteImageSegment(
|
|
|
244
249
|
digest: string,
|
|
245
250
|
trace: PullSectionTrace | undefined,
|
|
246
251
|
logEpoch: string,
|
|
252
|
+
existing: SegmentRecord | undefined,
|
|
253
|
+
probe: StoredRow[],
|
|
247
254
|
): AsyncGenerator<ResponseFrame, boolean> {
|
|
248
255
|
const { storage, segments, partition } = ctx;
|
|
249
256
|
const now = clockOf(ctx)();
|
|
250
|
-
const existing = await segments.find(
|
|
251
|
-
{
|
|
252
|
-
partition,
|
|
253
|
-
logEpoch,
|
|
254
|
-
table: plan.table.name,
|
|
255
|
-
schemaVersion: schema.version,
|
|
256
|
-
mediaType: 'sqlite',
|
|
257
|
-
scopeDigest: digest,
|
|
258
|
-
asOfCommitSeq: asOf,
|
|
259
|
-
},
|
|
260
|
-
now,
|
|
261
|
-
);
|
|
262
257
|
if (existing !== undefined) {
|
|
263
258
|
trace?.segments.push({
|
|
264
259
|
mediaType: 'sqlite',
|
|
@@ -276,12 +271,7 @@ async function* sqliteImageSegment(
|
|
|
276
271
|
}
|
|
277
272
|
// Eligibility probe (§5.3): image only when the snapshot exceeds one
|
|
278
273
|
// rows page — smaller tables stay on the (typically inline) rows lane.
|
|
279
|
-
|
|
280
|
-
table: plan.table.name,
|
|
281
|
-
scopeFilter: plan.effective,
|
|
282
|
-
afterRowId: null,
|
|
283
|
-
limit: limits.limitSnapshotRows + 1,
|
|
284
|
-
});
|
|
274
|
+
// The probe is the rows lane's first page, read by `prepareSections`.
|
|
285
275
|
if (probe.length <= limits.limitSnapshotRows) return false;
|
|
286
276
|
// §5.3: building an image needs a SQLite engine. The host injects the
|
|
287
277
|
// builder through `sqliteImageBuilder`; when omitted we default to the
|
|
@@ -389,7 +379,8 @@ async function* bootstrapSegments(
|
|
|
389
379
|
startRowCursor: string | null,
|
|
390
380
|
trace: PullSectionTrace | undefined,
|
|
391
381
|
logEpoch: string,
|
|
392
|
-
firstPage: StoredRow[]
|
|
382
|
+
firstPage: StoredRow[],
|
|
383
|
+
image: SegmentRecord | undefined,
|
|
393
384
|
): AsyncGenerator<
|
|
394
385
|
ResponseFrame,
|
|
395
386
|
{ complete: boolean; rowCursor: string | null }
|
|
@@ -408,13 +399,15 @@ async function* bootstrapSegments(
|
|
|
408
399
|
digest,
|
|
409
400
|
trace,
|
|
410
401
|
logEpoch,
|
|
402
|
+
image,
|
|
403
|
+
firstPage,
|
|
411
404
|
);
|
|
412
405
|
if (imaged) return { complete: true, rowCursor: null };
|
|
413
406
|
}
|
|
414
407
|
let rowCursor = startRowCursor;
|
|
415
408
|
for (let page = 0; page < limits.maxSnapshotPages; page++) {
|
|
416
409
|
const scanned =
|
|
417
|
-
page === 0
|
|
410
|
+
page === 0
|
|
418
411
|
? firstPage
|
|
419
412
|
: await storage.scanRows(partition, {
|
|
420
413
|
table: plan.table.name,
|
|
@@ -492,27 +485,45 @@ async function* bootstrapSegments(
|
|
|
492
485
|
/**
|
|
493
486
|
* The storage reads a section needs before its first frame. A pull starts
|
|
494
487
|
* them for every subscription before awaiting any, so a storage that batches
|
|
495
|
-
* concurrent page reads (PostgreSQL) answers all subscriptions of one
|
|
496
|
-
* with one statement, and the horizon re-check runs once per pull.
|
|
488
|
+
* concurrent page reads (PostgreSQL, D1) answers all subscriptions of one
|
|
489
|
+
* table with one statement, and the horizon re-check runs once per pull.
|
|
497
490
|
*/
|
|
498
491
|
export interface PreparedSection {
|
|
499
492
|
/** §4.5 window read for an incremental cursor; empty when none was read. */
|
|
500
493
|
readonly commits: StoredCommit[];
|
|
501
494
|
/** Horizon to judge the cursor against, re-read after any window read. */
|
|
502
495
|
readonly horizonSeq: number;
|
|
503
|
-
/**
|
|
496
|
+
/**
|
|
497
|
+
* A bootstrap's first snapshot page (`limitSnapshotRows + 1` rows from the
|
|
498
|
+
* resume cursor), which is also the §5.3 image eligibility probe.
|
|
499
|
+
*/
|
|
504
500
|
readonly firstPage: StoredRow[] | undefined;
|
|
501
|
+
/** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
|
|
502
|
+
readonly pin: number | undefined;
|
|
503
|
+
/** §5.3 stored image for a sqlite-lane bootstrap's reuse key, if any. */
|
|
504
|
+
readonly image: SegmentRecord | undefined;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
interface PendingSection {
|
|
508
|
+
readonly commits?: Promise<StoredCommit[]>;
|
|
509
|
+
readonly firstPage?: Promise<StoredRow[]>;
|
|
510
|
+
readonly lane?: Promise<{
|
|
511
|
+
readonly pin: number | undefined;
|
|
512
|
+
readonly image: SegmentRecord | undefined;
|
|
513
|
+
}>;
|
|
505
514
|
}
|
|
506
515
|
|
|
507
516
|
/** Run the pre-frame storage reads of every subscription concurrently. */
|
|
508
517
|
export async function prepareSections(
|
|
509
518
|
ctx: SyncRequestContext,
|
|
519
|
+
schema: CompiledSchema,
|
|
510
520
|
limits: PullLimits,
|
|
511
521
|
plans: readonly SubscriptionPlan[],
|
|
512
522
|
maxSeq: number,
|
|
513
523
|
horizonSeq: number,
|
|
524
|
+
logEpoch: string,
|
|
514
525
|
): Promise<PreparedSection[]> {
|
|
515
|
-
const reads = plans.map((plan) => {
|
|
526
|
+
const reads = plans.map((plan): PendingSection => {
|
|
516
527
|
const sub = plan.frame;
|
|
517
528
|
if (plan.status === 'revoked') return {};
|
|
518
529
|
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
@@ -537,24 +548,55 @@ export async function prepareSections(
|
|
|
537
548
|
return {};
|
|
538
549
|
}
|
|
539
550
|
// Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
|
|
540
|
-
const
|
|
551
|
+
const resume =
|
|
541
552
|
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
542
|
-
? token
|
|
543
|
-
:
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
+
? token
|
|
554
|
+
: undefined;
|
|
555
|
+
const startCursor = resume?.rowCursor ?? null;
|
|
556
|
+
const firstPage = ctx.storage.scanRows(ctx.partition, {
|
|
557
|
+
table: plan.table.name,
|
|
558
|
+
scopeFilter: plan.effective,
|
|
559
|
+
afterRowId: startCursor,
|
|
560
|
+
limit: limits.limitSnapshotRows + 1,
|
|
561
|
+
});
|
|
562
|
+
if (!sqliteLaneEligible(limits, plan, startCursor)) return { firstPage };
|
|
563
|
+
// §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
|
|
564
|
+
// scope (never below the horizon, never above maxCommitSeq), so a stored
|
|
565
|
+
// image stays the reuse key's match until this scope changes (§5.3).
|
|
566
|
+
const pinned =
|
|
567
|
+
resume !== undefined
|
|
568
|
+
? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
|
|
569
|
+
: ctx.storage
|
|
570
|
+
.latestScopedChangeSeq(ctx.partition, {
|
|
571
|
+
table: plan.table.name,
|
|
572
|
+
scopeFilter: plan.effective,
|
|
573
|
+
})
|
|
574
|
+
.then((latest) => {
|
|
575
|
+
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
576
|
+
return { asOf: pin, pin };
|
|
577
|
+
});
|
|
578
|
+
const lane = pinned.then(async ({ asOf, pin }) => ({
|
|
579
|
+
pin,
|
|
580
|
+
image: await ctx.segments.find(
|
|
581
|
+
{
|
|
582
|
+
partition: ctx.partition,
|
|
583
|
+
logEpoch,
|
|
584
|
+
table: plan.table.name,
|
|
585
|
+
schemaVersion: schema.version,
|
|
586
|
+
mediaType: 'sqlite',
|
|
587
|
+
scopeDigest: await scopeDigest(plan.effective),
|
|
588
|
+
asOfCommitSeq: asOf,
|
|
589
|
+
},
|
|
590
|
+
clockOf(ctx)(),
|
|
591
|
+
),
|
|
592
|
+
}));
|
|
593
|
+
return { firstPage, lane };
|
|
553
594
|
});
|
|
554
595
|
const settled = await Promise.all(
|
|
555
596
|
reads.map(async (read) => ({
|
|
556
597
|
commits: read.commits === undefined ? undefined : await read.commits,
|
|
557
598
|
firstPage: await read.firstPage,
|
|
599
|
+
lane: await read.lane,
|
|
558
600
|
})),
|
|
559
601
|
);
|
|
560
602
|
// Validate continuity before committing to an active section. A prune
|
|
@@ -566,6 +608,8 @@ export async function prepareSections(
|
|
|
566
608
|
commits: read.commits ?? [],
|
|
567
609
|
horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
|
|
568
610
|
firstPage: read.firstPage,
|
|
611
|
+
pin: read.lane?.pin,
|
|
612
|
+
image: read.lane?.image,
|
|
569
613
|
}));
|
|
570
614
|
}
|
|
571
615
|
|
|
@@ -583,12 +627,17 @@ export async function* subscriptionSection(
|
|
|
583
627
|
trace?: PullSectionTrace,
|
|
584
628
|
logEpoch?: string,
|
|
585
629
|
): AsyncGenerator<ResponseFrame, SubscriptionResult> {
|
|
630
|
+
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
631
|
+
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
632
|
+
}
|
|
586
633
|
const [prepared] = await prepareSections(
|
|
587
634
|
ctx,
|
|
635
|
+
schema,
|
|
588
636
|
limits,
|
|
589
637
|
[plan],
|
|
590
638
|
maxSeq,
|
|
591
639
|
horizonSeq,
|
|
640
|
+
logEpoch,
|
|
592
641
|
);
|
|
593
642
|
if (prepared === undefined) throw new Error('prepareSections lost a plan');
|
|
594
643
|
return yield* preparedSection(
|
|
@@ -660,8 +709,11 @@ export async function* preparedSection(
|
|
|
660
709
|
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
661
710
|
? token
|
|
662
711
|
: undefined;
|
|
663
|
-
const asOf = resume?.asOfCommitSeq ?? maxSeq;
|
|
712
|
+
const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
|
|
664
713
|
const startCursor = resume?.rowCursor ?? null;
|
|
714
|
+
if (prepared.firstPage === undefined) {
|
|
715
|
+
throw new Error('prepareSections did not read the bootstrap page');
|
|
716
|
+
}
|
|
665
717
|
yield {
|
|
666
718
|
type: 'SUB_START',
|
|
667
719
|
id: sub.id,
|
|
@@ -680,6 +732,7 @@ export async function* preparedSection(
|
|
|
680
732
|
trace,
|
|
681
733
|
logEpoch,
|
|
682
734
|
prepared.firstPage,
|
|
735
|
+
prepared.image,
|
|
683
736
|
);
|
|
684
737
|
if (outcome.complete) {
|
|
685
738
|
yield { type: 'SUB_END', nextCursor: asOf };
|
|
@@ -740,3 +793,123 @@ export async function* preparedSection(
|
|
|
740
793
|
yield { type: 'SUB_END', nextCursor };
|
|
741
794
|
return { nextCursor, active: true };
|
|
742
795
|
}
|
|
796
|
+
|
|
797
|
+
export interface PublishSqliteImageInput {
|
|
798
|
+
/** The sync config the serving host runs, plus a `sqliteImageBuilder`. */
|
|
799
|
+
readonly config: SyncServerConfig;
|
|
800
|
+
readonly partition: string;
|
|
801
|
+
readonly table: string;
|
|
802
|
+
/**
|
|
803
|
+
* The effective scopes the subscribing clients hold for `table` (§3.2):
|
|
804
|
+
* the reuse key carries their digest, so publish once per distinct scope
|
|
805
|
+
* set that clients subscribe with.
|
|
806
|
+
*/
|
|
807
|
+
readonly scopes: ScopeMap;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
export interface PublishedSqliteImage {
|
|
811
|
+
readonly segmentId: string;
|
|
812
|
+
/** The §4.7 scope pin the image is stored under. */
|
|
813
|
+
readonly asOfCommitSeq: number;
|
|
814
|
+
readonly scopeDigest: string;
|
|
815
|
+
readonly rowCount: number;
|
|
816
|
+
readonly byteLength: number;
|
|
817
|
+
/** `reused` when an unexpired image for the same reuse key already existed. */
|
|
818
|
+
readonly origin: 'built' | 'reused';
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
/**
|
|
822
|
+
* Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
|
|
823
|
+
* up, from a host with a SQLite engine (a Bun process with the production
|
|
824
|
+
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
825
|
+
* reuse key, the build), so the stored identity is exactly the one a
|
|
826
|
+
* serving host without a builder finds. The image stays current until a
|
|
827
|
+
* change reaches the scope; its lifetime is the segment store's TTL at
|
|
828
|
+
* publication.
|
|
829
|
+
*/
|
|
830
|
+
export async function publishSqliteImage(
|
|
831
|
+
input: PublishSqliteImageInput,
|
|
832
|
+
): Promise<PublishedSqliteImage> {
|
|
833
|
+
const { config, partition } = input;
|
|
834
|
+
if (config.sqliteImageBuilder === undefined) {
|
|
835
|
+
throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
|
|
836
|
+
}
|
|
837
|
+
const schema = compileSchema(config.schema);
|
|
838
|
+
const table = schema.tables.get(input.table);
|
|
839
|
+
if (table === undefined) {
|
|
840
|
+
throw new Error('publishSqliteImage: the schema has no such table');
|
|
841
|
+
}
|
|
842
|
+
await config.storage.ensureSchema(schema, config.checkpoints);
|
|
843
|
+
const logEpoch = await config.storage.getPartitionLogEpoch(partition);
|
|
844
|
+
if (logEpoch === undefined) {
|
|
845
|
+
throw new Error('publishSqliteImage: the partition has no log epoch');
|
|
846
|
+
}
|
|
847
|
+
const ctx: SyncRequestContext = {
|
|
848
|
+
...config,
|
|
849
|
+
partition,
|
|
850
|
+
actorId: 'syncular.image-publisher',
|
|
851
|
+
};
|
|
852
|
+
const plan: SubscriptionPlan = {
|
|
853
|
+
frame: {
|
|
854
|
+
type: 'SUBSCRIPTION',
|
|
855
|
+
id: 'image',
|
|
856
|
+
table: input.table,
|
|
857
|
+
scopes: input.scopes,
|
|
858
|
+
cursor: -1,
|
|
859
|
+
},
|
|
860
|
+
table,
|
|
861
|
+
status: 'active',
|
|
862
|
+
effective: input.scopes,
|
|
863
|
+
};
|
|
864
|
+
// One-row pages make every table above one row image-eligible (§5.3).
|
|
865
|
+
const limits: PullLimits = {
|
|
866
|
+
limitCommits: 1,
|
|
867
|
+
limitSnapshotRows: 1,
|
|
868
|
+
maxSnapshotPages: 1,
|
|
869
|
+
accept: ACCEPT_SQLITE,
|
|
870
|
+
};
|
|
871
|
+
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
872
|
+
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
873
|
+
const trace: PullSectionTrace = { segments: [] };
|
|
874
|
+
const [prepared] = await prepareSections(
|
|
875
|
+
ctx,
|
|
876
|
+
schema,
|
|
877
|
+
limits,
|
|
878
|
+
[plan],
|
|
879
|
+
maxSeq,
|
|
880
|
+
horizonSeq,
|
|
881
|
+
logEpoch,
|
|
882
|
+
);
|
|
883
|
+
if (prepared === undefined) throw new Error('prepareSections lost a plan');
|
|
884
|
+
let ref: SegmentRefFrame | undefined;
|
|
885
|
+
for await (const frame of preparedSection(
|
|
886
|
+
ctx,
|
|
887
|
+
schema,
|
|
888
|
+
limits,
|
|
889
|
+
plan,
|
|
890
|
+
maxSeq,
|
|
891
|
+
prepared,
|
|
892
|
+
trace,
|
|
893
|
+
logEpoch,
|
|
894
|
+
)) {
|
|
895
|
+
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
896
|
+
ref = frame;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
const image = trace.segments.find(
|
|
900
|
+
(segment) => segment.mediaType === 'sqlite',
|
|
901
|
+
);
|
|
902
|
+
if (ref === undefined || image === undefined) {
|
|
903
|
+
throw new Error(
|
|
904
|
+
'publishSqliteImage: the scope holds at most one row, which pulls serve inline',
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
return {
|
|
908
|
+
segmentId: ref.segmentId,
|
|
909
|
+
asOfCommitSeq: ref.asOfCommitSeq,
|
|
910
|
+
scopeDigest: ref.scopeDigest,
|
|
911
|
+
rowCount: ref.rowCount,
|
|
912
|
+
byteLength: ref.byteLength,
|
|
913
|
+
origin: image.origin === 'reused' ? 'reused' : 'built',
|
|
914
|
+
};
|
|
915
|
+
}
|