@powersync/service-module-postgres-storage 0.13.3 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/@types/storage/PostgresBucketStorageFactory.d.ts +2 -1
- package/dist/@types/storage/PostgresReportStorage.d.ts +1 -1
- package/dist/@types/storage/PostgresSyncRulesStorage.d.ts +4 -2
- package/dist/@types/storage/batch/PostgresBucketBatch.d.ts +1 -1
- package/dist/@types/types/models/SyncRules.d.ts +9 -0
- package/dist/@types/types/models/models-index.d.ts +1 -1
- package/dist/@types/utils/test-utils.d.ts +2 -2
- package/dist/@types/utils/utils-index.d.ts +1 -1
- package/dist/migrations/migration-utils.js +2 -2
- package/dist/migrations/migration-utils.js.map +1 -1
- package/dist/migrations/scripts/1771424826685-current-data-pending-deletes.js +1 -1
- package/dist/storage/PostgresBucketStorageFactory.js +6 -5
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js +3 -3
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresSyncRulesStorage.js +72 -39
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/storage/batch/PostgresBucketBatch.js +5 -5
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/current-data-store.js +1 -1
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +3 -3
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +1 -1
- package/dist/types/models/SyncRules.js +3 -1
- package/dist/types/models/SyncRules.js.map +1 -1
- package/dist/types/models/models-index.js +1 -1
- package/dist/types/models/models-index.js.map +1 -1
- package/dist/utils/test-utils.js +3 -3
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/utils-index.js +1 -1
- package/dist/utils/utils-index.js.map +1 -1
- package/package.json +11 -11
- package/src/migrations/migration-utils.ts +2 -2
- package/src/migrations/scripts/1771424826685-current-data-pending-deletes.ts +1 -1
- package/src/storage/PostgresBucketStorageFactory.ts +7 -5
- package/src/storage/PostgresReportStorage.ts +5 -5
- package/src/storage/PostgresSyncRulesStorage.ts +81 -39
- package/src/storage/batch/PostgresBucketBatch.ts +5 -5
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +3 -1
- package/src/storage/current-data-store.ts +1 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +3 -6
- package/src/types/models/SyncRules.ts +3 -1
- package/src/types/models/models-index.ts +1 -1
- package/src/utils/test-utils.ts +3 -3
- package/src/utils/utils-index.ts +1 -1
- package/test/src/__snapshots__/storage_sync.test.ts.snap +282 -0
- package/test/src/connection-report-storage.test.ts +4 -4
- package/test/src/setup.ts +1 -1
- package/test/tsconfig.json +0 -1
- package/dist/@types/storage/current-data-table.d.ts +0 -9
- package/dist/storage/current-data-table.js +0 -22
- package/dist/storage/current-data-table.js.map +0 -1
- package/src/storage/current-data-table.ts +0 -26
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
internalToExternalOpId,
|
|
10
10
|
LastValueSink,
|
|
11
11
|
maxLsn,
|
|
12
|
+
ParameterSetLimitExceededError,
|
|
12
13
|
PartialChecksum,
|
|
13
14
|
PopulateChecksumCacheOptions,
|
|
14
15
|
PopulateChecksumCacheResults,
|
|
@@ -22,7 +23,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
|
|
|
22
23
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
23
24
|
import * as timers from 'timers/promises';
|
|
24
25
|
import * as uuid from 'uuid';
|
|
25
|
-
import { BIGINT_MAX } from '../types/codecs.js';
|
|
26
|
+
import { bigint, BIGINT_MAX } from '../types/codecs.js';
|
|
26
27
|
import { models, RequiredOperationBatchLimits } from '../types/types.js';
|
|
27
28
|
import { replicaIdToSubkey } from '../utils/bson.js';
|
|
28
29
|
import { mapOpEntry } from '../utils/bucket-data.js';
|
|
@@ -30,13 +31,14 @@ import { mapOpEntry } from '../utils/bucket-data.js';
|
|
|
30
31
|
import * as framework from '@powersync/lib-services-framework';
|
|
31
32
|
import { StatementParam } from '@powersync/service-jpgwire';
|
|
32
33
|
import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
|
|
34
|
+
import * as t from 'ts-codec';
|
|
33
35
|
import { SourceTableDecoded, StoredRelationId } from '../types/models/SourceTable.js';
|
|
34
36
|
import { pick } from '../utils/ts-codec.js';
|
|
35
37
|
import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
|
|
36
38
|
import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
|
|
39
|
+
import { PostgresCurrentDataStore } from './current-data-store.js';
|
|
37
40
|
import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
|
|
38
41
|
import { PostgresCompactor } from './PostgresCompactor.js';
|
|
39
|
-
import { PostgresCurrentDataStore } from './current-data-store.js';
|
|
40
42
|
|
|
41
43
|
export type PostgresSyncRulesStorageOptions = {
|
|
42
44
|
factory: PostgresBucketStorageFactory;
|
|
@@ -50,11 +52,14 @@ export class PostgresSyncRulesStorage
|
|
|
50
52
|
extends framework.BaseObserver<storage.SyncRulesBucketStorageListener>
|
|
51
53
|
implements storage.SyncRulesBucketStorage
|
|
52
54
|
{
|
|
55
|
+
[framework.DO_NOT_LOG] = true;
|
|
56
|
+
|
|
53
57
|
public readonly group_id: number;
|
|
54
58
|
public readonly sync_rules: storage.PersistedSyncRulesContent;
|
|
55
59
|
public readonly slot_name: string;
|
|
56
60
|
public readonly factory: PostgresBucketStorageFactory;
|
|
57
61
|
public readonly storageConfig: StorageVersionConfig;
|
|
62
|
+
public readonly logger: framework.Logger;
|
|
58
63
|
|
|
59
64
|
private sharedIterator = new BroadcastIterable((signal) => this.watchActiveCheckpoint(signal));
|
|
60
65
|
|
|
@@ -77,6 +82,7 @@ export class PostgresSyncRulesStorage
|
|
|
77
82
|
this.factory = options.factory;
|
|
78
83
|
this.storageConfig = options.sync_rules.getStorageConfig();
|
|
79
84
|
this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
|
|
85
|
+
this.logger = options.sync_rules.logger;
|
|
80
86
|
|
|
81
87
|
this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
|
|
82
88
|
db: this.db,
|
|
@@ -106,8 +112,8 @@ export class PostgresSyncRulesStorage
|
|
|
106
112
|
getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncRules {
|
|
107
113
|
const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
|
|
108
114
|
/**
|
|
109
|
-
* Check if the cached sync
|
|
110
|
-
* Parse sync
|
|
115
|
+
* Check if the cached sync config, if present, had the same options.
|
|
116
|
+
* Parse sync config if the options are different or if there is no cached value.
|
|
111
117
|
*/
|
|
112
118
|
if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
|
|
113
119
|
this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options };
|
|
@@ -137,11 +143,12 @@ export class PostgresSyncRulesStorage
|
|
|
137
143
|
|
|
138
144
|
return new PostgresCompactor(this.db, this.group_id, {
|
|
139
145
|
...options,
|
|
140
|
-
maxOpId
|
|
146
|
+
maxOpId,
|
|
147
|
+
logger: this.logger
|
|
141
148
|
}).compact();
|
|
142
149
|
}
|
|
143
150
|
|
|
144
|
-
async populatePersistentChecksumCache(
|
|
151
|
+
async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
|
|
145
152
|
// no-op - checksum cache is not implemented for Postgres yet
|
|
146
153
|
return { buckets: 0 };
|
|
147
154
|
}
|
|
@@ -361,7 +368,7 @@ export class PostgresSyncRulesStorage
|
|
|
361
368
|
const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null;
|
|
362
369
|
|
|
363
370
|
const writer = new PostgresBucketBatch({
|
|
364
|
-
logger: options.logger ??
|
|
371
|
+
logger: options.logger ?? this.logger,
|
|
365
372
|
db: this.db,
|
|
366
373
|
sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(),
|
|
367
374
|
group_id: this.group_id,
|
|
@@ -394,42 +401,69 @@ export class PostgresSyncRulesStorage
|
|
|
394
401
|
|
|
395
402
|
async getParameterSets(
|
|
396
403
|
checkpoint: ReplicationCheckpoint,
|
|
397
|
-
lookups: sync_rules.ScopedParameterLookup[]
|
|
398
|
-
|
|
404
|
+
lookups: sync_rules.ScopedParameterLookup[],
|
|
405
|
+
limit: number
|
|
406
|
+
): Promise<sync_rules.ParameterLookupRows[]> {
|
|
399
407
|
const rows = await this.db.sql`
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
bucket_parameters
|
|
406
|
-
FROM
|
|
407
|
-
bucket_parameters
|
|
408
|
-
WHERE
|
|
409
|
-
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
410
|
-
AND lookup = ANY (
|
|
411
|
-
SELECT
|
|
412
|
-
decode((FILTER ->> 0)::text, 'hex') -- Decode the hex string to bytea
|
|
408
|
+
WITH
|
|
409
|
+
rows AS (
|
|
410
|
+
SELECT DISTINCT
|
|
411
|
+
ON (lookup, source_table, source_key) requested.index - 1 AS index,
|
|
412
|
+
bucket_parameters
|
|
413
413
|
FROM
|
|
414
|
+
bucket_parameters,
|
|
414
415
|
jsonb_array_elements(${{
|
|
415
416
|
type: 'jsonb',
|
|
416
417
|
value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
|
|
417
|
-
}}) AS
|
|
418
|
+
}}) WITH ORDINALITY AS requested (value, index)
|
|
419
|
+
WHERE
|
|
420
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
421
|
+
AND lookup = decode((requested.value ->> 0)::text, 'hex') -- Decode the hex string to bytea
|
|
422
|
+
AND id <= ${{ type: 'int8', value: checkpoint.checkpoint }}
|
|
423
|
+
ORDER BY
|
|
424
|
+
lookup,
|
|
425
|
+
source_table,
|
|
426
|
+
source_key,
|
|
427
|
+
id DESC
|
|
418
428
|
)
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
429
|
+
SELECT
|
|
430
|
+
index,
|
|
431
|
+
bucket_parameters
|
|
432
|
+
FROM
|
|
433
|
+
rows
|
|
434
|
+
WHERE
|
|
435
|
+
bucket_parameters != '[]'
|
|
436
|
+
LIMIT
|
|
437
|
+
${{ type: 'int4', value: limit + 1 }}
|
|
425
438
|
`
|
|
426
|
-
.decoded(
|
|
439
|
+
.decoded(parameterSetsRow)
|
|
427
440
|
.rows();
|
|
428
441
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
442
|
+
let totalRows = 0;
|
|
443
|
+
const resultsByLookup = new Map<sync_rules.ScopedParameterLookup, sync_rules.SqliteJsonRow[]>();
|
|
444
|
+
for (const row of rows) {
|
|
445
|
+
const parameterRows = JSONBig.parse(row.bucket_parameters) as sync_rules.SqliteJsonRow[];
|
|
446
|
+
const lookup = lookups[Number(row.index)];
|
|
447
|
+
totalRows += parameterRows.length;
|
|
448
|
+
|
|
449
|
+
const existingResults = resultsByLookup.get(lookup);
|
|
450
|
+
if (existingResults != null) {
|
|
451
|
+
existingResults.push(...parameterRows);
|
|
452
|
+
} else {
|
|
453
|
+
resultsByLookup.set(lookup, parameterRows);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
if (totalRows > limit) {
|
|
458
|
+
// Note that the LIMIT in the query allows more rows than parameters (because each row stores an array of
|
|
459
|
+
// parameter results). That array is very small though, and it doesn't allow fewer rows (due to the != []), so
|
|
460
|
+
// the SQL limit is good enough.
|
|
461
|
+
throw new ParameterSetLimitExceededError(limit);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
const results: sync_rules.ParameterLookupRows[] = [];
|
|
465
|
+
resultsByLookup.forEach((rows, lookup) => results.push({ lookup, rows }));
|
|
466
|
+
return results;
|
|
433
467
|
}
|
|
434
468
|
|
|
435
469
|
async *getBucketDataBatch(
|
|
@@ -646,7 +680,7 @@ export class PostgresSyncRulesStorage
|
|
|
646
680
|
.first();
|
|
647
681
|
|
|
648
682
|
if (syncRulesRow == null) {
|
|
649
|
-
throw new Error('Cannot find
|
|
683
|
+
throw new Error('Cannot find replication stream status');
|
|
650
684
|
}
|
|
651
685
|
|
|
652
686
|
return {
|
|
@@ -839,7 +873,7 @@ export class PostgresSyncRulesStorage
|
|
|
839
873
|
|
|
840
874
|
if (doc == null) {
|
|
841
875
|
// Abort the connections - clients will have to retry later.
|
|
842
|
-
throw new framework.ServiceError(framework.ErrorCode.PSYNC_S2302, 'No active
|
|
876
|
+
throw new framework.ServiceError(framework.ErrorCode.PSYNC_S2302, 'No active replication stream available');
|
|
843
877
|
}
|
|
844
878
|
|
|
845
879
|
const sink = new LastValueSink<string>(undefined);
|
|
@@ -866,7 +900,7 @@ export class PostgresSyncRulesStorage
|
|
|
866
900
|
continue;
|
|
867
901
|
}
|
|
868
902
|
if (Number(notification.active_checkpoint.id) != doc.id) {
|
|
869
|
-
// Active
|
|
903
|
+
// Active replication stream changed - abort and restart the stream
|
|
870
904
|
break;
|
|
871
905
|
}
|
|
872
906
|
|
|
@@ -896,7 +930,15 @@ class PostgresReplicationCheckpoint implements storage.ReplicationCheckpoint {
|
|
|
896
930
|
public readonly lsn: string | null
|
|
897
931
|
) {}
|
|
898
932
|
|
|
899
|
-
getParameterSets(
|
|
900
|
-
|
|
933
|
+
getParameterSets(
|
|
934
|
+
lookups: sync_rules.ScopedParameterLookup[],
|
|
935
|
+
limit: number
|
|
936
|
+
): Promise<sync_rules.ParameterLookupRows[]> {
|
|
937
|
+
return this.storage.getParameterSets(this, lookups, limit);
|
|
901
938
|
}
|
|
902
939
|
}
|
|
940
|
+
|
|
941
|
+
const parameterSetsRow = t.object({
|
|
942
|
+
index: bigint,
|
|
943
|
+
bucket_parameters: t.string
|
|
944
|
+
});
|
|
@@ -20,16 +20,16 @@ import {
|
|
|
20
20
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
21
21
|
import * as timers from 'timers/promises';
|
|
22
22
|
import * as t from 'ts-codec';
|
|
23
|
+
import { bigint } from '../../types/codecs.js';
|
|
23
24
|
import { CurrentBucket, V3CurrentDataDecoded } from '../../types/models/CurrentData.js';
|
|
24
25
|
import { models, RequiredOperationBatchLimits } from '../../types/types.js';
|
|
25
26
|
import { NOTIFICATION_CHANNEL } from '../../utils/db.js';
|
|
26
27
|
import { pick } from '../../utils/ts-codec.js';
|
|
27
28
|
import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js';
|
|
28
|
-
import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
29
|
-
import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
|
|
30
|
-
import { bigint } from '../../types/codecs.js';
|
|
31
29
|
import { PostgresCurrentDataStore } from '../current-data-store.js';
|
|
32
30
|
import { postgresTableId } from '../table-id.js';
|
|
31
|
+
import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
32
|
+
import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
|
|
33
33
|
|
|
34
34
|
export interface PostgresBucketBatchOptions {
|
|
35
35
|
logger: Logger;
|
|
@@ -52,7 +52,7 @@ export interface PostgresBucketBatchOptions {
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
|
-
* Intermediate type which helps for only watching the active
|
|
55
|
+
* Intermediate type which helps for only watching the active replication stream
|
|
56
56
|
* via the Postgres NOTIFY protocol.
|
|
57
57
|
*/
|
|
58
58
|
const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) }));
|
|
@@ -1054,7 +1054,7 @@ export class PostgresBucketBatch
|
|
|
1054
1054
|
}
|
|
1055
1055
|
});
|
|
1056
1056
|
if (didActivate) {
|
|
1057
|
-
this.logger.info(`Activated new
|
|
1057
|
+
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
1058
1058
|
}
|
|
1059
1059
|
}
|
|
1060
1060
|
|
|
@@ -64,7 +64,9 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
64
64
|
switch (this.writeCheckpointMode) {
|
|
65
65
|
case storage.WriteCheckpointMode.CUSTOM:
|
|
66
66
|
if (false == 'sync_rules_id' in filters) {
|
|
67
|
-
throw new framework.errors.ValidationError(
|
|
67
|
+
throw new framework.errors.ValidationError(
|
|
68
|
+
`Replication stream ID is required for custom Write Checkpoint filtering`
|
|
69
|
+
);
|
|
68
70
|
}
|
|
69
71
|
return this.lastCustomWriteCheckpoint(filters as storage.CustomWriteCheckpointFilters);
|
|
70
72
|
case storage.WriteCheckpointMode.MANAGED:
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
2
|
import { storage } from '@powersync/service-core';
|
|
3
3
|
import * as t from 'ts-codec';
|
|
4
|
-
import { pick } from '../utils/ts-codec.js';
|
|
5
4
|
import * as models from '../types/models/CurrentData.js';
|
|
5
|
+
import { pick } from '../utils/ts-codec.js';
|
|
6
6
|
|
|
7
7
|
type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows'>;
|
|
8
8
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { ErrorCode,
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
4
|
import { models } from '../../types/types.js';
|
|
5
5
|
|
|
@@ -31,17 +31,14 @@ export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRule
|
|
|
31
31
|
});
|
|
32
32
|
const lockHandle = await manager.acquire();
|
|
33
33
|
if (!lockHandle) {
|
|
34
|
-
throw new ServiceError(
|
|
35
|
-
ErrorCode.PSYNC_S1003,
|
|
36
|
-
`Sync rules: ${this.id} have been locked by another process for replication.`
|
|
37
|
-
);
|
|
34
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
|
|
38
35
|
}
|
|
39
36
|
|
|
40
37
|
const interval = setInterval(async () => {
|
|
41
38
|
try {
|
|
42
39
|
await lockHandle.refresh();
|
|
43
40
|
} catch (e) {
|
|
44
|
-
logger.error('Failed to refresh lock', e);
|
|
41
|
+
this.logger.error('Failed to refresh lock', e);
|
|
45
42
|
clearInterval(interval);
|
|
46
43
|
}
|
|
47
44
|
}, 30_130);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { framework, storage } from '@powersync/service-core';
|
|
2
|
+
import { ReplicationError } from '@powersync/service-types';
|
|
2
3
|
import * as t from 'ts-codec';
|
|
3
4
|
import { bigint, pgwire_number } from '../codecs.js';
|
|
4
5
|
import { jsonContainerObject } from './json.js';
|
|
@@ -59,7 +60,8 @@ export const SyncRules = t.object({
|
|
|
59
60
|
overrides: t.record(t.boolean),
|
|
60
61
|
maxTimeValuePrecision: t.number.optional()
|
|
61
62
|
}),
|
|
62
|
-
eventDescriptors: t.record(t.array(t.string))
|
|
63
|
+
eventDescriptors: t.record(t.array(t.string)),
|
|
64
|
+
errors: t.array(ReplicationError).optional()
|
|
63
65
|
})
|
|
64
66
|
)
|
|
65
67
|
)
|
|
@@ -5,7 +5,7 @@ export * from './BucketParameters.js';
|
|
|
5
5
|
export * from './CurrentData.js';
|
|
6
6
|
export * from './Instance.js';
|
|
7
7
|
export * from './Migration.js';
|
|
8
|
+
export * from './SdkReporting.js';
|
|
8
9
|
export * from './SourceTable.js';
|
|
9
10
|
export * from './SyncRules.js';
|
|
10
11
|
export * from './WriteCheckpoint.js';
|
|
11
|
-
export * from './SdkReporting.js';
|
package/src/utils/test-utils.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import { createLogger, logger as defaultLogger, transports } from '@powersync/lib-services-framework';
|
|
1
2
|
import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core';
|
|
2
3
|
import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
|
|
3
|
-
import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
|
|
4
|
-
import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
|
|
5
4
|
import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
|
|
6
|
-
import {
|
|
5
|
+
import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
|
|
6
|
+
import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
|
|
7
7
|
import { truncateTables } from './db.js';
|
|
8
8
|
|
|
9
9
|
export type PostgresTestStorageOptions = {
|
package/src/utils/utils-index.ts
CHANGED
|
@@ -1,5 +1,99 @@
|
|
|
1
1
|
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
|
2
2
|
|
|
3
|
+
exports[`sync - postgres > storage v1 > can override priority when subscribing to stream 1`] = `
|
|
4
|
+
[
|
|
5
|
+
{
|
|
6
|
+
"checkpoint": {
|
|
7
|
+
"buckets": [
|
|
8
|
+
{
|
|
9
|
+
"bucket": "1#todos|0["a"]",
|
|
10
|
+
"checksum": -1712802421,
|
|
11
|
+
"count": 1,
|
|
12
|
+
"priority": 0,
|
|
13
|
+
"subscriptions": [
|
|
14
|
+
{
|
|
15
|
+
"sub": 0,
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"sub": 1,
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"bucket": "1#todos|0["b"]",
|
|
24
|
+
"checksum": -1291414318,
|
|
25
|
+
"count": 1,
|
|
26
|
+
"priority": 3,
|
|
27
|
+
"subscriptions": [
|
|
28
|
+
{
|
|
29
|
+
"sub": 1,
|
|
30
|
+
},
|
|
31
|
+
],
|
|
32
|
+
},
|
|
33
|
+
],
|
|
34
|
+
"last_op_id": "2",
|
|
35
|
+
"streams": [
|
|
36
|
+
{
|
|
37
|
+
"errors": [],
|
|
38
|
+
"is_default": false,
|
|
39
|
+
"name": "todos",
|
|
40
|
+
},
|
|
41
|
+
],
|
|
42
|
+
"write_checkpoint": undefined,
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"data": {
|
|
47
|
+
"after": "0",
|
|
48
|
+
"bucket": "1#todos|0["a"]",
|
|
49
|
+
"data": [
|
|
50
|
+
{
|
|
51
|
+
"checksum": 2582164875,
|
|
52
|
+
"data": "{"id":"a","description":"Test 1"}",
|
|
53
|
+
"object_id": "a",
|
|
54
|
+
"object_type": "test",
|
|
55
|
+
"op": "PUT",
|
|
56
|
+
"op_id": "1",
|
|
57
|
+
"subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
|
|
58
|
+
},
|
|
59
|
+
],
|
|
60
|
+
"has_more": false,
|
|
61
|
+
"next_after": "1",
|
|
62
|
+
},
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"partial_checkpoint_complete": {
|
|
66
|
+
"last_op_id": "2",
|
|
67
|
+
"priority": 0,
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"data": {
|
|
72
|
+
"after": "0",
|
|
73
|
+
"bucket": "1#todos|0["b"]",
|
|
74
|
+
"data": [
|
|
75
|
+
{
|
|
76
|
+
"checksum": 3003552978,
|
|
77
|
+
"data": "{"id":"b","description":"Test 2"}",
|
|
78
|
+
"object_id": "b",
|
|
79
|
+
"object_type": "test",
|
|
80
|
+
"op": "PUT",
|
|
81
|
+
"op_id": "2",
|
|
82
|
+
"subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64",
|
|
83
|
+
},
|
|
84
|
+
],
|
|
85
|
+
"has_more": false,
|
|
86
|
+
"next_after": "2",
|
|
87
|
+
},
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"checkpoint_complete": {
|
|
91
|
+
"last_op_id": "2",
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
]
|
|
95
|
+
`;
|
|
96
|
+
|
|
3
97
|
exports[`sync - postgres > storage v1 > compacting data - invalidate checkpoint 1`] = `
|
|
4
98
|
[
|
|
5
99
|
{
|
|
@@ -1095,6 +1189,100 @@ exports[`sync - postgres > storage v1 > sync updates to parameter query only 2`]
|
|
|
1095
1189
|
]
|
|
1096
1190
|
`;
|
|
1097
1191
|
|
|
1192
|
+
exports[`sync - postgres > storage v2 > can override priority when subscribing to stream 1`] = `
|
|
1193
|
+
[
|
|
1194
|
+
{
|
|
1195
|
+
"checkpoint": {
|
|
1196
|
+
"buckets": [
|
|
1197
|
+
{
|
|
1198
|
+
"bucket": "1#todos|0["a"]",
|
|
1199
|
+
"checksum": -1712802421,
|
|
1200
|
+
"count": 1,
|
|
1201
|
+
"priority": 0,
|
|
1202
|
+
"subscriptions": [
|
|
1203
|
+
{
|
|
1204
|
+
"sub": 0,
|
|
1205
|
+
},
|
|
1206
|
+
{
|
|
1207
|
+
"sub": 1,
|
|
1208
|
+
},
|
|
1209
|
+
],
|
|
1210
|
+
},
|
|
1211
|
+
{
|
|
1212
|
+
"bucket": "1#todos|0["b"]",
|
|
1213
|
+
"checksum": -1291414318,
|
|
1214
|
+
"count": 1,
|
|
1215
|
+
"priority": 3,
|
|
1216
|
+
"subscriptions": [
|
|
1217
|
+
{
|
|
1218
|
+
"sub": 1,
|
|
1219
|
+
},
|
|
1220
|
+
],
|
|
1221
|
+
},
|
|
1222
|
+
],
|
|
1223
|
+
"last_op_id": "2",
|
|
1224
|
+
"streams": [
|
|
1225
|
+
{
|
|
1226
|
+
"errors": [],
|
|
1227
|
+
"is_default": false,
|
|
1228
|
+
"name": "todos",
|
|
1229
|
+
},
|
|
1230
|
+
],
|
|
1231
|
+
"write_checkpoint": undefined,
|
|
1232
|
+
},
|
|
1233
|
+
},
|
|
1234
|
+
{
|
|
1235
|
+
"data": {
|
|
1236
|
+
"after": "0",
|
|
1237
|
+
"bucket": "1#todos|0["a"]",
|
|
1238
|
+
"data": [
|
|
1239
|
+
{
|
|
1240
|
+
"checksum": 2582164875,
|
|
1241
|
+
"data": "{"id":"a","description":"Test 1"}",
|
|
1242
|
+
"object_id": "a",
|
|
1243
|
+
"object_type": "test",
|
|
1244
|
+
"op": "PUT",
|
|
1245
|
+
"op_id": "1",
|
|
1246
|
+
"subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
|
|
1247
|
+
},
|
|
1248
|
+
],
|
|
1249
|
+
"has_more": false,
|
|
1250
|
+
"next_after": "1",
|
|
1251
|
+
},
|
|
1252
|
+
},
|
|
1253
|
+
{
|
|
1254
|
+
"partial_checkpoint_complete": {
|
|
1255
|
+
"last_op_id": "2",
|
|
1256
|
+
"priority": 0,
|
|
1257
|
+
},
|
|
1258
|
+
},
|
|
1259
|
+
{
|
|
1260
|
+
"data": {
|
|
1261
|
+
"after": "0",
|
|
1262
|
+
"bucket": "1#todos|0["b"]",
|
|
1263
|
+
"data": [
|
|
1264
|
+
{
|
|
1265
|
+
"checksum": 3003552978,
|
|
1266
|
+
"data": "{"id":"b","description":"Test 2"}",
|
|
1267
|
+
"object_id": "b",
|
|
1268
|
+
"object_type": "test",
|
|
1269
|
+
"op": "PUT",
|
|
1270
|
+
"op_id": "2",
|
|
1271
|
+
"subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64",
|
|
1272
|
+
},
|
|
1273
|
+
],
|
|
1274
|
+
"has_more": false,
|
|
1275
|
+
"next_after": "2",
|
|
1276
|
+
},
|
|
1277
|
+
},
|
|
1278
|
+
{
|
|
1279
|
+
"checkpoint_complete": {
|
|
1280
|
+
"last_op_id": "2",
|
|
1281
|
+
},
|
|
1282
|
+
},
|
|
1283
|
+
]
|
|
1284
|
+
`;
|
|
1285
|
+
|
|
1098
1286
|
exports[`sync - postgres > storage v2 > compacting data - invalidate checkpoint 1`] = `
|
|
1099
1287
|
[
|
|
1100
1288
|
{
|
|
@@ -2190,6 +2378,100 @@ exports[`sync - postgres > storage v2 > sync updates to parameter query only 2`]
|
|
|
2190
2378
|
]
|
|
2191
2379
|
`;
|
|
2192
2380
|
|
|
2381
|
+
exports[`sync - postgres > storage v3 > can override priority when subscribing to stream 1`] = `
|
|
2382
|
+
[
|
|
2383
|
+
{
|
|
2384
|
+
"checkpoint": {
|
|
2385
|
+
"buckets": [
|
|
2386
|
+
{
|
|
2387
|
+
"bucket": "1#todos|0["a"]",
|
|
2388
|
+
"checksum": -1712802421,
|
|
2389
|
+
"count": 1,
|
|
2390
|
+
"priority": 0,
|
|
2391
|
+
"subscriptions": [
|
|
2392
|
+
{
|
|
2393
|
+
"sub": 0,
|
|
2394
|
+
},
|
|
2395
|
+
{
|
|
2396
|
+
"sub": 1,
|
|
2397
|
+
},
|
|
2398
|
+
],
|
|
2399
|
+
},
|
|
2400
|
+
{
|
|
2401
|
+
"bucket": "1#todos|0["b"]",
|
|
2402
|
+
"checksum": -1291414318,
|
|
2403
|
+
"count": 1,
|
|
2404
|
+
"priority": 3,
|
|
2405
|
+
"subscriptions": [
|
|
2406
|
+
{
|
|
2407
|
+
"sub": 1,
|
|
2408
|
+
},
|
|
2409
|
+
],
|
|
2410
|
+
},
|
|
2411
|
+
],
|
|
2412
|
+
"last_op_id": "2",
|
|
2413
|
+
"streams": [
|
|
2414
|
+
{
|
|
2415
|
+
"errors": [],
|
|
2416
|
+
"is_default": false,
|
|
2417
|
+
"name": "todos",
|
|
2418
|
+
},
|
|
2419
|
+
],
|
|
2420
|
+
"write_checkpoint": undefined,
|
|
2421
|
+
},
|
|
2422
|
+
},
|
|
2423
|
+
{
|
|
2424
|
+
"data": {
|
|
2425
|
+
"after": "0",
|
|
2426
|
+
"bucket": "1#todos|0["a"]",
|
|
2427
|
+
"data": [
|
|
2428
|
+
{
|
|
2429
|
+
"checksum": 2582164875,
|
|
2430
|
+
"data": "{"id":"a","description":"Test 1"}",
|
|
2431
|
+
"object_id": "a",
|
|
2432
|
+
"object_type": "test",
|
|
2433
|
+
"op": "PUT",
|
|
2434
|
+
"op_id": "1",
|
|
2435
|
+
"subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
|
|
2436
|
+
},
|
|
2437
|
+
],
|
|
2438
|
+
"has_more": false,
|
|
2439
|
+
"next_after": "1",
|
|
2440
|
+
},
|
|
2441
|
+
},
|
|
2442
|
+
{
|
|
2443
|
+
"partial_checkpoint_complete": {
|
|
2444
|
+
"last_op_id": "2",
|
|
2445
|
+
"priority": 0,
|
|
2446
|
+
},
|
|
2447
|
+
},
|
|
2448
|
+
{
|
|
2449
|
+
"data": {
|
|
2450
|
+
"after": "0",
|
|
2451
|
+
"bucket": "1#todos|0["b"]",
|
|
2452
|
+
"data": [
|
|
2453
|
+
{
|
|
2454
|
+
"checksum": 3003552978,
|
|
2455
|
+
"data": "{"id":"b","description":"Test 2"}",
|
|
2456
|
+
"object_id": "b",
|
|
2457
|
+
"object_type": "test",
|
|
2458
|
+
"op": "PUT",
|
|
2459
|
+
"op_id": "2",
|
|
2460
|
+
"subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64",
|
|
2461
|
+
},
|
|
2462
|
+
],
|
|
2463
|
+
"has_more": false,
|
|
2464
|
+
"next_after": "2",
|
|
2465
|
+
},
|
|
2466
|
+
},
|
|
2467
|
+
{
|
|
2468
|
+
"checkpoint_complete": {
|
|
2469
|
+
"last_op_id": "2",
|
|
2470
|
+
},
|
|
2471
|
+
},
|
|
2472
|
+
]
|
|
2473
|
+
`;
|
|
2474
|
+
|
|
2193
2475
|
exports[`sync - postgres > storage v3 > compacting data - invalidate checkpoint 1`] = `
|
|
2194
2476
|
[
|
|
2195
2477
|
{
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
2
|
-
import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js';
|
|
3
|
-
import { event_types } from '@powersync/service-types';
|
|
4
1
|
import { register, ReportUserData } from '@powersync/service-core-tests';
|
|
5
|
-
import { PostgresReportStorage } from '../../src/storage/PostgresReportStorage.js';
|
|
6
2
|
import { DateTimeValue } from '@powersync/service-sync-rules';
|
|
3
|
+
import { event_types } from '@powersync/service-types';
|
|
4
|
+
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
5
|
+
import { PostgresReportStorage } from '../../src/storage/PostgresReportStorage.js';
|
|
6
|
+
import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js';
|
|
7
7
|
|
|
8
8
|
const factory = await POSTGRES_REPORT_STORAGE_FACTORY();
|
|
9
9
|
const userData = register.REPORT_TEST_USERS;
|