@powersync/service-module-postgres-storage 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -7
- package/dist/.tsbuildinfo +1 -1
- package/dist/@types/migrations/scripts/1771424826685-current-data-pending-deletes.d.ts +3 -0
- package/dist/@types/storage/PostgresBucketStorageFactory.d.ts +6 -1
- package/dist/@types/storage/PostgresCompactor.d.ts +8 -2
- package/dist/@types/storage/PostgresReportStorage.d.ts +1 -1
- package/dist/@types/storage/PostgresSyncRulesStorage.d.ts +14 -6
- package/dist/@types/storage/batch/OperationBatch.d.ts +2 -2
- package/dist/@types/storage/batch/PostgresBucketBatch.d.ts +14 -10
- package/dist/@types/storage/batch/PostgresPersistedBatch.d.ts +17 -5
- package/dist/@types/storage/current-data-store.d.ts +85 -0
- package/dist/@types/storage/table-id.d.ts +2 -0
- package/dist/@types/types/models/CurrentData.d.ts +18 -3
- package/dist/@types/types/models/SyncRules.d.ts +9 -0
- package/dist/@types/types/models/models-index.d.ts +1 -1
- package/dist/@types/utils/bson.d.ts +1 -1
- package/dist/@types/utils/test-utils.d.ts +3 -3
- package/dist/@types/utils/utils-index.d.ts +1 -1
- package/dist/migrations/migration-utils.js +2 -2
- package/dist/migrations/migration-utils.js.map +1 -1
- package/dist/migrations/scripts/1684951997326-init.js +24 -100
- package/dist/migrations/scripts/1684951997326-init.js.map +1 -1
- package/dist/migrations/scripts/1749024804042-snapshot-progress.js +12 -88
- package/dist/migrations/scripts/1749024804042-snapshot-progress.js.map +1 -1
- package/dist/migrations/scripts/1756282360128-connection-reporting.js +12 -88
- package/dist/migrations/scripts/1756282360128-connection-reporting.js.map +1 -1
- package/dist/migrations/scripts/1771232439485-storage-version.js +14 -90
- package/dist/migrations/scripts/1771232439485-storage-version.js.map +1 -1
- package/dist/migrations/scripts/1771424826685-current-data-pending-deletes.js +8 -0
- package/dist/migrations/scripts/1771424826685-current-data-pending-deletes.js.map +1 -0
- package/dist/migrations/scripts/1771491856000-sync-plan.js +6 -82
- package/dist/migrations/scripts/1771491856000-sync-plan.js.map +1 -1
- package/dist/storage/PostgresBucketStorageFactory.js +47 -9
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresCompactor.js +14 -6
- package/dist/storage/PostgresCompactor.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js +3 -3
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresSyncRulesStorage.js +93 -60
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/storage/batch/OperationBatch.js +2 -1
- package/dist/storage/batch/OperationBatch.js.map +1 -1
- package/dist/storage/batch/PostgresBucketBatch.js +297 -215
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/batch/PostgresPersistedBatch.js +86 -81
- package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/current-data-store.js +270 -0
- package/dist/storage/current-data-store.js.map +1 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +3 -3
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/table-id.js +8 -0
- package/dist/storage/table-id.js.map +1 -0
- package/dist/types/models/CurrentData.js +11 -2
- package/dist/types/models/CurrentData.js.map +1 -1
- package/dist/types/models/SyncRules.js +3 -1
- package/dist/types/models/SyncRules.js.map +1 -1
- package/dist/types/models/models-index.js +1 -1
- package/dist/types/models/models-index.js.map +1 -1
- package/dist/utils/bson.js.map +1 -1
- package/dist/utils/db.js +9 -0
- package/dist/utils/db.js.map +1 -1
- package/dist/utils/test-utils.js +39 -108
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/utils-index.js +1 -1
- package/dist/utils/utils-index.js.map +1 -1
- package/package.json +12 -13
- package/src/migrations/migration-utils.ts +2 -2
- package/src/migrations/scripts/1771424826685-current-data-pending-deletes.ts +10 -0
- package/src/storage/PostgresBucketStorageFactory.ts +60 -10
- package/src/storage/PostgresCompactor.ts +17 -8
- package/src/storage/PostgresReportStorage.ts +5 -5
- package/src/storage/PostgresSyncRulesStorage.ts +108 -65
- package/src/storage/batch/OperationBatch.ts +4 -3
- package/src/storage/batch/PostgresBucketBatch.ts +318 -240
- package/src/storage/batch/PostgresPersistedBatch.ts +92 -84
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +3 -1
- package/src/storage/current-data-store.ts +326 -0
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +3 -6
- package/src/storage/table-id.ts +9 -0
- package/src/types/models/CurrentData.ts +17 -4
- package/src/types/models/SyncRules.ts +3 -1
- package/src/types/models/models-index.ts +1 -1
- package/src/utils/bson.ts +1 -1
- package/src/utils/db.ts +10 -0
- package/src/utils/test-utils.ts +16 -9
- package/src/utils/utils-index.ts +1 -1
- package/test/src/__snapshots__/storage.test.ts.snap +151 -0
- package/test/src/__snapshots__/storage_compacting.test.ts.snap +17 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +1393 -16
- package/test/src/connection-report-storage.test.ts +4 -4
- package/test/src/env.ts +1 -1
- package/test/src/migrations.test.ts +1 -1
- package/test/src/setup.ts +1 -1
- package/test/src/storage.test.ts +138 -131
- package/test/src/storage_compacting.test.ts +80 -11
- package/test/src/storage_sync.test.ts +57 -54
- package/test/src/util.ts +4 -4
- package/test/tsconfig.json +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core';
|
|
1
|
+
import { framework, GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core';
|
|
2
2
|
import * as pg_wire from '@powersync/service-jpgwire';
|
|
3
3
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
4
4
|
import { NormalizedPostgresStorageConfig } from '../types/types.js';
|
|
@@ -9,6 +9,7 @@ export type PostgresBucketStorageOptions = {
|
|
|
9
9
|
};
|
|
10
10
|
export declare class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
11
11
|
protected options: PostgresBucketStorageOptions;
|
|
12
|
+
[framework.DO_NOT_LOG]: boolean;
|
|
12
13
|
readonly db: lib_postgres.DatabaseClient;
|
|
13
14
|
readonly slot_name_prefix: string;
|
|
14
15
|
private activeStorageCache;
|
|
@@ -20,6 +21,10 @@ export declare class PostgresBucketStorageFactory extends storage.BucketStorageF
|
|
|
20
21
|
getPowerSyncInstanceId(): Promise<string>;
|
|
21
22
|
getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier>;
|
|
22
23
|
updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedSyncRulesContent>;
|
|
24
|
+
/**
|
|
25
|
+
* Lazy-initializes storage-version-specific structures, if needed.
|
|
26
|
+
*/
|
|
27
|
+
private initializeStorageVersion;
|
|
23
28
|
restartReplication(sync_rules_group_id: number): Promise<void>;
|
|
24
29
|
getActiveSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null>;
|
|
25
30
|
getNextSyncRulesContent(): Promise<storage.PersistedSyncRulesContent | null>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { InternalOpId, storage } from '@powersync/service-core';
|
|
3
3
|
/**
|
|
4
4
|
* Additional options, primarily for testing.
|
|
5
5
|
*/
|
|
@@ -15,7 +15,7 @@ export declare class PostgresCompactor {
|
|
|
15
15
|
private clearBatchLimit;
|
|
16
16
|
private maxOpId;
|
|
17
17
|
private buckets;
|
|
18
|
-
constructor(db: lib_postgres.DatabaseClient, group_id: number, options
|
|
18
|
+
constructor(db: lib_postgres.DatabaseClient, group_id: number, options: PostgresCompactOptions);
|
|
19
19
|
/**
|
|
20
20
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
21
21
|
*
|
|
@@ -25,6 +25,12 @@ export declare class PostgresCompactor {
|
|
|
25
25
|
private compactAllBuckets;
|
|
26
26
|
private compactSingleBucket;
|
|
27
27
|
private flush;
|
|
28
|
+
/**
|
|
29
|
+
* Expose the internal clearBucket() method to tests.
|
|
30
|
+
*
|
|
31
|
+
* @deprecated Only for tests
|
|
32
|
+
*/
|
|
33
|
+
clearBucketForTests(bucket: string, op: InternalOpId): Promise<void>;
|
|
28
34
|
/**
|
|
29
35
|
* Perform a CLEAR compact for a bucket.
|
|
30
36
|
*
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
1
2
|
import { storage } from '@powersync/service-core';
|
|
2
3
|
import * as pg_wire from '@powersync/service-jpgwire';
|
|
3
4
|
import { event_types } from '@powersync/service-types';
|
|
4
|
-
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
5
5
|
import { NormalizedPostgresStorageConfig } from '../types/types.js';
|
|
6
6
|
export type PostgresReportStorageOptions = {
|
|
7
7
|
config: NormalizedPostgresStorageConfig;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, PopulateChecksumCacheOptions, PopulateChecksumCacheResults, ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
|
|
2
|
+
import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, PopulateChecksumCacheOptions, PopulateChecksumCacheResults, ReplicationCheckpoint, storage, StorageVersionConfig, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
|
|
3
3
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
4
4
|
import { RequiredOperationBatchLimits } from '../types/types.js';
|
|
5
5
|
import * as framework from '@powersync/lib-services-framework';
|
|
@@ -14,13 +14,17 @@ export type PostgresSyncRulesStorageOptions = {
|
|
|
14
14
|
};
|
|
15
15
|
export declare class PostgresSyncRulesStorage extends framework.BaseObserver<storage.SyncRulesBucketStorageListener> implements storage.SyncRulesBucketStorage {
|
|
16
16
|
protected options: PostgresSyncRulesStorageOptions;
|
|
17
|
+
[framework.DO_NOT_LOG]: boolean;
|
|
17
18
|
readonly group_id: number;
|
|
18
19
|
readonly sync_rules: storage.PersistedSyncRulesContent;
|
|
19
20
|
readonly slot_name: string;
|
|
20
21
|
readonly factory: PostgresBucketStorageFactory;
|
|
22
|
+
readonly storageConfig: StorageVersionConfig;
|
|
23
|
+
readonly logger: framework.Logger;
|
|
21
24
|
private sharedIterator;
|
|
22
25
|
protected db: lib_postgres.DatabaseClient;
|
|
23
26
|
protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
|
|
27
|
+
private readonly currentDataStore;
|
|
24
28
|
private parsedSyncRulesCache;
|
|
25
29
|
private _checksumCache;
|
|
26
30
|
constructor(options: PostgresSyncRulesStorageOptions);
|
|
@@ -34,16 +38,20 @@ export declare class PostgresSyncRulesStorage extends framework.BaseObserver<sto
|
|
|
34
38
|
getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncRules;
|
|
35
39
|
reportError(e: any): Promise<void>;
|
|
36
40
|
compact(options?: storage.CompactOptions): Promise<void>;
|
|
37
|
-
populatePersistentChecksumCache(
|
|
41
|
+
populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults>;
|
|
38
42
|
lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
|
|
39
43
|
setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void;
|
|
40
44
|
createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint>;
|
|
41
45
|
getCheckpoint(): Promise<storage.ReplicationCheckpoint>;
|
|
42
46
|
resolveTable(options: storage.ResolveTableOptions): Promise<storage.ResolveTableResult>;
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
+
createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
|
|
48
|
+
/**
|
|
49
|
+
* @deprecated Use `createWriter()` with `await using` instead.
|
|
50
|
+
*/
|
|
51
|
+
startBatch(options: storage.CreateWriterOptions, callback: (batch: storage.BucketStorageBatch) => Promise<void>): Promise<storage.FlushedResult | null>;
|
|
52
|
+
getParameterSets(checkpoint: ReplicationCheckpoint, lookups: sync_rules.ScopedParameterLookup[], limit: number): Promise<sync_rules.SqliteJsonRow[]>;
|
|
53
|
+
getBucketDataBatch(checkpoint: InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
|
|
54
|
+
getChecksums(checkpoint: utils.InternalOpId, buckets: storage.BucketChecksumRequest[]): Promise<utils.ChecksumMap>;
|
|
47
55
|
clearChecksumCache(): void;
|
|
48
56
|
terminate(options?: storage.TerminateOptions): Promise<void>;
|
|
49
57
|
getStatus(): Promise<storage.SyncRuleStatus>;
|
|
@@ -40,8 +40,8 @@ export declare class RecordOperation {
|
|
|
40
40
|
/**
|
|
41
41
|
* In-memory cache key - must not be persisted.
|
|
42
42
|
*/
|
|
43
|
-
export declare function cacheKey(sourceTableId:
|
|
43
|
+
export declare function cacheKey(sourceTableId: storage.SourceTableId, id: storage.ReplicaId): string;
|
|
44
44
|
/**
|
|
45
45
|
* Calculates a cache key for a stored ReplicaId. This is usually stored as a bytea/Buffer.
|
|
46
46
|
*/
|
|
47
|
-
export declare function encodedCacheKey(sourceTableId:
|
|
47
|
+
export declare function encodedCacheKey(sourceTableId: storage.SourceTableId, storedKey: Buffer): string;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
2
|
import { BaseObserver, Logger } from '@powersync/lib-services-framework';
|
|
3
|
-
import { BucketStorageMarkRecordUnavailable, InternalOpId, storage } from '@powersync/service-core';
|
|
3
|
+
import { BucketStorageMarkRecordUnavailable, CheckpointResult, InternalOpId, storage } from '@powersync/service-core';
|
|
4
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
5
|
import * as t from 'ts-codec';
|
|
6
|
-
import {
|
|
6
|
+
import { V3CurrentDataDecoded } from '../../types/models/CurrentData.js';
|
|
7
7
|
import { RequiredOperationBatchLimits } from '../../types/types.js';
|
|
8
8
|
import { OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
9
9
|
import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
|
|
@@ -14,7 +14,6 @@ export interface PostgresBucketBatchOptions {
|
|
|
14
14
|
group_id: number;
|
|
15
15
|
slot_name: string;
|
|
16
16
|
last_checkpoint_lsn: string | null;
|
|
17
|
-
no_checkpoint_before_lsn: string;
|
|
18
17
|
store_current_data: boolean;
|
|
19
18
|
keep_alive_op?: InternalOpId | null;
|
|
20
19
|
resumeFromLsn: string | null;
|
|
@@ -24,9 +23,10 @@ export interface PostgresBucketBatchOptions {
|
|
|
24
23
|
skip_existing_rows: boolean;
|
|
25
24
|
batch_limits: RequiredOperationBatchLimits;
|
|
26
25
|
markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
|
|
26
|
+
storageConfig: storage.StorageVersionConfig;
|
|
27
27
|
}
|
|
28
28
|
/**
|
|
29
|
-
* Intermediate type which helps for only watching the active
|
|
29
|
+
* Intermediate type which helps for only watching the active replication stream
|
|
30
30
|
* via the Postgres NOTIFY protocol.
|
|
31
31
|
*/
|
|
32
32
|
declare const StatefulCheckpoint: t.Intersection<t.Codec<{
|
|
@@ -49,7 +49,6 @@ export declare class PostgresBucketBatch extends BaseObserver<storage.BucketBatc
|
|
|
49
49
|
protected db: lib_postgres.DatabaseClient;
|
|
50
50
|
protected group_id: number;
|
|
51
51
|
protected last_checkpoint_lsn: string | null;
|
|
52
|
-
protected no_checkpoint_before_lsn: string;
|
|
53
52
|
protected persisted_op: InternalOpId | null;
|
|
54
53
|
protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[];
|
|
55
54
|
protected readonly sync_rules: sync_rules.HydratedSyncRules;
|
|
@@ -58,24 +57,28 @@ export declare class PostgresBucketBatch extends BaseObserver<storage.BucketBatc
|
|
|
58
57
|
private markRecordUnavailable;
|
|
59
58
|
private needsActivation;
|
|
60
59
|
private clearedError;
|
|
60
|
+
private readonly storageConfig;
|
|
61
|
+
private readonly currentDataStore;
|
|
61
62
|
constructor(options: PostgresBucketBatchOptions);
|
|
62
63
|
get lastCheckpointLsn(): string | null;
|
|
63
|
-
get noCheckpointBeforeLsn(): string;
|
|
64
64
|
[Symbol.asyncDispose](): Promise<void>;
|
|
65
|
+
dispose(): Promise<void>;
|
|
65
66
|
save(record: storage.SaveOptions): Promise<storage.FlushedResult | null>;
|
|
66
67
|
truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
|
|
67
68
|
protected truncateSingle(sourceTable: storage.SourceTable): Promise<bigint | null>;
|
|
68
69
|
drop(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
|
|
69
70
|
flush(): Promise<storage.FlushedResult | null>;
|
|
70
71
|
private flushInner;
|
|
71
|
-
commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<
|
|
72
|
-
keepalive(lsn: string): Promise<
|
|
72
|
+
commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<CheckpointResult>;
|
|
73
|
+
keepalive(lsn: string): Promise<CheckpointResult>;
|
|
73
74
|
setResumeLsn(lsn: string): Promise<void>;
|
|
74
|
-
|
|
75
|
+
markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
|
|
76
|
+
markTableSnapshotRequired(table: storage.SourceTable): Promise<void>;
|
|
77
|
+
markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
|
|
75
78
|
updateTableProgress(table: storage.SourceTable, progress: Partial<storage.TableSnapshotStatus>): Promise<storage.SourceTable>;
|
|
76
79
|
addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void;
|
|
77
80
|
protected replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch): Promise<OperationBatch | null>;
|
|
78
|
-
protected saveOperation(persistedBatch: PostgresPersistedBatch, operation: RecordOperation, currentData?:
|
|
81
|
+
protected saveOperation(persistedBatch: PostgresPersistedBatch, operation: RecordOperation, currentData?: V3CurrentDataDecoded | null): Promise<{
|
|
79
82
|
group_id: number;
|
|
80
83
|
source_table: string;
|
|
81
84
|
source_key: Buffer<ArrayBuffer>;
|
|
@@ -86,6 +89,7 @@ export declare class PostgresBucketBatch extends BaseObserver<storage.BucketBatc
|
|
|
86
89
|
table: string;
|
|
87
90
|
}[];
|
|
88
91
|
lookups: Buffer<ArrayBuffer>[];
|
|
92
|
+
pending_delete: bigint | null;
|
|
89
93
|
} | null>;
|
|
90
94
|
/**
|
|
91
95
|
* Switch from processing -> active if relevant.
|
|
@@ -18,7 +18,7 @@ export type SaveParameterDataOptions = {
|
|
|
18
18
|
existing_lookups: Buffer[];
|
|
19
19
|
};
|
|
20
20
|
export type DeleteCurrentDataOptions = {
|
|
21
|
-
source_table_id:
|
|
21
|
+
source_table_id: string;
|
|
22
22
|
/**
|
|
23
23
|
* ReplicaID which needs to be serialized in order to be queried
|
|
24
24
|
* or inserted into the DB
|
|
@@ -28,12 +28,19 @@ export type DeleteCurrentDataOptions = {
|
|
|
28
28
|
* Optionally provide the serialized source key directly
|
|
29
29
|
*/
|
|
30
30
|
serialized_source_key?: Buffer;
|
|
31
|
+
/**
|
|
32
|
+
* Streaming replication needs soft deletes, while truncating tables can use a hard delete directly.
|
|
33
|
+
*/
|
|
34
|
+
soft: boolean;
|
|
31
35
|
};
|
|
32
36
|
export type PostgresPersistedBatchOptions = RequiredOperationBatchLimits & {
|
|
33
37
|
group_id: number;
|
|
38
|
+
storageConfig: storage.StorageVersionConfig;
|
|
34
39
|
};
|
|
35
40
|
export declare class PostgresPersistedBatch {
|
|
36
41
|
group_id: number;
|
|
42
|
+
private readonly storageConfig;
|
|
43
|
+
private readonly currentDataStore;
|
|
37
44
|
/**
|
|
38
45
|
* Very rough estimate of current operations size in bytes
|
|
39
46
|
*/
|
|
@@ -45,16 +52,21 @@ export declare class PostgresPersistedBatch {
|
|
|
45
52
|
*/
|
|
46
53
|
protected bucketDataInserts: models.BucketData[];
|
|
47
54
|
protected parameterDataInserts: models.BucketParameters[];
|
|
48
|
-
protected currentDataDeletes: Pick<models.CurrentData, 'group_id' | 'source_key' | 'source_table'>[];
|
|
49
55
|
/**
|
|
50
|
-
* This is stored as a map to avoid multiple inserts (or conflicts) for the same key
|
|
56
|
+
* This is stored as a map to avoid multiple inserts (or conflicts) for the same key.
|
|
57
|
+
*
|
|
58
|
+
* Each key may only occur in one of these two maps.
|
|
51
59
|
*/
|
|
52
|
-
protected currentDataInserts: Map<string, models.
|
|
60
|
+
protected currentDataInserts: Map<string, models.V3CurrentData>;
|
|
61
|
+
protected currentDataDeletes: Map<string, {
|
|
62
|
+
source_key_hex: string;
|
|
63
|
+
source_table: string;
|
|
64
|
+
}>;
|
|
53
65
|
constructor(options: PostgresPersistedBatchOptions);
|
|
54
66
|
saveBucketData(options: SaveBucketDataOptions): void;
|
|
55
67
|
saveParameterData(options: SaveParameterDataOptions): void;
|
|
56
68
|
deleteCurrentData(options: DeleteCurrentDataOptions): void;
|
|
57
|
-
upsertCurrentData(options: models.
|
|
69
|
+
upsertCurrentData(options: models.V3CurrentDataDecoded, serialized_source_key?: Buffer): void;
|
|
58
70
|
shouldFlushTransaction(): boolean;
|
|
59
71
|
flush(db: lib_postgres.WrappedConnection): Promise<{
|
|
60
72
|
flushedAny: boolean;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { storage } from '@powersync/service-core';
|
|
3
|
+
import * as t from 'ts-codec';
|
|
4
|
+
import * as models from '../types/models/CurrentData.js';
|
|
5
|
+
type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows'>;
|
|
6
|
+
declare const TruncateCurrentDataCodec: t.ObjectCodec<Pick<{
|
|
7
|
+
buckets: t.Codec<{
|
|
8
|
+
id: string;
|
|
9
|
+
bucket: string;
|
|
10
|
+
table: string;
|
|
11
|
+
}[], string, string, t.CodecProps>;
|
|
12
|
+
data: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
13
|
+
group_id: t.Codec<number, number, "pg_number", t.CodecProps>;
|
|
14
|
+
lookups: t.ArrayCodec<t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>>;
|
|
15
|
+
source_key: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
16
|
+
source_table: t.IdentityCodec<t.CodecType.String>;
|
|
17
|
+
}, "source_key" | "buckets" | "lookups">>;
|
|
18
|
+
declare const LookupKeyCodec: t.ObjectCodec<Pick<{
|
|
19
|
+
buckets: t.Codec<{
|
|
20
|
+
id: string;
|
|
21
|
+
bucket: string;
|
|
22
|
+
table: string;
|
|
23
|
+
}[], string, string, t.CodecProps>;
|
|
24
|
+
data: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
25
|
+
group_id: t.Codec<number, number, "pg_number", t.CodecProps>;
|
|
26
|
+
lookups: t.ArrayCodec<t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>>;
|
|
27
|
+
source_key: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
28
|
+
source_table: t.IdentityCodec<t.CodecType.String>;
|
|
29
|
+
}, "source_table" | "source_key">>;
|
|
30
|
+
export type TruncateCurrentDataRow = t.Decoded<typeof TruncateCurrentDataCodec>;
|
|
31
|
+
export type CurrentDataLookupRow = t.Decoded<typeof LookupKeyCodec> | t.Decoded<typeof models.V1CurrentData> | t.Decoded<typeof models.V3CurrentData>;
|
|
32
|
+
export declare const V1_CURRENT_DATA_TABLE = "current_data";
|
|
33
|
+
export declare const V3_CURRENT_DATA_TABLE = "v3_current_data";
|
|
34
|
+
export declare class PostgresCurrentDataStore {
|
|
35
|
+
readonly table: string;
|
|
36
|
+
readonly softDeleteEnabled: boolean;
|
|
37
|
+
constructor(storageConfig: storage.StorageVersionConfig);
|
|
38
|
+
streamTruncateRows(db: Queryable, options: {
|
|
39
|
+
groupId: number;
|
|
40
|
+
sourceTableId: string;
|
|
41
|
+
limit: number;
|
|
42
|
+
}): AsyncIterableIterator<{
|
|
43
|
+
source_key: string;
|
|
44
|
+
buckets: string;
|
|
45
|
+
lookups: string[];
|
|
46
|
+
}[]>;
|
|
47
|
+
decodeTruncateRow(row: t.Encoded<typeof TruncateCurrentDataCodec>): TruncateCurrentDataRow;
|
|
48
|
+
streamSizeRows(db: Queryable, options: {
|
|
49
|
+
groupId: number;
|
|
50
|
+
lookups: {
|
|
51
|
+
source_table: string;
|
|
52
|
+
source_key: string;
|
|
53
|
+
}[];
|
|
54
|
+
}): AsyncIterableIterator<{
|
|
55
|
+
source_table: string;
|
|
56
|
+
source_key: storage.ReplicaId;
|
|
57
|
+
data_size: number;
|
|
58
|
+
}[]>;
|
|
59
|
+
streamLookupRows(db: Queryable, options: {
|
|
60
|
+
groupId: number;
|
|
61
|
+
lookups: {
|
|
62
|
+
source_table: string;
|
|
63
|
+
source_key: string;
|
|
64
|
+
}[];
|
|
65
|
+
skipExistingRows: boolean;
|
|
66
|
+
}): AsyncIterableIterator<any[]>;
|
|
67
|
+
decodeLookupRow(row: any, skipExistingRows: boolean): CurrentDataLookupRow;
|
|
68
|
+
flushUpserts(db: Queryable, updates: models.V3CurrentData[]): Promise<void>;
|
|
69
|
+
flushDeletes(db: Queryable, options: {
|
|
70
|
+
groupId: number;
|
|
71
|
+
deletes: {
|
|
72
|
+
source_key_hex: string;
|
|
73
|
+
source_table: string;
|
|
74
|
+
}[];
|
|
75
|
+
}): Promise<void>;
|
|
76
|
+
cleanupPendingDeletes(db: Queryable, options: {
|
|
77
|
+
groupId: number;
|
|
78
|
+
lastCheckpoint: bigint;
|
|
79
|
+
}): Promise<void>;
|
|
80
|
+
deleteGroupRows(db: Queryable, options: {
|
|
81
|
+
groupId: number;
|
|
82
|
+
}): Promise<void>;
|
|
83
|
+
private wherePendingDelete;
|
|
84
|
+
}
|
|
85
|
+
export {};
|
|
@@ -6,7 +6,7 @@ export declare const CurrentBucket: t.ObjectCodec<{
|
|
|
6
6
|
}>;
|
|
7
7
|
export type CurrentBucket = t.Encoded<typeof CurrentBucket>;
|
|
8
8
|
export type CurrentBucketDecoded = t.Decoded<typeof CurrentBucket>;
|
|
9
|
-
export declare const
|
|
9
|
+
export declare const V1CurrentData: t.ObjectCodec<{
|
|
10
10
|
buckets: t.Codec<{
|
|
11
11
|
id: string;
|
|
12
12
|
bucket: string;
|
|
@@ -18,5 +18,20 @@ export declare const CurrentData: t.ObjectCodec<{
|
|
|
18
18
|
source_key: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
19
19
|
source_table: t.IdentityCodec<t.CodecType.String>;
|
|
20
20
|
}>;
|
|
21
|
-
export
|
|
22
|
-
|
|
21
|
+
export declare const V3CurrentData: t.ObjectCodec<{
|
|
22
|
+
buckets: t.Codec<{
|
|
23
|
+
id: string;
|
|
24
|
+
bucket: string;
|
|
25
|
+
table: string;
|
|
26
|
+
}[], string, string, t.CodecProps>;
|
|
27
|
+
data: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
28
|
+
group_id: t.Codec<number, number, "pg_number", t.CodecProps>;
|
|
29
|
+
lookups: t.ArrayCodec<t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>>;
|
|
30
|
+
source_key: t.Codec<Buffer<ArrayBuffer>, string, "hexBuffer", t.CodecProps>;
|
|
31
|
+
source_table: t.IdentityCodec<t.CodecType.String>;
|
|
32
|
+
pending_delete: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<bigint, string | number, string, t.CodecProps>>;
|
|
33
|
+
}>;
|
|
34
|
+
export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
|
|
35
|
+
export type V1CurrentDataDecoded = t.Decoded<typeof V1CurrentData>;
|
|
36
|
+
export type V3CurrentData = t.Encoded<typeof V3CurrentData>;
|
|
37
|
+
export type V3CurrentDataDecoded = t.Decoded<typeof V3CurrentData>;
|
|
@@ -55,6 +55,15 @@ export declare const SyncRules: t.ObjectCodec<{
|
|
|
55
55
|
maxTimeValuePrecision?: number | undefined;
|
|
56
56
|
};
|
|
57
57
|
eventDescriptors: Record<string, string[]>;
|
|
58
|
+
errors?: {
|
|
59
|
+
level: "warning" | "fatal";
|
|
60
|
+
message: string;
|
|
61
|
+
location?: {
|
|
62
|
+
start_offset: number;
|
|
63
|
+
end_offset: number;
|
|
64
|
+
} | undefined;
|
|
65
|
+
ts?: string | undefined;
|
|
66
|
+
}[] | undefined;
|
|
58
67
|
}, import("@powersync/service-jsonbig").JsonContainer>>;
|
|
59
68
|
}>;
|
|
60
69
|
export type SyncRules = t.Encoded<typeof SyncRules>;
|
|
@@ -5,7 +5,7 @@ export * from './BucketParameters.js';
|
|
|
5
5
|
export * from './CurrentData.js';
|
|
6
6
|
export * from './Instance.js';
|
|
7
7
|
export * from './Migration.js';
|
|
8
|
+
export * from './SdkReporting.js';
|
|
8
9
|
export * from './SourceTable.js';
|
|
9
10
|
export * from './SyncRules.js';
|
|
10
11
|
export * from './WriteCheckpoint.js';
|
|
11
|
-
export * from './SdkReporting.js';
|
|
@@ -3,4 +3,4 @@ import { storage } from '@powersync/service-core';
|
|
|
3
3
|
* BSON is used to serialize certain documents for storage in BYTEA columns.
|
|
4
4
|
* JSONB columns do not directly support storing binary data which could be required in future.
|
|
5
5
|
*/
|
|
6
|
-
export declare function replicaIdToSubkey(tableId:
|
|
6
|
+
export declare function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { framework, TestStorageOptions } from '@powersync/service-core';
|
|
2
2
|
import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
|
|
3
|
-
import { PostgresStorageConfigDecoded } from '../types/types.js';
|
|
4
|
-
import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
|
|
5
3
|
import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
|
|
4
|
+
import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
|
|
5
|
+
import { PostgresStorageConfigDecoded } from '../types/types.js';
|
|
6
6
|
export type PostgresTestStorageOptions = {
|
|
7
7
|
url: string;
|
|
8
8
|
/**
|
|
@@ -15,5 +15,5 @@ export declare function postgresTestSetup(factoryOptions: PostgresTestStorageOpt
|
|
|
15
15
|
reportFactory: (options?: TestStorageOptions) => Promise<PostgresReportStorage>;
|
|
16
16
|
factory: (options?: TestStorageOptions) => Promise<PostgresBucketStorageFactory>;
|
|
17
17
|
migrate: (direction: framework.migrations.Direction) => Promise<void>;
|
|
18
|
+
tableIdStrings: boolean;
|
|
18
19
|
};
|
|
19
|
-
export declare function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions): (options?: TestStorageOptions) => Promise<PostgresBucketStorageFactory>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
|
|
3
|
-
import { STORAGE_SCHEMA_NAME } from '../utils/db.js';
|
|
4
2
|
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
|
|
5
4
|
import { getStorageApplicationName } from '../utils/application-name.js';
|
|
5
|
+
import { STORAGE_SCHEMA_NAME } from '../utils/db.js';
|
|
6
6
|
export const openMigrationDB = (config) => {
|
|
7
7
|
if (!isPostgresStorageConfig(config)) {
|
|
8
8
|
throw new ServiceAssertionError(`Input storage configuration is not for Postgres`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migration-utils.js","sourceRoot":"","sources":["../../src/migrations/migration-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"migration-utils.js","sourceRoot":"","sources":["../../src/migrations/migration-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AACnH,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAoC,EAAE,EAAE;IACtE,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,qBAAqB,CAAC,iDAAiD,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,IAAI,YAAY,CAAC,cAAc,CAAC;QACrC,MAAM,EAAE,8BAA8B,CAAC,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5E,MAAM,EAAE,mBAAmB;QAC3B,eAAe,EAAE,yBAAyB,EAAE;KAC7C,CAAC,CAAC;AACL,CAAC,CAAC"}
|