@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -4
- package/dist/api/RouteAPI.d.ts +42 -14
- package/dist/api/diagnostics.d.ts +12 -1
- package/dist/api/diagnostics.js +34 -22
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +5 -1
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +6 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +26 -7
- package/dist/replication/AbstractReplicator.js +148 -68
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +3 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +1 -0
- package/dist/routes/RouterEngine.js +6 -1
- package/dist/routes/RouterEngine.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +95 -56
- package/dist/routes/configure-fastify.js +15 -2
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +24 -24
- package/dist/routes/endpoints/admin.js +46 -35
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +62 -0
- package/dist/routes/endpoints/checkpointing.js +65 -7
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +3 -2
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +22 -13
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +1 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +8 -8
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +63 -17
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +22 -27
- package/dist/storage/BucketStorageFactory.js +4 -18
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
- package/dist/storage/ChecksumCache.d.ts +11 -6
- package/dist/storage/ChecksumCache.js +5 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
- package/dist/storage/ParsedSyncConfigSet.js +2 -0
- package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/PersistedReplicationStream.d.ts +32 -0
- package/dist/storage/PersistedReplicationStream.js +26 -0
- package/dist/storage/PersistedReplicationStream.js.map +1 -0
- package/dist/storage/PersistedSyncConfigContent.d.ts +63 -0
- package/dist/storage/PersistedSyncConfigContent.js +125 -0
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
- package/dist/storage/PersistedSyncConfigStatus.js +2 -0
- package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
- package/dist/storage/SourceEntity.d.ts +6 -0
- package/dist/storage/SourceTable.d.ts +21 -3
- package/dist/storage/SourceTable.js +26 -3
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +119 -21
- package/dist/storage/SyncRulesBucketStorage.js +3 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +60 -3
- package/dist/storage/WriteCheckpointAPI.js +33 -0
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +117 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js +277 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +62 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
- package/dist/storage/storage-index.d.ts +9 -1
- package/dist/storage/storage-index.js +9 -1
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +12 -2
- package/dist/sync/BucketChecksumState.js +69 -21
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +7 -0
- package/dist/sync/RequestTracker.js +12 -0
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync.js +154 -41
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +5 -0
- package/dist/sync/util.js +9 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +2 -0
- package/dist/system/ServiceContext.js +2 -0
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +23 -1
- package/dist/tracing/PerformanceTracer.js +11 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -1
- package/dist/util/checkpointing.d.ts +4 -8
- package/dist/util/checkpointing.js +1 -16
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +18 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/defaults.d.ts +2 -0
- package/dist/util/config/defaults.js +2 -0
- package/dist/util/config/defaults.js.map +1 -1
- package/dist/util/config/types.d.ts +3 -1
- package/dist/util/lsn.d.ts +4 -0
- package/dist/util/lsn.js +10 -0
- package/dist/util/lsn.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/dist/util/utils.js +1 -1
- package/dist/util/write-checkpoint-batcher.d.ts +19 -0
- package/dist/util/write-checkpoint-batcher.js +100 -0
- package/dist/util/write-checkpoint-batcher.js.map +1 -0
- package/package.json +6 -7
- package/src/api/RouteAPI.ts +43 -14
- package/src/api/diagnostics.ts +43 -26
- package/src/entry/commands/compact-action.ts +7 -1
- package/src/modules/AbstractModule.ts +6 -2
- package/src/replication/AbstractReplicator.ts +181 -72
- package/src/replication/ReplicationModule.ts +2 -3
- package/src/routes/RouterEngine.ts +7 -1
- package/src/routes/configure-fastify.ts +25 -2
- package/src/routes/endpoints/admin.ts +72 -50
- package/src/routes/endpoints/checkpointing.ts +79 -8
- package/src/routes/endpoints/socket-route.ts +3 -2
- package/src/routes/endpoints/sync-rules.ts +23 -13
- package/src/routes/endpoints/sync-stream.ts +1 -1
- package/src/runner/teardown.ts +8 -8
- package/src/storage/BucketStorageBatch.ts +64 -18
- package/src/storage/BucketStorageFactory.ts +30 -39
- package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
- package/src/storage/ChecksumCache.ts +27 -14
- package/src/storage/ParsedSyncConfigSet.ts +25 -0
- package/src/storage/PersistedReplicationStream.ts +55 -0
- package/src/storage/PersistedSyncConfigContent.ts +192 -0
- package/src/storage/PersistedSyncConfigStatus.ts +14 -0
- package/src/storage/SourceEntity.ts +6 -0
- package/src/storage/SourceTable.ts +34 -3
- package/src/storage/SyncRulesBucketStorage.ts +138 -22
- package/src/storage/WriteCheckpointAPI.ts +108 -3
- package/src/storage/implementation/BucketDefinitionMapping.ts +436 -0
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +109 -0
- package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
- package/src/storage/storage-index.ts +10 -1
- package/src/sync/BucketChecksumState.ts +103 -21
- package/src/sync/RequestTracker.ts +20 -0
- package/src/sync/sync.ts +183 -48
- package/src/sync/util.ts +12 -0
- package/src/system/ServiceContext.ts +7 -0
- package/src/tracing/PerformanceTracer.ts +41 -1
- package/src/util/checkpointing.ts +4 -22
- package/src/util/config/compound-config-collector.ts +25 -1
- package/src/util/config/defaults.ts +2 -0
- package/src/util/config/types.ts +3 -1
- package/src/util/lsn.ts +9 -0
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +1 -1
- package/src/util/write-checkpoint-batcher.ts +142 -0
- package/test/src/AbstractReplicator.test.ts +115 -0
- package/test/src/checksum_cache.test.ts +22 -0
- package/test/src/config.test.ts +102 -0
- package/test/src/diagnostics.test.ts +62 -34
- package/test/src/module-loader.test.ts +33 -1
- package/test/src/routes/admin.test.ts +182 -18
- package/test/src/routes/checkpointing.test.ts +54 -0
- package/test/src/routes/mocks.ts +6 -2
- package/test/src/storage/SourceTable.test.ts +89 -0
- package/test/src/sync/BucketChecksumState.test.ts +98 -12
- package/test/src/sync/util.test.ts +13 -1
- package/test/src/util/checkpointing.test.ts +249 -0
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
- package/dist/storage/PersistedSyncRulesContent.js +0 -100
- package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
- package/src/storage/PersistedSyncRulesContent.ts +0 -174
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { SyncConfigWithErrors } from '@powersync/service-sync-rules';
|
|
2
|
+
import { Logger } from 'winston';
|
|
3
|
+
import { SyncRuleState } from './BucketStorage.js';
|
|
4
|
+
import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
|
|
5
|
+
import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
|
|
6
|
+
import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
|
|
7
|
+
import { StorageVersionConfig } from './StorageVersionConfig.js';
|
|
8
|
+
export interface ParsePersistedSyncConfigContentOptions {
|
|
9
|
+
content: string;
|
|
10
|
+
compiledPlan: SerializedSyncPlan | null;
|
|
11
|
+
storageVersion: number;
|
|
12
|
+
parseOptions: ParseSyncConfigOptions;
|
|
13
|
+
}
|
|
14
|
+
export declare function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfigContentOptions): SyncConfigWithErrors;
|
|
15
|
+
/**
|
|
16
|
+
* Immutable sync config content for one sync config inside a replication stream.
|
|
17
|
+
*
|
|
18
|
+
* This represents the persisted config content. Fetch per-config status with
|
|
19
|
+
* {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
|
|
20
|
+
* lifecycle concerns such as locking and termination.
|
|
21
|
+
*/
|
|
22
|
+
export declare abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData {
|
|
23
|
+
readonly replicationStreamId: number;
|
|
24
|
+
readonly sync_rules_content: string;
|
|
25
|
+
readonly compiled_plan: SerializedSyncPlan | null;
|
|
26
|
+
readonly replicationStreamName: string;
|
|
27
|
+
readonly storageVersion: number;
|
|
28
|
+
readonly logger: Logger;
|
|
29
|
+
readonly syncConfigId: PersistedSyncConfigId | null;
|
|
30
|
+
readonly syncConfigState: SyncRuleState;
|
|
31
|
+
constructor(data: PersistedSyncConfigContentData);
|
|
32
|
+
/**
|
|
33
|
+
* Load the storage config.
|
|
34
|
+
*
|
|
35
|
+
* This may throw if the persisted storage version is not supported.
|
|
36
|
+
*/
|
|
37
|
+
getStorageConfig(): StorageVersionConfig;
|
|
38
|
+
/**
|
|
39
|
+
* Parse only this config's content into a single {@link SyncConfigWithErrors}.
|
|
40
|
+
*
|
|
41
|
+
* This does not depend on any other configs in the same replication stream.
|
|
42
|
+
*/
|
|
43
|
+
protected parseSingleConfig(options: ParseSyncConfigOptions): SyncConfigWithErrors;
|
|
44
|
+
parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
|
|
45
|
+
asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions;
|
|
46
|
+
/**
|
|
47
|
+
* Fetch the current persisted state for this exact sync config.
|
|
48
|
+
*/
|
|
49
|
+
abstract getSyncConfigStatus(): Promise<PersistedSyncConfigStatus | null>;
|
|
50
|
+
}
|
|
51
|
+
export interface PersistedSyncConfigContentData {
|
|
52
|
+
readonly replicationStreamId: number;
|
|
53
|
+
readonly sync_rules_content: string;
|
|
54
|
+
readonly compiled_plan: SerializedSyncPlan | null;
|
|
55
|
+
readonly replicationStreamName: string;
|
|
56
|
+
readonly storageVersion: number;
|
|
57
|
+
readonly syncConfigId?: PersistedSyncConfigId | null;
|
|
58
|
+
readonly syncConfigState: SyncRuleState;
|
|
59
|
+
}
|
|
60
|
+
export type PersistedSyncConfigId = string;
|
|
61
|
+
export interface ParseSyncConfigOptions {
|
|
62
|
+
defaultSchema: string;
|
|
63
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { CompatibilityContext, CompatibilityOption, DEFAULT_HYDRATION_STATE, deserializeSyncPlan, nodeSqlite, PrecompiledSyncConfig, SqlEventDescriptor, SqlSyncRules, versionedHydrationState, YamlError } from '@powersync/service-sync-rules';
|
|
3
|
+
import * as sqlite from 'node:sqlite';
|
|
4
|
+
import { STORAGE_VERSION_CONFIG } from './StorageVersionConfig.js';
|
|
5
|
+
export function parsePersistedSyncConfigContent(options) {
|
|
6
|
+
const { content, compiledPlan, storageVersion, parseOptions } = options;
|
|
7
|
+
if (compiledPlan == null) {
|
|
8
|
+
// Fallback: Only parse from YAML if no compiled plan is available.
|
|
9
|
+
return SqlSyncRules.fromYaml(content, parseOptions);
|
|
10
|
+
}
|
|
11
|
+
const plan = deserializeSyncPlan(compiledPlan.plan);
|
|
12
|
+
const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility);
|
|
13
|
+
const eventDefinitions = [];
|
|
14
|
+
for (const [name, queries] of Object.entries(compiledPlan.eventDescriptors)) {
|
|
15
|
+
const descriptor = new SqlEventDescriptor(name, compatibility);
|
|
16
|
+
for (const query of queries) {
|
|
17
|
+
descriptor.addSourceQuery(query, parseOptions);
|
|
18
|
+
}
|
|
19
|
+
eventDefinitions.push(descriptor);
|
|
20
|
+
}
|
|
21
|
+
const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
|
|
22
|
+
defaultSchema: parseOptions.defaultSchema,
|
|
23
|
+
sourceText: content
|
|
24
|
+
});
|
|
25
|
+
// Note: If the original content did not define a storage version, this will still set the storage version.
|
|
26
|
+
// This means asUpdateOptions will not change the storage version, even if the default changes.
|
|
27
|
+
precompiled.storageVersion = storageVersion;
|
|
28
|
+
const errors = [];
|
|
29
|
+
if (compiledPlan.errors) {
|
|
30
|
+
for (const error of compiledPlan.errors) {
|
|
31
|
+
const location = error.location && {
|
|
32
|
+
start: error.location.start_offset,
|
|
33
|
+
end: error.location.end_offset
|
|
34
|
+
};
|
|
35
|
+
const asYamlError = new YamlError(new Error(error.message), location);
|
|
36
|
+
asYamlError.type = error.level;
|
|
37
|
+
errors.push(asYamlError);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return { config: precompiled, errors };
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Immutable sync config content for one sync config inside a replication stream.
|
|
44
|
+
*
|
|
45
|
+
* This represents the persisted config content. Fetch per-config status with
|
|
46
|
+
* {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
|
|
47
|
+
* lifecycle concerns such as locking and termination.
|
|
48
|
+
*/
|
|
49
|
+
export class PersistedSyncConfigContent {
|
|
50
|
+
replicationStreamId;
|
|
51
|
+
sync_rules_content;
|
|
52
|
+
compiled_plan;
|
|
53
|
+
replicationStreamName;
|
|
54
|
+
storageVersion;
|
|
55
|
+
logger;
|
|
56
|
+
syncConfigId;
|
|
57
|
+
syncConfigState;
|
|
58
|
+
constructor(data) {
|
|
59
|
+
this.replicationStreamId = data.replicationStreamId;
|
|
60
|
+
this.sync_rules_content = data.sync_rules_content;
|
|
61
|
+
this.compiled_plan = data.compiled_plan;
|
|
62
|
+
this.replicationStreamName = data.replicationStreamName;
|
|
63
|
+
this.storageVersion = data.storageVersion;
|
|
64
|
+
this.syncConfigId = data.syncConfigId ?? null;
|
|
65
|
+
this.syncConfigState = data.syncConfigState;
|
|
66
|
+
this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Load the storage config.
|
|
70
|
+
*
|
|
71
|
+
* This may throw if the persisted storage version is not supported.
|
|
72
|
+
*/
|
|
73
|
+
getStorageConfig() {
|
|
74
|
+
const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
|
|
75
|
+
if (storageConfig == null) {
|
|
76
|
+
throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`);
|
|
77
|
+
}
|
|
78
|
+
return storageConfig;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Parse only this config's content into a single {@link SyncConfigWithErrors}.
|
|
82
|
+
*
|
|
83
|
+
* This does not depend on any other configs in the same replication stream.
|
|
84
|
+
*/
|
|
85
|
+
parseSingleConfig(options) {
|
|
86
|
+
return parsePersistedSyncConfigContent({
|
|
87
|
+
content: this.sync_rules_content,
|
|
88
|
+
compiledPlan: this.compiled_plan,
|
|
89
|
+
storageVersion: this.storageVersion,
|
|
90
|
+
parseOptions: options
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
parsed(options) {
|
|
94
|
+
let hydrationState;
|
|
95
|
+
const config = this.parseSingleConfig(options);
|
|
96
|
+
const storageConfig = this.getStorageConfig();
|
|
97
|
+
if (storageConfig.versionedBuckets ||
|
|
98
|
+
config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)) {
|
|
99
|
+
hydrationState = versionedHydrationState(this.replicationStreamId);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
hydrationState = DEFAULT_HYDRATION_STATE;
|
|
103
|
+
}
|
|
104
|
+
let hydrated;
|
|
105
|
+
return {
|
|
106
|
+
replicationStreamId: this.replicationStreamId,
|
|
107
|
+
replicationStreamName: this.replicationStreamName,
|
|
108
|
+
syncConfigs: [config],
|
|
109
|
+
hydrationState,
|
|
110
|
+
get hydratedSyncConfig() {
|
|
111
|
+
hydrated ??= config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
|
|
112
|
+
return hydrated;
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
asUpdateOptions(options) {
|
|
117
|
+
// defaultSchema is not relevant for the parsed version here
|
|
118
|
+
const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' });
|
|
119
|
+
return {
|
|
120
|
+
config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed },
|
|
121
|
+
...options
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=PersistedSyncConfigContent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PersistedSyncConfigContent.js","sourceRoot":"","sources":["../../src/storage/PersistedSyncConfigContent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACrG,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EAInB,UAAU,EACV,qBAAqB,EACrB,kBAAkB,EAClB,YAAY,EAEZ,uBAAuB,EACvB,SAAS,EACV,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAMtC,OAAO,EAAE,sBAAsB,EAAwB,MAAM,2BAA2B,CAAC;AASzF,MAAM,UAAU,+BAA+B,CAAC,OAA+C;IAC7F,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAExE,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;QACzB,mEAAmE;QACnE,OAAO,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,oBAAoB,CAAC,WAAW,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;IACnF,MAAM,gBAAgB,GAAyB,EAAE,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC/D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,UAAU,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACjD,CAAC;QAED,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,qBAAqB,CAAC,IAAI,EAAE,aAAa,EAAE,gBAAgB,EAAE;QACnF,aAAa,EAAE,YAAY,CAAC,aAAa;QACzC,UAAU,EAAE,OAAO;KACpB,CAAC,CAAC;IAEH,2GAA2G;IAC3G,+FAA+F;IAC/F,WAAW,CAAC,cAAc,GAAG,cAAc,CAAC;IAE5C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,QAAQ,GAA8B,KAAK,CAAC,QAAQ,IAAI;gBAC5D,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;gBAClC,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;aAC/B,CAAC;YACF,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;YACtE,WAAW,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;YAE/B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AAEH,MAAM,OAAgB,0BAA0B;IACrC,mBAAmB,CAAS;IAC5B,kBAAkB,CAAS;IAC3B,aAAa,CAA4B;IACzC,qBAAqB,CAAS;IAC9B,cAAc,CAAS;IACvB,MAAM,CAAS;IACf,YAAY,CAA+B;IAC3C,eAAe,CAAgB;IAExC,YAAY,IAAoC;QAC9C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACpD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC;QAClD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACpF,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClE,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,+BAA+B,IAAI,CAAC,cAAc,2BAA2B,IAAI,CAAC,mBAAmB,EAAE,CACxG,CAAC;QACJ,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACO,iBAAiB,CAAC,OAA+B;QACzD,OAAO,+BAA+B,CAAC;YACrC,OAAO,EAAE,IAAI,CAAC,kBAAkB;YAChC,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,YAAY,EAAE,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,OAA+B;QACpC,IAAI,cAA8B,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IACE,aAAa,CAAC,gBAAgB;YAC9B,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,EAC7E,CAAC;YACD,cAAc,GAAG,uBAAuB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,uBAAuB,CAAC;QAC3C,CAAC;QAED,IAAI,QAAwC,CAAC;QAC7C,OAAO;YACL,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,WAAW,EAAE,CAAC,MAAM,CAAC;YACrB,cAAc;YACd,IAAI,kBAAkB;gBACpB,QAAQ,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACnF,OAAO,QAAQ,CAAC;YAClB,CAAC;SACF,CAAC;IACJ,CAAC;IAED,eAAe,CAAC,OAAgD;QAC9D,4DAA4D;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC3E,OAAO;YACL,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE;YAC3E,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;CAMF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { SyncRuleState } from './BucketStorage.js';
|
|
2
|
+
import { PersistedSyncConfigId } from './PersistedSyncConfigContent.js';
|
|
3
|
+
export interface PersistedSyncConfigStatus {
|
|
4
|
+
readonly id: PersistedSyncConfigId;
|
|
5
|
+
readonly replicationStreamId: number;
|
|
6
|
+
readonly state: SyncRuleState;
|
|
7
|
+
readonly snapshot_done?: boolean;
|
|
8
|
+
readonly last_checkpoint_lsn: string | null;
|
|
9
|
+
readonly last_fatal_error?: string | null;
|
|
10
|
+
readonly last_fatal_error_ts?: Date | null;
|
|
11
|
+
readonly last_keepalive_ts?: Date | null;
|
|
12
|
+
readonly last_checkpoint_ts?: Date | null;
|
|
13
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PersistedSyncConfigStatus.js","sourceRoot":"","sources":["../../src/storage/PersistedSyncConfigStatus.ts"],"names":[],"mappings":""}
|
|
@@ -23,4 +23,10 @@ export interface SourceEntityDescriptor extends SourceTableRef {
|
|
|
23
23
|
* The columns that are used to uniquely identify a record in the source entity.
|
|
24
24
|
*/
|
|
25
25
|
replicaIdColumns: ColumnDescriptor[];
|
|
26
|
+
/**
|
|
27
|
+
* Whether the source always sends complete row data with each operation (e.g. Postgres REPLICA
|
|
28
|
+
* IDENTITY FULL). When true, no current_data copy is needed. Undefined means the source does not
|
|
29
|
+
* report this, in which case we default to keeping a copy.
|
|
30
|
+
*/
|
|
31
|
+
sendsCompleteRows?: boolean;
|
|
26
32
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BucketDataSource, ParameterIndexLookupCreator, SourceTableRef } from '@powersync/service-sync-rules';
|
|
1
|
+
import { BucketDataSource, BucketDefinitionId, ParameterIndexId, ParameterIndexLookupCreator, SourceTableRef } from '@powersync/service-sync-rules';
|
|
2
2
|
import { bson } from '../index.js';
|
|
3
3
|
import { ColumnDescriptor } from './SourceEntity.js';
|
|
4
4
|
/**
|
|
@@ -13,6 +13,8 @@ export interface SourceTableOptions {
|
|
|
13
13
|
snapshotComplete: boolean;
|
|
14
14
|
bucketDataSources: BucketDataSource[];
|
|
15
15
|
parameterLookupSources: ParameterIndexLookupCreator[];
|
|
16
|
+
bucketDataSourceIds?: Set<BucketDefinitionId>;
|
|
17
|
+
parameterLookupSourceIds?: Set<ParameterIndexId>;
|
|
16
18
|
}
|
|
17
19
|
export interface TableSnapshotStatus {
|
|
18
20
|
totalEstimatedCount: number;
|
|
@@ -45,13 +47,27 @@ export declare class SourceTable {
|
|
|
45
47
|
*/
|
|
46
48
|
syncParameters: boolean;
|
|
47
49
|
/**
|
|
48
|
-
* True if
|
|
50
|
+
* True if this table should fire events for row changes.
|
|
49
51
|
*
|
|
50
|
-
* This value is resolved externally, and cached here.
|
|
52
|
+
* This value is resolved externally, and cached here. When multiple SourceTables exist
|
|
53
|
+
* for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
|
|
54
|
+
* as the event carrier, so that a row change saved once per table fires each event once.
|
|
51
55
|
*
|
|
52
56
|
* Defaults to true for tests.
|
|
53
57
|
*/
|
|
54
58
|
syncEvent: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* True if raw data should be stored in current_data collection.
|
|
61
|
+
*
|
|
62
|
+
* This is needed when the source sends partial row data (e.g. TOAST values).
|
|
63
|
+
* When REPLICA IDENTITY FULL is configured, complete rows are always sent,
|
|
64
|
+
* so we don't need to store raw data.
|
|
65
|
+
*
|
|
66
|
+
* This value is resolved externally based on table configuration.
|
|
67
|
+
*
|
|
68
|
+
* Defaults to true for tests (conservative approach).
|
|
69
|
+
*/
|
|
70
|
+
storeCurrentData: boolean;
|
|
55
71
|
/**
|
|
56
72
|
* Always undefined if snapshotComplete = true.
|
|
57
73
|
*
|
|
@@ -68,6 +84,8 @@ export declare class SourceTable {
|
|
|
68
84
|
get replicaIdColumns(): ColumnDescriptor[];
|
|
69
85
|
get bucketDataSources(): BucketDataSource[];
|
|
70
86
|
get parameterLookupSources(): ParameterIndexLookupCreator[];
|
|
87
|
+
get bucketDataSourceIds(): Set<string> | undefined;
|
|
88
|
+
get parameterLookupSourceIds(): Set<string> | undefined;
|
|
71
89
|
/**
|
|
72
90
|
* Sanitized name of the entity in the format of "{schema}.{entity name}".
|
|
73
91
|
* Suitable for safe use in Postgres queries.
|
|
@@ -26,13 +26,27 @@ export class SourceTable {
|
|
|
26
26
|
*/
|
|
27
27
|
syncParameters = true;
|
|
28
28
|
/**
|
|
29
|
-
* True if
|
|
29
|
+
* True if this table should fire events for row changes.
|
|
30
30
|
*
|
|
31
|
-
* This value is resolved externally, and cached here.
|
|
31
|
+
* This value is resolved externally, and cached here. When multiple SourceTables exist
|
|
32
|
+
* for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
|
|
33
|
+
* as the event carrier, so that a row change saved once per table fires each event once.
|
|
32
34
|
*
|
|
33
35
|
* Defaults to true for tests.
|
|
34
36
|
*/
|
|
35
37
|
syncEvent = true;
|
|
38
|
+
/**
|
|
39
|
+
* True if raw data should be stored in current_data collection.
|
|
40
|
+
*
|
|
41
|
+
* This is needed when the source sends partial row data (e.g. TOAST values).
|
|
42
|
+
* When REPLICA IDENTITY FULL is configured, complete rows are always sent,
|
|
43
|
+
* so we don't need to store raw data.
|
|
44
|
+
*
|
|
45
|
+
* This value is resolved externally based on table configuration.
|
|
46
|
+
*
|
|
47
|
+
* Defaults to true for tests (conservative approach).
|
|
48
|
+
*/
|
|
49
|
+
storeCurrentData = true;
|
|
36
50
|
/**
|
|
37
51
|
* Always undefined if snapshotComplete = true.
|
|
38
52
|
*
|
|
@@ -68,6 +82,12 @@ export class SourceTable {
|
|
|
68
82
|
get parameterLookupSources() {
|
|
69
83
|
return this.options.parameterLookupSources;
|
|
70
84
|
}
|
|
85
|
+
get bucketDataSourceIds() {
|
|
86
|
+
return this.options.bucketDataSourceIds;
|
|
87
|
+
}
|
|
88
|
+
get parameterLookupSourceIds() {
|
|
89
|
+
return this.options.parameterLookupSourceIds;
|
|
90
|
+
}
|
|
71
91
|
/**
|
|
72
92
|
* Sanitized name of the entity in the format of "{schema}.{entity name}".
|
|
73
93
|
* Suitable for safe use in Postgres queries.
|
|
@@ -89,11 +109,14 @@ export class SourceTable {
|
|
|
89
109
|
replicaIdColumns: this.replicaIdColumns,
|
|
90
110
|
snapshotComplete: this.snapshotComplete,
|
|
91
111
|
bucketDataSources: this.bucketDataSources,
|
|
92
|
-
parameterLookupSources: this.parameterLookupSources
|
|
112
|
+
parameterLookupSources: this.parameterLookupSources,
|
|
113
|
+
bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
|
|
114
|
+
parameterLookupSourceIds: this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
|
|
93
115
|
});
|
|
94
116
|
copy.syncData = this.syncData;
|
|
95
117
|
copy.syncParameters = this.syncParameters;
|
|
96
118
|
copy.syncEvent = this.syncEvent;
|
|
119
|
+
copy.storeCurrentData = this.storeCurrentData;
|
|
97
120
|
copy.snapshotStatus = this.snapshotStatus;
|
|
98
121
|
return copy;
|
|
99
122
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,WAAW,EAIZ,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AA0B9C;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IAsDM;IArD5B,MAAM,CAAU,WAAW,GAAG,WAAW,CAAC;IAE1C;;;;;;OAMG;IACI,QAAQ,GAAG,IAAI,CAAC;IAEvB;;;;;;OAMG;IACI,cAAc,GAAG,IAAI,CAAC;IAE7B;;;;;;;;OAQG;IACI,SAAS,GAAG,IAAI,CAAC;IAExB;;;;;;;;;;OAUG;IACI,gBAAgB,GAAG,IAAI,CAAC;IAE/B;;;;OAIG;IACI,cAAc,GAAoC,SAAS,CAAC;IAE5D,gBAAgB,CAAU;IAEjC,YAA4B,OAA2B;QAA3B,YAAO,GAAP,OAAO,CAAoB;QACrD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,IAAI,EAAE;QACJ,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;IACxC,CAAC;IAED,IAAI,sBAAsB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC;IAC7C,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC;IAC1C,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC;IAC/C,CAAC;IAED;;;OAGG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC;YAC3B,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC;YACrG,wBAAwB,EACtB,IAAI,CAAC,wBAAwB,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC;SAC7F,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC9C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB;QACpB,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YACzD,oBAAoB;YACpB,OAAO,GAAG,CAAC;QACb,CAAC;aAAM,IAAI,IAAI,CAAC,cAAc,CAAC,mBAAmB,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,IAAI,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,KAAK,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QAC9F,CAAC;IACH,CAAC"}
|
|
@@ -5,31 +5,54 @@ import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
|
|
|
5
5
|
import * as util from '../util/util-index.js';
|
|
6
6
|
import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
|
|
7
7
|
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
8
|
-
import {
|
|
8
|
+
import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
|
|
9
|
+
import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
|
|
9
10
|
import { SourceEntityDescriptor } from './SourceEntity.js';
|
|
10
11
|
import { SourceTable } from './SourceTable.js';
|
|
11
12
|
import { StorageVersionConfig } from './StorageVersionConfig.js';
|
|
12
13
|
import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
|
|
13
14
|
/**
|
|
14
15
|
* Storage for a specific replication stream.
|
|
16
|
+
*
|
|
17
|
+
* Replication writes source changes, source table metadata, snapshot progress,
|
|
18
|
+
* resume positions, and checkpoint boundaries through this interface. The sync
|
|
19
|
+
* API reads the same stream state as checkpoints, checksums, bucket data,
|
|
20
|
+
* parameter lookup rows, and write-checkpoint acknowledgements.
|
|
15
21
|
*/
|
|
16
22
|
export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketStorageListener>, SyncStorageWriteCheckpointAPI {
|
|
17
|
-
readonly
|
|
18
|
-
readonly
|
|
23
|
+
readonly replicationStreamId: number;
|
|
24
|
+
readonly replicationStreamName: string;
|
|
19
25
|
readonly storageConfig: StorageVersionConfig;
|
|
20
26
|
readonly factory: BucketStorageFactory;
|
|
21
27
|
readonly logger: Logger;
|
|
22
28
|
/**
|
|
23
|
-
* Create a
|
|
29
|
+
* Create a writer for a source-specific unit of replication work.
|
|
24
30
|
*
|
|
25
|
-
* The writer
|
|
31
|
+
* The writer may be used for a long-running streaming attempt, a table
|
|
32
|
+
* snapshot, a page of snapshot work, or a short setup step. Pending work
|
|
33
|
+
* that should be persisted must be flushed, committed, or kept alive at the
|
|
34
|
+
* appropriate source boundary before the writer is disposed.
|
|
26
35
|
*/
|
|
27
36
|
createWriter(options: CreateWriterOptions): Promise<BucketStorageBatch>;
|
|
28
37
|
/**
|
|
38
|
+
* Callback wrapper around `createWriter()`.
|
|
39
|
+
*
|
|
29
40
|
* @deprecated Use `createWriter()` with `await using` instead.
|
|
30
41
|
*/
|
|
31
42
|
startBatch(options: CreateWriterOptions, callback: (batch: BucketStorageBatch) => Promise<void>): Promise<FlushedResult | null>;
|
|
32
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Get the canonical parsed sync config set for the given parse options.
|
|
45
|
+
*
|
|
46
|
+
* Repeated calls with the same options return the same instance for the lifetime of
|
|
47
|
+
* this storage instance. This is the identity boundary for parsed sync config state:
|
|
48
|
+
* all operations against this storage instance must use parsed objects from sets
|
|
49
|
+
* returned here, never from an independent parse of the same content.
|
|
50
|
+
*/
|
|
51
|
+
getParsedSyncConfigSet(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
|
|
52
|
+
/**
|
|
53
|
+
* Shortcut for `getParsedSyncConfigSet(options).hydratedSyncConfig`.
|
|
54
|
+
*/
|
|
55
|
+
getParsedSyncRules(options: ParseSyncConfigOptions): HydratedSyncConfig;
|
|
33
56
|
/**
|
|
34
57
|
* Terminate the replication stream.
|
|
35
58
|
*
|
|
@@ -38,7 +61,7 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
|
|
|
38
61
|
* Must only be called on stopped replication streams.
|
|
39
62
|
*/
|
|
40
63
|
terminate(options?: TerminateOptions): Promise<void>;
|
|
41
|
-
getStatus(): Promise<
|
|
64
|
+
getStatus(): Promise<ReplicationStreamStatus>;
|
|
42
65
|
/**
|
|
43
66
|
* Clear the storage, without changing state.
|
|
44
67
|
*/
|
|
@@ -64,7 +87,7 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
|
|
|
64
87
|
*
|
|
65
88
|
* This is a best-effort optimization:
|
|
66
89
|
* 1. This may include more changes than what actually occurred.
|
|
67
|
-
* 2. This may return invalidateDataBuckets or invalidateParameterBuckets instead of
|
|
90
|
+
* 2. This may return invalidateDataBuckets or invalidateParameterBuckets instead of returning
|
|
68
91
|
* specific changes.
|
|
69
92
|
* @param options
|
|
70
93
|
*/
|
|
@@ -82,11 +105,18 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
|
|
|
82
105
|
* 1. Separate buckets.
|
|
83
106
|
* 2. Limit the size of each individual chunk according to options.batchSizeLimitBytes.
|
|
84
107
|
*
|
|
108
|
+
* The batch may not contain all data for the checkpoint, if the checkpoint is large. The caller must
|
|
109
|
+
* continue querying if either:
|
|
110
|
+
* 1. The last chunk for any bucket has has_more = true.
|
|
111
|
+
* 2. A SyncBucketDataBatchEnd is returned with hasMore = true.
|
|
112
|
+
*
|
|
113
|
+
* The first check can be skipped if a SyncBucketDataBatchEnd is returned with hasMore = false.
|
|
114
|
+
*
|
|
85
115
|
* @param checkpoint the checkpoint
|
|
86
116
|
* @param dataBuckets current bucket states
|
|
87
117
|
* @param options batch size options
|
|
88
118
|
*/
|
|
89
|
-
getBucketDataBatch(checkpoint:
|
|
119
|
+
getBucketDataBatch(checkpoint: ReplicationCheckpoint, dataBuckets: BucketDataRequest[], options?: BucketDataBatchOptions): AsyncIterable<SyncBucketDataChunk | SyncBucketDataBatchEnd>;
|
|
90
120
|
/**
|
|
91
121
|
* Compute checksums for a given list of buckets.
|
|
92
122
|
*
|
|
@@ -95,11 +125,16 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
|
|
|
95
125
|
* This may be slow, depending on the size of the buckets.
|
|
96
126
|
* The checksums are cached internally to compensate for this, but does not cover all cases.
|
|
97
127
|
*/
|
|
98
|
-
getChecksums(checkpoint:
|
|
128
|
+
getChecksums(checkpoint: ReplicationCheckpoint, buckets: BucketChecksumRequest[], options?: BucketChecksumOptions): Promise<util.ChecksumMap>;
|
|
99
129
|
/**
|
|
100
130
|
* Clear checksum cache. Primarily intended for tests.
|
|
101
131
|
*/
|
|
102
132
|
clearChecksumCache(): void;
|
|
133
|
+
/**
|
|
134
|
+
* Optional storage-provider cleanup for sync configs that have been stopped
|
|
135
|
+
* inside this replication stream.
|
|
136
|
+
*/
|
|
137
|
+
cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise<CleanupStoppedSyncConfigsResult>;
|
|
103
138
|
}
|
|
104
139
|
export interface SyncRulesBucketStorageListener {
|
|
105
140
|
batchStarted: (batch: BucketStorageBatch) => void;
|
|
@@ -113,33 +148,52 @@ export interface BucketChecksumRequest {
|
|
|
113
148
|
bucket: string;
|
|
114
149
|
source: BucketDataSource;
|
|
115
150
|
}
|
|
116
|
-
export interface
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
151
|
+
export interface BucketChecksumOptions {
|
|
152
|
+
requestHint?: BucketRequestHint;
|
|
153
|
+
}
|
|
154
|
+
export type BucketRequestHint = 'bulk' | 'incremental';
|
|
155
|
+
export interface ReplicationStreamStatus {
|
|
121
156
|
/**
|
|
122
|
-
*
|
|
157
|
+
* Source position to resume replication from.
|
|
123
158
|
*/
|
|
124
|
-
|
|
159
|
+
resumeLsn: string | null;
|
|
160
|
+
/**
|
|
161
|
+
* True if _every_ active/processing sync config has a client-visible checkpoint.
|
|
162
|
+
*/
|
|
163
|
+
snapshotDone: boolean;
|
|
125
164
|
}
|
|
126
165
|
export interface ResolveTablesOptions {
|
|
127
166
|
connection_id: number;
|
|
167
|
+
/**
|
|
168
|
+
* Source table or collection metadata discovered during snapshot or streaming.
|
|
169
|
+
*/
|
|
128
170
|
source: SourceEntityDescriptor;
|
|
129
171
|
/**
|
|
130
172
|
* For tests only - custom id generator for stable ids.
|
|
131
173
|
*/
|
|
132
174
|
idGenerator?: () => string | bson.ObjectId;
|
|
133
175
|
/**
|
|
134
|
-
* For tests only - override the sync
|
|
176
|
+
* For tests only - override the parsed sync config set used.
|
|
135
177
|
*/
|
|
136
|
-
|
|
178
|
+
parsedSyncConfig?: ParsedSyncConfigSet;
|
|
137
179
|
}
|
|
138
180
|
export interface ResolveTablesResult {
|
|
181
|
+
/**
|
|
182
|
+
* Current source table mappings that should receive replicated data.
|
|
183
|
+
*/
|
|
139
184
|
tables: SourceTable[];
|
|
185
|
+
/**
|
|
186
|
+
* Outdated source table mappings that should be removed by the connector.
|
|
187
|
+
*/
|
|
140
188
|
dropTables: SourceTable[];
|
|
141
189
|
}
|
|
142
|
-
|
|
190
|
+
/**
|
|
191
|
+
* Options for creating a storage writer.
|
|
192
|
+
*/
|
|
193
|
+
export interface CreateWriterOptions extends ParseSyncConfigOptions {
|
|
194
|
+
/**
|
|
195
|
+
* Source-specific zero or empty replication position.
|
|
196
|
+
*/
|
|
143
197
|
zeroLSN: string;
|
|
144
198
|
/**
|
|
145
199
|
* Whether or not to store a copy of the current data.
|
|
@@ -204,12 +258,20 @@ export interface CompactOptions {
|
|
|
204
258
|
*/
|
|
205
259
|
compactBuckets?: string[];
|
|
206
260
|
compactParameterData?: boolean;
|
|
261
|
+
/**
|
|
262
|
+
* Delete client-requested write checkpoints created before this time.
|
|
263
|
+
*
|
|
264
|
+
* Generated write checkpoints are not affected.
|
|
265
|
+
*/
|
|
266
|
+
deleteCheckpointRequestsBefore?: Date;
|
|
207
267
|
/** Minimum of 2 */
|
|
208
268
|
clearBatchLimit?: number;
|
|
209
269
|
/** Minimum of 1 */
|
|
210
270
|
moveBatchLimit?: number;
|
|
211
271
|
/** Minimum of 1 */
|
|
212
272
|
moveBatchQueryLimit?: number;
|
|
273
|
+
/** Byte cap per read batch for streaming compaction. Default 16MB. */
|
|
274
|
+
moveBatchByteLimit?: number;
|
|
213
275
|
/**
|
|
214
276
|
* Minimum number new operations in a bucket to trigger compaction of that bucket.
|
|
215
277
|
*
|
|
@@ -230,7 +292,14 @@ export interface CompactOptions {
|
|
|
230
292
|
logger?: Logger;
|
|
231
293
|
}
|
|
232
294
|
export interface PopulateChecksumCacheOptions {
|
|
233
|
-
|
|
295
|
+
/**
|
|
296
|
+
* Compute checksums up to this op id.
|
|
297
|
+
*
|
|
298
|
+
* Defaults to the highest persisted op id for the replication stream, which covers
|
|
299
|
+
* the common case of populating the cache right after initial replication, before
|
|
300
|
+
* the first checkpoint exists.
|
|
301
|
+
*/
|
|
302
|
+
maxOpId?: util.InternalOpId;
|
|
234
303
|
minBucketChanges?: number;
|
|
235
304
|
signal?: AbortSignal;
|
|
236
305
|
}
|
|
@@ -243,6 +312,22 @@ export interface PopulateChecksumCacheResults {
|
|
|
243
312
|
export interface ClearStorageOptions {
|
|
244
313
|
signal?: AbortSignal;
|
|
245
314
|
}
|
|
315
|
+
export interface CleanupStoppedSyncConfigsOptions {
|
|
316
|
+
signal?: AbortSignal;
|
|
317
|
+
logger?: Logger;
|
|
318
|
+
defaultSchema: string;
|
|
319
|
+
sourceConnectionTag: string;
|
|
320
|
+
}
|
|
321
|
+
export interface CleanupStoppedSyncConfigsResult {
|
|
322
|
+
stoppedSyncConfigsRemoved: number;
|
|
323
|
+
bucketDataCollectionsDropped: number;
|
|
324
|
+
parameterIndexCollectionsDropped: number;
|
|
325
|
+
/** Best-effort number, not accurate if the operation takes longer than the timeout. */
|
|
326
|
+
bucketStateDocumentsDeleted: number;
|
|
327
|
+
sourceRecordCollectionsDropped: number;
|
|
328
|
+
sourceTablesUpdated: number;
|
|
329
|
+
sourceTablesDeleted: number;
|
|
330
|
+
}
|
|
246
331
|
export interface TerminateOptions extends ClearStorageOptions {
|
|
247
332
|
/**
|
|
248
333
|
* If true, also clear the storage before terminating.
|
|
@@ -250,6 +335,9 @@ export interface TerminateOptions extends ClearStorageOptions {
|
|
|
250
335
|
clearStorage: boolean;
|
|
251
336
|
}
|
|
252
337
|
export interface BucketDataBatchOptions {
|
|
338
|
+
requestHint?: BucketRequestHint;
|
|
339
|
+
/** Abort any in-progress work for this batch, including object-storage downloads. */
|
|
340
|
+
signal?: AbortSignal;
|
|
253
341
|
/** Limit number of documents returned. Defaults to 1000. */
|
|
254
342
|
limit?: number;
|
|
255
343
|
/**
|
|
@@ -267,6 +355,16 @@ export interface SyncBucketDataChunk {
|
|
|
267
355
|
chunkData: util.SyncBucketData;
|
|
268
356
|
targetOp: util.InternalOpId | null;
|
|
269
357
|
}
|
|
358
|
+
export interface SyncBucketDataBatchEnd {
|
|
359
|
+
/**
|
|
360
|
+
* True if there may be more data for this checkpoint, and the caller should continue querying.
|
|
361
|
+
*
|
|
362
|
+
* This is different from `SyncBucketDataChunk.has_more`, which is per-bucket. This is a global signal for the
|
|
363
|
+
* entire request, and may be true even if there is no returned chunk with has_more: true.
|
|
364
|
+
*/
|
|
365
|
+
hasMore: boolean;
|
|
366
|
+
}
|
|
367
|
+
export declare function isBatchEnd(chunk: SyncBucketDataChunk | SyncBucketDataBatchEnd): chunk is SyncBucketDataBatchEnd;
|
|
270
368
|
export interface ReplicationCheckpoint {
|
|
271
369
|
readonly checkpoint: util.InternalOpId;
|
|
272
370
|
readonly lsn: string | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA6bA,MAAM,UAAU,UAAU,CAAC,KAAmD;IAC5E,OAAQ,KAAgC,CAAC,OAAO,KAAK,SAAS,CAAC;AACjE,CAAC;AAiBD;;;;GAIG;AACH,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAE5C;IACA;IAFX,YACW,KAAa,EACb,SAAyC;QAElD,KAAK,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;QAHhD,UAAK,GAAL,KAAK,CAAQ;QACb,cAAS,GAAT,SAAS,CAAgC;IAGpD,CAAC;CACF;AAqDD,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,IAAI,GAAG,EAAU;IACrC,qBAAqB,EAAE,IAAI;IAC3B,uBAAuB,EAAE,IAAI,GAAG,EAAU;IAC1C,0BAA0B,EAAE,IAAI;CACjC,CAAC"}
|