@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CHANGELOG.md +83 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +34 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +5 -1
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/modules/AbstractModule.d.ts +6 -2
  9. package/dist/modules/AbstractModule.js.map +1 -1
  10. package/dist/replication/AbstractReplicator.d.ts +26 -7
  11. package/dist/replication/AbstractReplicator.js +148 -68
  12. package/dist/replication/AbstractReplicator.js.map +1 -1
  13. package/dist/replication/ReplicationModule.d.ts +3 -3
  14. package/dist/replication/ReplicationModule.js.map +1 -1
  15. package/dist/routes/RouterEngine.d.ts +1 -0
  16. package/dist/routes/RouterEngine.js +6 -1
  17. package/dist/routes/RouterEngine.js.map +1 -1
  18. package/dist/routes/configure-fastify.d.ts +95 -56
  19. package/dist/routes/configure-fastify.js +15 -2
  20. package/dist/routes/configure-fastify.js.map +1 -1
  21. package/dist/routes/endpoints/admin.d.ts +24 -24
  22. package/dist/routes/endpoints/admin.js +46 -35
  23. package/dist/routes/endpoints/admin.js.map +1 -1
  24. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  25. package/dist/routes/endpoints/checkpointing.js +65 -7
  26. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  27. package/dist/routes/endpoints/socket-route.js +3 -2
  28. package/dist/routes/endpoints/socket-route.js.map +1 -1
  29. package/dist/routes/endpoints/sync-rules.js +22 -13
  30. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  31. package/dist/routes/endpoints/sync-stream.js +1 -1
  32. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  33. package/dist/runner/teardown.js +8 -8
  34. package/dist/runner/teardown.js.map +1 -1
  35. package/dist/storage/BucketStorageBatch.d.ts +63 -17
  36. package/dist/storage/BucketStorageBatch.js.map +1 -1
  37. package/dist/storage/BucketStorageFactory.d.ts +22 -27
  38. package/dist/storage/BucketStorageFactory.js +4 -18
  39. package/dist/storage/BucketStorageFactory.js.map +1 -1
  40. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  41. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  42. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  43. package/dist/storage/ChecksumCache.d.ts +11 -6
  44. package/dist/storage/ChecksumCache.js +5 -4
  45. package/dist/storage/ChecksumCache.js.map +1 -1
  46. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  47. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  48. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  49. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  50. package/dist/storage/PersistedReplicationStream.js +26 -0
  51. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  52. package/dist/storage/PersistedSyncConfigContent.d.ts +63 -0
  53. package/dist/storage/PersistedSyncConfigContent.js +125 -0
  54. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  55. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  56. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  57. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  58. package/dist/storage/SourceEntity.d.ts +6 -0
  59. package/dist/storage/SourceTable.d.ts +21 -3
  60. package/dist/storage/SourceTable.js +26 -3
  61. package/dist/storage/SourceTable.js.map +1 -1
  62. package/dist/storage/SyncRulesBucketStorage.d.ts +119 -21
  63. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  64. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  65. package/dist/storage/WriteCheckpointAPI.d.ts +60 -3
  66. package/dist/storage/WriteCheckpointAPI.js +33 -0
  67. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  68. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +117 -0
  69. package/dist/storage/implementation/BucketDefinitionMapping.js +277 -0
  70. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  71. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  72. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  73. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  74. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  75. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +62 -0
  76. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  77. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  78. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  79. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  80. package/dist/storage/storage-index.d.ts +9 -1
  81. package/dist/storage/storage-index.js +9 -1
  82. package/dist/storage/storage-index.js.map +1 -1
  83. package/dist/sync/BucketChecksumState.d.ts +12 -2
  84. package/dist/sync/BucketChecksumState.js +69 -21
  85. package/dist/sync/BucketChecksumState.js.map +1 -1
  86. package/dist/sync/RequestTracker.d.ts +7 -0
  87. package/dist/sync/RequestTracker.js +12 -0
  88. package/dist/sync/RequestTracker.js.map +1 -1
  89. package/dist/sync/sync.js +154 -41
  90. package/dist/sync/sync.js.map +1 -1
  91. package/dist/sync/util.d.ts +5 -0
  92. package/dist/sync/util.js +9 -0
  93. package/dist/sync/util.js.map +1 -1
  94. package/dist/system/ServiceContext.d.ts +2 -0
  95. package/dist/system/ServiceContext.js +2 -0
  96. package/dist/system/ServiceContext.js.map +1 -1
  97. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  98. package/dist/tracing/PerformanceTracer.js +11 -0
  99. package/dist/tracing/PerformanceTracer.js.map +1 -1
  100. package/dist/util/checkpointing.d.ts +4 -8
  101. package/dist/util/checkpointing.js +1 -16
  102. package/dist/util/checkpointing.js.map +1 -1
  103. package/dist/util/config/compound-config-collector.js +18 -2
  104. package/dist/util/config/compound-config-collector.js.map +1 -1
  105. package/dist/util/config/defaults.d.ts +2 -0
  106. package/dist/util/config/defaults.js +2 -0
  107. package/dist/util/config/defaults.js.map +1 -1
  108. package/dist/util/config/types.d.ts +3 -1
  109. package/dist/util/lsn.d.ts +4 -0
  110. package/dist/util/lsn.js +10 -0
  111. package/dist/util/lsn.js.map +1 -1
  112. package/dist/util/util-index.d.ts +1 -0
  113. package/dist/util/util-index.js +1 -0
  114. package/dist/util/util-index.js.map +1 -1
  115. package/dist/util/utils.d.ts +1 -1
  116. package/dist/util/utils.js +1 -1
  117. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  118. package/dist/util/write-checkpoint-batcher.js +100 -0
  119. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  120. package/package.json +6 -7
  121. package/src/api/RouteAPI.ts +43 -14
  122. package/src/api/diagnostics.ts +43 -26
  123. package/src/entry/commands/compact-action.ts +7 -1
  124. package/src/modules/AbstractModule.ts +6 -2
  125. package/src/replication/AbstractReplicator.ts +181 -72
  126. package/src/replication/ReplicationModule.ts +2 -3
  127. package/src/routes/RouterEngine.ts +7 -1
  128. package/src/routes/configure-fastify.ts +25 -2
  129. package/src/routes/endpoints/admin.ts +72 -50
  130. package/src/routes/endpoints/checkpointing.ts +79 -8
  131. package/src/routes/endpoints/socket-route.ts +3 -2
  132. package/src/routes/endpoints/sync-rules.ts +23 -13
  133. package/src/routes/endpoints/sync-stream.ts +1 -1
  134. package/src/runner/teardown.ts +8 -8
  135. package/src/storage/BucketStorageBatch.ts +64 -18
  136. package/src/storage/BucketStorageFactory.ts +30 -39
  137. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  138. package/src/storage/ChecksumCache.ts +27 -14
  139. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  140. package/src/storage/PersistedReplicationStream.ts +55 -0
  141. package/src/storage/PersistedSyncConfigContent.ts +192 -0
  142. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  143. package/src/storage/SourceEntity.ts +6 -0
  144. package/src/storage/SourceTable.ts +34 -3
  145. package/src/storage/SyncRulesBucketStorage.ts +138 -22
  146. package/src/storage/WriteCheckpointAPI.ts +108 -3
  147. package/src/storage/implementation/BucketDefinitionMapping.ts +436 -0
  148. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  149. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +109 -0
  150. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  151. package/src/storage/storage-index.ts +10 -1
  152. package/src/sync/BucketChecksumState.ts +103 -21
  153. package/src/sync/RequestTracker.ts +20 -0
  154. package/src/sync/sync.ts +183 -48
  155. package/src/sync/util.ts +12 -0
  156. package/src/system/ServiceContext.ts +7 -0
  157. package/src/tracing/PerformanceTracer.ts +41 -1
  158. package/src/util/checkpointing.ts +4 -22
  159. package/src/util/config/compound-config-collector.ts +25 -1
  160. package/src/util/config/defaults.ts +2 -0
  161. package/src/util/config/types.ts +3 -1
  162. package/src/util/lsn.ts +9 -0
  163. package/src/util/util-index.ts +1 -0
  164. package/src/util/utils.ts +1 -1
  165. package/src/util/write-checkpoint-batcher.ts +142 -0
  166. package/test/src/AbstractReplicator.test.ts +115 -0
  167. package/test/src/checksum_cache.test.ts +22 -0
  168. package/test/src/config.test.ts +102 -0
  169. package/test/src/diagnostics.test.ts +62 -34
  170. package/test/src/module-loader.test.ts +33 -1
  171. package/test/src/routes/admin.test.ts +182 -18
  172. package/test/src/routes/checkpointing.test.ts +54 -0
  173. package/test/src/routes/mocks.ts +6 -2
  174. package/test/src/storage/SourceTable.test.ts +89 -0
  175. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  176. package/test/src/sync/util.test.ts +13 -1
  177. package/test/src/util/checkpointing.test.ts +249 -0
  178. package/tsconfig.json +1 -4
  179. package/tsconfig.tsbuildinfo +1 -1
  180. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  181. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  182. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  183. package/src/storage/PersistedSyncRulesContent.ts +0 -174
@@ -0,0 +1,63 @@
1
+ import { SyncConfigWithErrors } from '@powersync/service-sync-rules';
2
+ import { Logger } from 'winston';
3
+ import { SyncRuleState } from './BucketStorage.js';
4
+ import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
5
+ import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
6
+ import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
7
+ import { StorageVersionConfig } from './StorageVersionConfig.js';
8
+ export interface ParsePersistedSyncConfigContentOptions {
9
+ content: string;
10
+ compiledPlan: SerializedSyncPlan | null;
11
+ storageVersion: number;
12
+ parseOptions: ParseSyncConfigOptions;
13
+ }
14
+ export declare function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfigContentOptions): SyncConfigWithErrors;
15
+ /**
16
+ * Immutable sync config content for one sync config inside a replication stream.
17
+ *
18
+ * This represents the persisted config content. Fetch per-config status with
19
+ * {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
20
+ * lifecycle concerns such as locking and termination.
21
+ */
22
+ export declare abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData {
23
+ readonly replicationStreamId: number;
24
+ readonly sync_rules_content: string;
25
+ readonly compiled_plan: SerializedSyncPlan | null;
26
+ readonly replicationStreamName: string;
27
+ readonly storageVersion: number;
28
+ readonly logger: Logger;
29
+ readonly syncConfigId: PersistedSyncConfigId | null;
30
+ readonly syncConfigState: SyncRuleState;
31
+ constructor(data: PersistedSyncConfigContentData);
32
+ /**
33
+ * Load the storage config.
34
+ *
35
+ * This may throw if the persisted storage version is not supported.
36
+ */
37
+ getStorageConfig(): StorageVersionConfig;
38
+ /**
39
+ * Parse only this config's content into a single {@link SyncConfigWithErrors}.
40
+ *
41
+ * This does not depend on any other configs in the same replication stream.
42
+ */
43
+ protected parseSingleConfig(options: ParseSyncConfigOptions): SyncConfigWithErrors;
44
+ parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
45
+ asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions;
46
+ /**
47
+ * Fetch the current persisted state for this exact sync config.
48
+ */
49
+ abstract getSyncConfigStatus(): Promise<PersistedSyncConfigStatus | null>;
50
+ }
51
+ export interface PersistedSyncConfigContentData {
52
+ readonly replicationStreamId: number;
53
+ readonly sync_rules_content: string;
54
+ readonly compiled_plan: SerializedSyncPlan | null;
55
+ readonly replicationStreamName: string;
56
+ readonly storageVersion: number;
57
+ readonly syncConfigId?: PersistedSyncConfigId | null;
58
+ readonly syncConfigState: SyncRuleState;
59
+ }
60
+ export type PersistedSyncConfigId = string;
61
+ export interface ParseSyncConfigOptions {
62
+ defaultSchema: string;
63
+ }
@@ -0,0 +1,125 @@
1
+ import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
+ import { CompatibilityContext, CompatibilityOption, DEFAULT_HYDRATION_STATE, deserializeSyncPlan, nodeSqlite, PrecompiledSyncConfig, SqlEventDescriptor, SqlSyncRules, versionedHydrationState, YamlError } from '@powersync/service-sync-rules';
3
+ import * as sqlite from 'node:sqlite';
4
+ import { STORAGE_VERSION_CONFIG } from './StorageVersionConfig.js';
5
+ export function parsePersistedSyncConfigContent(options) {
6
+ const { content, compiledPlan, storageVersion, parseOptions } = options;
7
+ if (compiledPlan == null) {
8
+ // Fallback: Only parse from YAML if no compiled plan is available.
9
+ return SqlSyncRules.fromYaml(content, parseOptions);
10
+ }
11
+ const plan = deserializeSyncPlan(compiledPlan.plan);
12
+ const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility);
13
+ const eventDefinitions = [];
14
+ for (const [name, queries] of Object.entries(compiledPlan.eventDescriptors)) {
15
+ const descriptor = new SqlEventDescriptor(name, compatibility);
16
+ for (const query of queries) {
17
+ descriptor.addSourceQuery(query, parseOptions);
18
+ }
19
+ eventDefinitions.push(descriptor);
20
+ }
21
+ const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
22
+ defaultSchema: parseOptions.defaultSchema,
23
+ sourceText: content
24
+ });
25
+ // Note: If the original content did not define a storage version, this will still set the storage version.
26
+ // This means asUpdateOptions will not change the storage version, even if the default changes.
27
+ precompiled.storageVersion = storageVersion;
28
+ const errors = [];
29
+ if (compiledPlan.errors) {
30
+ for (const error of compiledPlan.errors) {
31
+ const location = error.location && {
32
+ start: error.location.start_offset,
33
+ end: error.location.end_offset
34
+ };
35
+ const asYamlError = new YamlError(new Error(error.message), location);
36
+ asYamlError.type = error.level;
37
+ errors.push(asYamlError);
38
+ }
39
+ }
40
+ return { config: precompiled, errors };
41
+ }
42
+ /**
43
+ * Immutable sync config content for one sync config inside a replication stream.
44
+ *
45
+ * This represents the persisted config content. Fetch per-config status with
46
+ * {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
47
+ * lifecycle concerns such as locking and termination.
48
+ */
49
+ export class PersistedSyncConfigContent {
50
+ replicationStreamId;
51
+ sync_rules_content;
52
+ compiled_plan;
53
+ replicationStreamName;
54
+ storageVersion;
55
+ logger;
56
+ syncConfigId;
57
+ syncConfigState;
58
+ constructor(data) {
59
+ this.replicationStreamId = data.replicationStreamId;
60
+ this.sync_rules_content = data.sync_rules_content;
61
+ this.compiled_plan = data.compiled_plan;
62
+ this.replicationStreamName = data.replicationStreamName;
63
+ this.storageVersion = data.storageVersion;
64
+ this.syncConfigId = data.syncConfigId ?? null;
65
+ this.syncConfigState = data.syncConfigState;
66
+ this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
67
+ }
68
+ /**
69
+ * Load the storage config.
70
+ *
71
+ * This may throw if the persisted storage version is not supported.
72
+ */
73
+ getStorageConfig() {
74
+ const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
75
+ if (storageConfig == null) {
76
+ throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`);
77
+ }
78
+ return storageConfig;
79
+ }
80
+ /**
81
+ * Parse only this config's content into a single {@link SyncConfigWithErrors}.
82
+ *
83
+ * This does not depend on any other configs in the same replication stream.
84
+ */
85
+ parseSingleConfig(options) {
86
+ return parsePersistedSyncConfigContent({
87
+ content: this.sync_rules_content,
88
+ compiledPlan: this.compiled_plan,
89
+ storageVersion: this.storageVersion,
90
+ parseOptions: options
91
+ });
92
+ }
93
+ parsed(options) {
94
+ let hydrationState;
95
+ const config = this.parseSingleConfig(options);
96
+ const storageConfig = this.getStorageConfig();
97
+ if (storageConfig.versionedBuckets ||
98
+ config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)) {
99
+ hydrationState = versionedHydrationState(this.replicationStreamId);
100
+ }
101
+ else {
102
+ hydrationState = DEFAULT_HYDRATION_STATE;
103
+ }
104
+ let hydrated;
105
+ return {
106
+ replicationStreamId: this.replicationStreamId,
107
+ replicationStreamName: this.replicationStreamName,
108
+ syncConfigs: [config],
109
+ hydrationState,
110
+ get hydratedSyncConfig() {
111
+ hydrated ??= config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
112
+ return hydrated;
113
+ }
114
+ };
115
+ }
116
+ asUpdateOptions(options) {
117
+ // defaultSchema is not relevant for the parsed version here
118
+ const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' });
119
+ return {
120
+ config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed },
121
+ ...options
122
+ };
123
+ }
124
+ }
125
+ //# sourceMappingURL=PersistedSyncConfigContent.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"PersistedSyncConfigContent.js","sourceRoot":"","sources":["../../src/storage/PersistedSyncConfigContent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACrG,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EAInB,UAAU,EACV,qBAAqB,EACrB,kBAAkB,EAClB,YAAY,EAEZ,uBAAuB,EACvB,SAAS,EACV,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAMtC,OAAO,EAAE,sBAAsB,EAAwB,MAAM,2BAA2B,CAAC;AASzF,MAAM,UAAU,+BAA+B,CAAC,OAA+C;IAC7F,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAExE,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;QACzB,mEAAmE;QACnE,OAAO,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,oBAAoB,CAAC,WAAW,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;IACnF,MAAM,gBAAgB,GAAyB,EAAE,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC/D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,UAAU,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACjD,CAAC;QAED,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,qBAAqB,CAAC,IAAI,EAAE,aAAa,EAAE,gBAAgB,EAAE;QACnF,aAAa,EAAE,YAAY,CAAC,aAAa;QACzC,UAAU,EAAE,OAAO;KACpB,CAAC,CAAC;IAEH,2GAA2G;IAC3G,+FAA+F;IAC/F,WAAW,CAAC,cAAc,GAAG,cAAc,CAAC;IAE5C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,QAAQ,GAA8B,KAAK,CAAC,QAAQ,IAAI;gBAC5D,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;gBAClC,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;aAC/B,CAAC;YACF,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;YACtE,WAAW,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;YAE/B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AAEH,MAAM,OAAgB,0BAA0B;IACrC,mBAAmB,CAAS;IAC5B,kBAAkB,CAAS;IAC3B,aAAa,CAA4B;IACzC,qBAAqB,CAAS;IAC9B,cAAc,CAAS;IACvB,MAAM,CAAS;IACf,YAAY,CAA+B;IAC3C,eAAe,CAAgB;IAExC,YAAY,IAAoC;QAC9C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACpD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC;QAClD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACpF,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClE,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,+BAA+B,IAAI,CAAC,cAAc,2BAA2B,IAAI,CAAC,mBAAmB,EAAE,CACxG,CAAC;QACJ,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACO,iBAAiB,CAAC,OAA+B;QACzD,OAAO,+BAA+B,CAAC;YACrC,OAAO,EAAE,IAAI,CAAC,kBAAkB;YAChC,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,YAAY,EAAE,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,OAA+B;QACpC,IAAI,cAA8B,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IACE,aAAa,CAAC,gBAAgB;YAC9B,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,EAC7E,CAAC;YACD,cAAc,GAAG,uBAAuB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,uBAAuB,CAAC;QAC3C,CAAC;QAED,IAAI,QAAwC,CAAC;QAC7C,OAAO;YACL,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,WAAW,EAAE,CAAC,MAAM,CAAC;YACrB,cAAc;YACd,IAAI,kBAAkB;gBACpB,QAAQ,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACnF,OAAO,QAAQ,CAAC;YAClB,CAAC;SACF,CAAC;IACJ,CAAC;IAED,eAAe,CAAC,OAAgD;QAC9D,4DAA4D;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC3E,OAAO;YACL,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE;YAC3E,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;CAMF"}
@@ -0,0 +1,13 @@
1
+ import { SyncRuleState } from './BucketStorage.js';
2
+ import { PersistedSyncConfigId } from './PersistedSyncConfigContent.js';
3
+ export interface PersistedSyncConfigStatus {
4
+ readonly id: PersistedSyncConfigId;
5
+ readonly replicationStreamId: number;
6
+ readonly state: SyncRuleState;
7
+ readonly snapshot_done?: boolean;
8
+ readonly last_checkpoint_lsn: string | null;
9
+ readonly last_fatal_error?: string | null;
10
+ readonly last_fatal_error_ts?: Date | null;
11
+ readonly last_keepalive_ts?: Date | null;
12
+ readonly last_checkpoint_ts?: Date | null;
13
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=PersistedSyncConfigStatus.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"PersistedSyncConfigStatus.js","sourceRoot":"","sources":["../../src/storage/PersistedSyncConfigStatus.ts"],"names":[],"mappings":""}
@@ -23,4 +23,10 @@ export interface SourceEntityDescriptor extends SourceTableRef {
23
23
  * The columns that are used to uniquely identify a record in the source entity.
24
24
  */
25
25
  replicaIdColumns: ColumnDescriptor[];
26
+ /**
27
+ * Whether the source always sends complete row data with each operation (e.g. Postgres REPLICA
28
+ * IDENTITY FULL). When true, no current_data copy is needed. Undefined means the source does not
29
+ * report this, in which case we default to keeping a copy.
30
+ */
31
+ sendsCompleteRows?: boolean;
26
32
  }
@@ -1,4 +1,4 @@
1
- import { BucketDataSource, ParameterIndexLookupCreator, SourceTableRef } from '@powersync/service-sync-rules';
1
+ import { BucketDataSource, BucketDefinitionId, ParameterIndexId, ParameterIndexLookupCreator, SourceTableRef } from '@powersync/service-sync-rules';
2
2
  import { bson } from '../index.js';
3
3
  import { ColumnDescriptor } from './SourceEntity.js';
4
4
  /**
@@ -13,6 +13,8 @@ export interface SourceTableOptions {
13
13
  snapshotComplete: boolean;
14
14
  bucketDataSources: BucketDataSource[];
15
15
  parameterLookupSources: ParameterIndexLookupCreator[];
16
+ bucketDataSourceIds?: Set<BucketDefinitionId>;
17
+ parameterLookupSourceIds?: Set<ParameterIndexId>;
16
18
  }
17
19
  export interface TableSnapshotStatus {
18
20
  totalEstimatedCount: number;
@@ -45,13 +47,27 @@ export declare class SourceTable {
45
47
  */
46
48
  syncParameters: boolean;
47
49
  /**
48
- * True if the table is used in sync config for events.
50
+ * True if this table should fire events for row changes.
49
51
  *
50
- * This value is resolved externally, and cached here.
52
+ * This value is resolved externally, and cached here. When multiple SourceTables exist
53
+ * for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
54
+ * as the event carrier, so that a row change saved once per table fires each event once.
51
55
  *
52
56
  * Defaults to true for tests.
53
57
  */
54
58
  syncEvent: boolean;
59
+ /**
60
+ * True if raw data should be stored in current_data collection.
61
+ *
62
+ * This is needed when the source sends partial row data (e.g. TOAST values).
63
+ * When REPLICA IDENTITY FULL is configured, complete rows are always sent,
64
+ * so we don't need to store raw data.
65
+ *
66
+ * This value is resolved externally based on table configuration.
67
+ *
68
+ * Defaults to true for tests (conservative approach).
69
+ */
70
+ storeCurrentData: boolean;
55
71
  /**
56
72
  * Always undefined if snapshotComplete = true.
57
73
  *
@@ -68,6 +84,8 @@ export declare class SourceTable {
68
84
  get replicaIdColumns(): ColumnDescriptor[];
69
85
  get bucketDataSources(): BucketDataSource[];
70
86
  get parameterLookupSources(): ParameterIndexLookupCreator[];
87
+ get bucketDataSourceIds(): Set<string> | undefined;
88
+ get parameterLookupSourceIds(): Set<string> | undefined;
71
89
  /**
72
90
  * Sanitized name of the entity in the format of "{schema}.{entity name}".
73
91
  * Suitable for safe use in Postgres queries.
@@ -26,13 +26,27 @@ export class SourceTable {
26
26
  */
27
27
  syncParameters = true;
28
28
  /**
29
- * True if the table is used in sync config for events.
29
+ * True if this table should fire events for row changes.
30
30
  *
31
- * This value is resolved externally, and cached here.
31
+ * This value is resolved externally, and cached here. When multiple SourceTables exist
32
+ * for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
33
+ * as the event carrier, so that a row change saved once per table fires each event once.
32
34
  *
33
35
  * Defaults to true for tests.
34
36
  */
35
37
  syncEvent = true;
38
+ /**
39
+ * True if raw data should be stored in current_data collection.
40
+ *
41
+ * This is needed when the source sends partial row data (e.g. TOAST values).
42
+ * When REPLICA IDENTITY FULL is configured, complete rows are always sent,
43
+ * so we don't need to store raw data.
44
+ *
45
+ * This value is resolved externally based on table configuration.
46
+ *
47
+ * Defaults to true for tests (conservative approach).
48
+ */
49
+ storeCurrentData = true;
36
50
  /**
37
51
  * Always undefined if snapshotComplete = true.
38
52
  *
@@ -68,6 +82,12 @@ export class SourceTable {
68
82
  get parameterLookupSources() {
69
83
  return this.options.parameterLookupSources;
70
84
  }
85
+ get bucketDataSourceIds() {
86
+ return this.options.bucketDataSourceIds;
87
+ }
88
+ get parameterLookupSourceIds() {
89
+ return this.options.parameterLookupSourceIds;
90
+ }
71
91
  /**
72
92
  * Sanitized name of the entity in the format of "{schema}.{entity name}".
73
93
  * Suitable for safe use in Postgres queries.
@@ -89,11 +109,14 @@ export class SourceTable {
89
109
  replicaIdColumns: this.replicaIdColumns,
90
110
  snapshotComplete: this.snapshotComplete,
91
111
  bucketDataSources: this.bucketDataSources,
92
- parameterLookupSources: this.parameterLookupSources
112
+ parameterLookupSources: this.parameterLookupSources,
113
+ bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
114
+ parameterLookupSourceIds: this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
93
115
  });
94
116
  copy.syncData = this.syncData;
95
117
  copy.syncParameters = this.syncParameters;
96
118
  copy.syncEvent = this.syncEvent;
119
+ copy.storeCurrentData = this.storeCurrentData;
97
120
  copy.snapshotStatus = this.snapshotStatus;
98
121
  return copy;
99
122
  }
@@ -1 +1 @@
1
- {"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,WAAW,EAGZ,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAwB9C;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IAuCM;IAtC5B,MAAM,CAAU,WAAW,GAAG,WAAW,CAAC;IAE1C;;;;;;OAMG;IACI,QAAQ,GAAG,IAAI,CAAC;IAEvB;;;;;;OAMG;IACI,cAAc,GAAG,IAAI,CAAC;IAE7B;;;;;;OAMG;IACI,SAAS,GAAG,IAAI,CAAC;IAExB;;;;OAIG;IACI,cAAc,GAAoC,SAAS,CAAC;IAE5D,gBAAgB,CAAU;IAEjC,YAA4B,OAA2B;QAA3B,YAAO,GAAP,OAAO,CAAoB;QACrD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,IAAI,EAAE;QACJ,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;IACxC,CAAC;IAED,IAAI,sBAAsB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC;YAC3B,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;SACpD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB;QACpB,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YACzD,oBAAoB;YACpB,OAAO,GAAG,CAAC;QACb,CAAC;aAAM,IAAI,IAAI,CAAC,cAAc,CAAC,mBAAmB,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,IAAI,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,KAAK,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QAC9F,CAAC;IACH,CAAC"}
1
+ {"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,WAAW,EAIZ,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AA0B9C;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IAsDM;IArD5B,MAAM,CAAU,WAAW,GAAG,WAAW,CAAC;IAE1C;;;;;;OAMG;IACI,QAAQ,GAAG,IAAI,CAAC;IAEvB;;;;;;OAMG;IACI,cAAc,GAAG,IAAI,CAAC;IAE7B;;;;;;;;OAQG;IACI,SAAS,GAAG,IAAI,CAAC;IAExB;;;;;;;;;;OAUG;IACI,gBAAgB,GAAG,IAAI,CAAC;IAE/B;;;;OAIG;IACI,cAAc,GAAoC,SAAS,CAAC;IAE5D,gBAAgB,CAAU;IAEjC,YAA4B,OAA2B;QAA3B,YAAO,GAAP,OAAO,CAAoB;QACrD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,IAAI,EAAE;QACJ,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;IACxC,CAAC;IAED,IAAI,sBAAsB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC;IAC7C,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC;IAC1C,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC;IAC/C,CAAC;IAED;;;OAGG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC;YAC3B,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC;YACrG,wBAAwB,EACtB,IAAI,CAAC,wBAAwB,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC;SAC7F,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC9C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB;QACpB,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YACzD,oBAAoB;YACpB,OAAO,GAAG,CAAC;QACb,CAAC;aAAM,IAAI,IAAI,CAAC,cAAc,CAAC,mBAAmB,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,IAAI,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,KAAK,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QAC9F,CAAC;IACH,CAAC"}
@@ -5,31 +5,54 @@ import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
5
5
  import * as util from '../util/util-index.js';
6
6
  import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
7
7
  import { BucketStorageFactory } from './BucketStorageFactory.js';
8
- import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
8
+ import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
9
+ import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
9
10
  import { SourceEntityDescriptor } from './SourceEntity.js';
10
11
  import { SourceTable } from './SourceTable.js';
11
12
  import { StorageVersionConfig } from './StorageVersionConfig.js';
12
13
  import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
13
14
  /**
14
15
  * Storage for a specific replication stream.
16
+ *
17
+ * Replication writes source changes, source table metadata, snapshot progress,
18
+ * resume positions, and checkpoint boundaries through this interface. The sync
19
+ * API reads the same stream state as checkpoints, checksums, bucket data,
20
+ * parameter lookup rows, and write-checkpoint acknowledgements.
15
21
  */
16
22
  export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketStorageListener>, SyncStorageWriteCheckpointAPI {
17
- readonly group_id: number;
18
- readonly slot_name: string;
23
+ readonly replicationStreamId: number;
24
+ readonly replicationStreamName: string;
19
25
  readonly storageConfig: StorageVersionConfig;
20
26
  readonly factory: BucketStorageFactory;
21
27
  readonly logger: Logger;
22
28
  /**
23
- * Create a new writer.
29
+ * Create a writer for a source-specific unit of replication work.
24
30
  *
25
- * The writer must be flushed and disposed when done.
31
+ * The writer may be used for a long-running streaming attempt, a table
32
+ * snapshot, a page of snapshot work, or a short setup step. Pending work
33
+ * that should be persisted must be flushed, committed, or kept alive at the
34
+ * appropriate source boundary before the writer is disposed.
26
35
  */
27
36
  createWriter(options: CreateWriterOptions): Promise<BucketStorageBatch>;
28
37
  /**
38
+ * Callback wrapper around `createWriter()`.
39
+ *
29
40
  * @deprecated Use `createWriter()` with `await using` instead.
30
41
  */
31
42
  startBatch(options: CreateWriterOptions, callback: (batch: BucketStorageBatch) => Promise<void>): Promise<FlushedResult | null>;
32
- getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncConfig;
43
+ /**
44
+ * Get the canonical parsed sync config set for the given parse options.
45
+ *
46
+ * Repeated calls with the same options return the same instance for the lifetime of
47
+ * this storage instance. This is the identity boundary for parsed sync config state:
48
+ * all operations against this storage instance must use parsed objects from sets
49
+ * returned here, never from an independent parse of the same content.
50
+ */
51
+ getParsedSyncConfigSet(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
52
+ /**
53
+ * Shortcut for `getParsedSyncConfigSet(options).hydratedSyncConfig`.
54
+ */
55
+ getParsedSyncRules(options: ParseSyncConfigOptions): HydratedSyncConfig;
33
56
  /**
34
57
  * Terminate the replication stream.
35
58
  *
@@ -38,7 +61,7 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
38
61
  * Must only be called on stopped replication streams.
39
62
  */
40
63
  terminate(options?: TerminateOptions): Promise<void>;
41
- getStatus(): Promise<SyncRuleStatus>;
64
+ getStatus(): Promise<ReplicationStreamStatus>;
42
65
  /**
43
66
  * Clear the storage, without changing state.
44
67
  */
@@ -64,7 +87,7 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
64
87
  *
65
88
  * This is a best-effort optimization:
66
89
  * 1. This may include more changes than what actually occurred.
67
- * 2. This may return invalidateDataBuckets or invalidateParameterBuckets instead of of returning
90
+ * 2. This may return invalidateDataBuckets or invalidateParameterBuckets instead of returning
68
91
  * specific changes.
69
92
  * @param options
70
93
  */
@@ -82,11 +105,18 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
82
105
  * 1. Separate buckets.
83
106
  * 2. Limit the size of each individual chunk according to options.batchSizeLimitBytes.
84
107
  *
108
+ * The batch may not contain all data for the checkpoint, if the checkpoint is large. The caller must
109
+ * continue querying if either:
110
+ * 1. The last chunk for any bucket has has_more = true.
111
+ * 2. A SyncBucketDataBatchEnd is returned with hasMore = true.
112
+ *
113
+ * The first check can be skipped if a SyncBucketDataBatchEnd is returned with hasMore = false.
114
+ *
85
115
  * @param checkpoint the checkpoint
86
116
  * @param dataBuckets current bucket states
87
117
  * @param options batch size options
88
118
  */
89
- getBucketDataBatch(checkpoint: util.InternalOpId, dataBuckets: BucketDataRequest[], options?: BucketDataBatchOptions): AsyncIterable<SyncBucketDataChunk>;
119
+ getBucketDataBatch(checkpoint: ReplicationCheckpoint, dataBuckets: BucketDataRequest[], options?: BucketDataBatchOptions): AsyncIterable<SyncBucketDataChunk | SyncBucketDataBatchEnd>;
90
120
  /**
91
121
  * Compute checksums for a given list of buckets.
92
122
  *
@@ -95,11 +125,16 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
95
125
  * This may be slow, depending on the size of the buckets.
96
126
  * The checksums are cached internally to compensate for this, but does not cover all cases.
97
127
  */
98
- getChecksums(checkpoint: util.InternalOpId, buckets: BucketChecksumRequest[]): Promise<util.ChecksumMap>;
128
+ getChecksums(checkpoint: ReplicationCheckpoint, buckets: BucketChecksumRequest[], options?: BucketChecksumOptions): Promise<util.ChecksumMap>;
99
129
  /**
100
130
  * Clear checksum cache. Primarily intended for tests.
101
131
  */
102
132
  clearChecksumCache(): void;
133
+ /**
134
+ * Optional storage-provider cleanup for sync configs that have been stopped
135
+ * inside this replication stream.
136
+ */
137
+ cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise<CleanupStoppedSyncConfigsResult>;
103
138
  }
104
139
  export interface SyncRulesBucketStorageListener {
105
140
  batchStarted: (batch: BucketStorageBatch) => void;
@@ -113,33 +148,52 @@ export interface BucketChecksumRequest {
113
148
  bucket: string;
114
149
  source: BucketDataSource;
115
150
  }
116
- export interface SyncRuleStatus {
117
- checkpoint_lsn: string | null;
118
- active: boolean;
119
- snapshot_done: boolean;
120
- snapshot_lsn: string | null;
151
+ export interface BucketChecksumOptions {
152
+ requestHint?: BucketRequestHint;
153
+ }
154
+ export type BucketRequestHint = 'bulk' | 'incremental';
155
+ export interface ReplicationStreamStatus {
121
156
  /**
122
- * Last persisted operation that must be included in the next checkpoint once checkpointing is unblocked.
157
+ * Source position to resume replication from.
123
158
  */
124
- keepalive_op: util.InternalOpId | null;
159
+ resumeLsn: string | null;
160
+ /**
161
+ * True if _every_ active/processing sync config has a client-visible checkpoint.
162
+ */
163
+ snapshotDone: boolean;
125
164
  }
126
165
  export interface ResolveTablesOptions {
127
166
  connection_id: number;
167
+ /**
168
+ * Source table or collection metadata discovered during snapshot or streaming.
169
+ */
128
170
  source: SourceEntityDescriptor;
129
171
  /**
130
172
  * For tests only - custom id generator for stable ids.
131
173
  */
132
174
  idGenerator?: () => string | bson.ObjectId;
133
175
  /**
134
- * For tests only - override the sync rules used.
176
+ * For tests only - override the parsed sync config set used.
135
177
  */
136
- syncRules?: HydratedSyncConfig;
178
+ parsedSyncConfig?: ParsedSyncConfigSet;
137
179
  }
138
180
  export interface ResolveTablesResult {
181
+ /**
182
+ * Current source table mappings that should receive replicated data.
183
+ */
139
184
  tables: SourceTable[];
185
+ /**
186
+ * Outdated source table mappings that should be removed by the connector.
187
+ */
140
188
  dropTables: SourceTable[];
141
189
  }
142
- export interface CreateWriterOptions extends ParseSyncRulesOptions {
190
+ /**
191
+ * Options for creating a storage writer.
192
+ */
193
+ export interface CreateWriterOptions extends ParseSyncConfigOptions {
194
+ /**
195
+ * Source-specific zero or empty replication position.
196
+ */
143
197
  zeroLSN: string;
144
198
  /**
145
199
  * Whether or not to store a copy of the current data.
@@ -204,12 +258,20 @@ export interface CompactOptions {
204
258
  */
205
259
  compactBuckets?: string[];
206
260
  compactParameterData?: boolean;
261
+ /**
262
+ * Delete client-requested write checkpoints created before this time.
263
+ *
264
+ * Generated write checkpoints are not affected.
265
+ */
266
+ deleteCheckpointRequestsBefore?: Date;
207
267
  /** Minimum of 2 */
208
268
  clearBatchLimit?: number;
209
269
  /** Minimum of 1 */
210
270
  moveBatchLimit?: number;
211
271
  /** Minimum of 1 */
212
272
  moveBatchQueryLimit?: number;
273
+ /** Byte cap per read batch for streaming compaction. Default 16MB. */
274
+ moveBatchByteLimit?: number;
213
275
  /**
214
276
  * Minimum number new operations in a bucket to trigger compaction of that bucket.
215
277
  *
@@ -230,7 +292,14 @@ export interface CompactOptions {
230
292
  logger?: Logger;
231
293
  }
232
294
  export interface PopulateChecksumCacheOptions {
233
- maxOpId: util.InternalOpId;
295
+ /**
296
+ * Compute checksums up to this op id.
297
+ *
298
+ * Defaults to the highest persisted op id for the replication stream, which covers
299
+ * the common case of populating the cache right after initial replication, before
300
+ * the first checkpoint exists.
301
+ */
302
+ maxOpId?: util.InternalOpId;
234
303
  minBucketChanges?: number;
235
304
  signal?: AbortSignal;
236
305
  }
@@ -243,6 +312,22 @@ export interface PopulateChecksumCacheResults {
243
312
  export interface ClearStorageOptions {
244
313
  signal?: AbortSignal;
245
314
  }
315
+ export interface CleanupStoppedSyncConfigsOptions {
316
+ signal?: AbortSignal;
317
+ logger?: Logger;
318
+ defaultSchema: string;
319
+ sourceConnectionTag: string;
320
+ }
321
+ export interface CleanupStoppedSyncConfigsResult {
322
+ stoppedSyncConfigsRemoved: number;
323
+ bucketDataCollectionsDropped: number;
324
+ parameterIndexCollectionsDropped: number;
325
+ /** Best-effort number, not accurate if the operation takes longer than the timeout. */
326
+ bucketStateDocumentsDeleted: number;
327
+ sourceRecordCollectionsDropped: number;
328
+ sourceTablesUpdated: number;
329
+ sourceTablesDeleted: number;
330
+ }
246
331
  export interface TerminateOptions extends ClearStorageOptions {
247
332
  /**
248
333
  * If true, also clear the storage before terminating.
@@ -250,6 +335,9 @@ export interface TerminateOptions extends ClearStorageOptions {
250
335
  clearStorage: boolean;
251
336
  }
252
337
  export interface BucketDataBatchOptions {
338
+ requestHint?: BucketRequestHint;
339
+ /** Abort any in-progress work for this batch, including object-storage downloads. */
340
+ signal?: AbortSignal;
253
341
  /** Limit number of documents returned. Defaults to 1000. */
254
342
  limit?: number;
255
343
  /**
@@ -267,6 +355,16 @@ export interface SyncBucketDataChunk {
267
355
  chunkData: util.SyncBucketData;
268
356
  targetOp: util.InternalOpId | null;
269
357
  }
358
+ export interface SyncBucketDataBatchEnd {
359
+ /**
360
+ * True if there may be more data for this checkpoint, and the caller should continue querying.
361
+ *
362
+ * This is different from `SyncBucketDataChunk.has_more`, which is per-bucket. This is a global signal for the
363
+ * entire request, and may be true even if there is no returned chunk with has_more: true.
364
+ */
365
+ hasMore: boolean;
366
+ }
367
+ export declare function isBatchEnd(chunk: SyncBucketDataChunk | SyncBucketDataBatchEnd): chunk is SyncBucketDataBatchEnd;
270
368
  export interface ReplicationCheckpoint {
271
369
  readonly checkpoint: util.InternalOpId;
272
370
  readonly lsn: string | null;
@@ -1,3 +1,6 @@
1
+ export function isBatchEnd(chunk) {
2
+ return chunk.hasMore !== undefined;
3
+ }
1
4
  /**
2
5
  * An exception thrown by {@link ReplicationCheckpoint} implementations if there are too many parameter results.
3
6
  *
@@ -1 +1 @@
1
- {"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA4VA;;;;GAIG;AACH,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAE5C;IACA;IAFX,YACW,KAAa,EACb,SAAyC;QAElD,KAAK,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;QAHhD,UAAK,GAAL,KAAK,CAAQ;QACb,cAAS,GAAT,SAAS,CAAgC;IAGpD,CAAC;CACF;AAqDD,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,IAAI,GAAG,EAAU;IACrC,qBAAqB,EAAE,IAAI;IAC3B,uBAAuB,EAAE,IAAI,GAAG,EAAU;IAC1C,0BAA0B,EAAE,IAAI;CACjC,CAAC"}
1
+ {"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA6bA,MAAM,UAAU,UAAU,CAAC,KAAmD;IAC5E,OAAQ,KAAgC,CAAC,OAAO,KAAK,SAAS,CAAC;AACjE,CAAC;AAiBD;;;;GAIG;AACH,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAE5C;IACA;IAFX,YACW,KAAa,EACb,SAAyC;QAElD,KAAK,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;QAHhD,UAAK,GAAL,KAAK,CAAQ;QACb,cAAS,GAAT,SAAS,CAAgC;IAGpD,CAAC;CACF;AAqDD,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,IAAI,GAAG,EAAU;IACrC,qBAAqB,EAAE,IAAI;IAC3B,uBAAuB,EAAE,IAAI,GAAG,EAAU;IAC1C,0BAA0B,EAAE,IAAI;CACjC,CAAC"}