@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js

@@ -0,0 +1,70 @@
+import { DataPointType } from '@opentelemetry/sdk-metrics';
+/**
+ * Decorates a push exporter and removes selected attributes from every metric data point before
+ * delegating the export. The input metrics are not mutated, so another reader can export the
+ * original attributes.
+ *
+ * This exporter deliberately does not merge data points when filtering makes their attribute sets
+ * identical. Both points are forwarded with the same attributes. There is no generally correct
+ * merge operation at this layer: gauges, counters, and histograms have different aggregation
+ * semantics. Callers must therefore ensure that an excluded attribute does not distinguish data
+ * points for the same metric, resource, and instrumentation scope. Otherwise, the resulting
+ * duplicate series identity may be rejected or interpreted unpredictably downstream.
+ *
+ * Lifecycle methods and aggregation selection are delegated unchanged to the wrapped exporter.
+ */
+export class MetricAttributeFilteringExporter {
+    delegate;
+    excludedAttributes;
+    selectAggregation;
+    selectAggregationTemporality;
+    constructor(
+    /** The exporter that receives the filtered copy of the metrics. */
+    delegate, 
+    /** Attribute names to remove from every exported data point. */
+    excludedAttributes) {
+        this.delegate = delegate;
+        this.excludedAttributes = excludedAttributes;
+        this.selectAggregation = delegate.selectAggregation?.bind(delegate);
+        this.selectAggregationTemporality = delegate.selectAggregationTemporality?.bind(delegate);
+    }
+    export(metrics, resultCallback) {
+        this.delegate.export(this.filterAttributes(metrics), resultCallback);
+    }
+    forceFlush() {
+        return this.delegate.forceFlush();
+    }
+    shutdown() {
+        return this.delegate.shutdown();
+    }
+    filterAttributes(metrics) {
+        return {
+            ...metrics,
+            scopeMetrics: metrics.scopeMetrics.map((scopeMetrics) => ({
+                ...scopeMetrics,
+                metrics: scopeMetrics.metrics.map((metric) => this.filterMetricAttributes(metric))
+            }))
+        };
+    }
+    filterMetricAttributes(metric) {
+        switch (metric.dataPointType) {
+            case DataPointType.SUM:
+            case DataPointType.GAUGE:
+                return { ...metric, dataPoints: metric.dataPoints.map((dataPoint) => this.filterDataPoint(dataPoint)) };
+            case DataPointType.HISTOGRAM:
+                return { ...metric, dataPoints: metric.dataPoints.map((dataPoint) => this.filterDataPoint(dataPoint)) };
+            case DataPointType.EXPONENTIAL_HISTOGRAM:
+                return { ...metric, dataPoints: metric.dataPoints.map((dataPoint) => this.filterDataPoint(dataPoint)) };
+        }
+    }
+    filterDataPoint(dataPoint) {
+        return {
+            ...dataPoint,
+            attributes: this.filterDataPointAttributes(dataPoint.attributes)
+        };
+    }
+    filterDataPointAttributes(attributes) {
+        return Object.fromEntries(Object.entries(attributes).filter(([name]) => !this.excludedAttributes.has(name)));
+    }
+}
+//# sourceMappingURL=MetricAttributeFilteringExporter.js.map

package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetricAttributeFilteringExporter.js","sourceRoot":"","sources":["../../../src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,aAAa,EAId,MAAM,4BAA4B,CAAC;AAEpC;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,gCAAgC;IAMxB;IAEA;IAPV,iBAAiB,CAAuB;IACxC,4BAA4B,CAAkC;IAEvE;IACE,mEAAmE;IAClD,QAA4B;IAC7C,gEAAgE;IAC/C,kBAAuC;QAFvC,aAAQ,GAAR,QAAQ,CAAoB;QAE5B,uBAAkB,GAAlB,kBAAkB,CAAqB;QAExD,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpE,IAAI,CAAC,4BAA4B,GAAG,QAAQ,CAAC,4BAA4B,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,CAAC,OAAwB,EAAE,cAA8C;QAC7E,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;IACvE,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAClC,CAAC;IAEO,gBAAgB,CAAC,OAAwB;QAC/C,OAAO;YACL,GAAG,OAAO;YACV,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBACxD,GAAG,YAAY;gBACf,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;aACnF,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAEO,sBAAsB,CAAC,MAAkB;QAC/C,QAAQ,MAAM,CAAC,aAAa,EAAE,CAAC;YAC7B,KAAK,aAAa,CAAC,GAAG,CAAC;YACvB,KAAK,aAAa,CAAC,KAAK;gBACtB,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YAC1G,KAAK,aAAa,CAAC,SAAS;gBAC1B,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YAC1G,KAAK,aAAa,CAAC,qBAAqB;gBACtC,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC5G,CAAC;IACH,CAAC;IAEO,eAAe,CAAI,SAAuB;QAChD,OAAO;YACL,GAAG,SAAS;YACZ,UAAU,EAAE,IAAI,CAAC,yBAAyB,CAAC,SAAS,CAAC,UAAU,CAAC;SACjE,CAAC;IACJ,CAAC;IAEO,yBAAyB,CAAC,UAAsB;QACtD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/G,CAAC;CACF"}

package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js

@@ -22,9 +22,14 @@ export class OpenTelemetryMetricsFactory {
             setValueProvider(valueProvider) {
                 gauge.addCallback(async (result) => {
                     const value = await valueProvider();
-                    if (value != undefined) {
+                    if (typeof value === 'number') {
                         result.observe(value);
                     }
+                    else if (value != undefined) {
+                        for (const observation of value) {
+                            result.observe(observation.value, observation.attributes);
+                        }
+                    }
                 });
             }
         };

package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map

@@ -1 +1 @@
-{"version":3,"file":"OpenTelemetryMetricsFactory.js","sourceRoot":"","sources":["../../../src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAKL,SAAS,EAEV,MAAM,0BAA0B,CAAC;AAElC,MAAM,OAAO,2BAA2B;IAC9B,KAAK,CAAQ;IAErB,YAAY,KAAY;QACtB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,aAAa,CAAC,QAAwB;QACpC,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC7C,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;IAED,qBAAqB,CAAC,QAAwB;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC5D,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;SAChD,CAAC,CAAC;QAEH,OAAO;YACL,gBAAgB,CAAC,aAAgD;gBAC/D,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;oBACjC,MAAM,KAAK,GAAG,MAAM,aAAa,EAAE,CAAC;oBAEpC,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;wBACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACxB,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;SACF,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAAwB;QAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE;YACnD,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,SAAqB;QACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,SAAS,CAAC,GAAG,CAAC;QACvB,CAAC;QAED,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,SAAS,CAAC,GAAG;gBAChB,OAAO,SAAS,CAAC,GAAG,CAAC;YACvB,KAAK,SAAS,CAAC,MAAM;gBACnB,OAAO,SAAS,CAAC,MAAM,CAAC;QAC5B,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"OpenTelemetryMetricsFactory.js","sourceRoot":"","sources":["../../../src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAML,SAAS,EAEV,MAAM,0BAA0B,CAAC;AAElC,MAAM,OAAO,2BAA2B;IAC9B,KAAK,CAAQ;IAErB,YAAY,KAAY;QACtB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,aAAa,CAAC,QAAwB;QACpC,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC7C,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;IAED,qBAAqB,CAAC,QAAwB;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC5D,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;SAChD,CAAC,CAAC;QAEH,OAAO;YACL,gBAAgB,CAAC,aAA+E;gBAC9F,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;oBACjC,MAAM,KAAK,GAAG,MAAM,aAAa,EAAE,CAAC;oBAEpC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;wBAC9B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACxB,CAAC;yBAAM,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;wBAC9B,KAAK,MAAM,WAAW,IAAI,KAAK,EAAE,CAAC;4BAChC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;wBAC5D,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;SACF,CAAC;IACJ,CAAC;IAED,mBAAmB,CAAC,QAAwB;QAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE;YACnD,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,SAAqB;QACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,SAAS,CAAC,GAAG,CAAC;QACvB,CAAC;QAED,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,SAAS,CAAC,GAAG;gBAChB,OAAO,SAAS,CAAC,GAAG,CAAC;YACvB,KAAK,SAAS,CAAC,MAAM;gBACnB,OAAO,SAAS,CAAC,MAAM,CAAC;QAC5B,CAAC;IACH,CAAC;CACF"}

package/dist/metrics/open-telemetry/util.js

@@ -2,6 +2,7 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { logger } from '@powersync/lib-services-framework';
+import { MetricAttributeFilteringExporter } from './MetricAttributeFilteringExporter.js';
 import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
 import { resourceFromAttributes } from '@opentelemetry/resources';
 import pkg from '../../../package.json' with { type: 'json' };
@@ -22,10 +23,13 @@ export function createOpenTelemetryMetricsFactory(context) {
         });
     }
     if (!configuration.telemetry.disable_telemetry_sharing) {
+        const otlpExporter = new OTLPMetricExporter({
+            url: configuration.telemetry.internal_service_endpoint
+        });
         const periodicExporter = new PeriodicExportingMetricReader({
-            exporter: new OTLPMetricExporter({
-                url: configuration.telemetry.internal_service_endpoint
-            }),
+            // sync_config_id uniquely identifies these metric series after version_label is removed,
+            // so filtering cannot create duplicate data points.
+            exporter: new MetricAttributeFilteringExporter(otlpExporter, new Set(['version_label'])),
             exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
         });
         configuredExporters.push(periodicExporter);

package/dist/metrics/open-telemetry/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/metrics/open-telemetry/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAgB,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AACxG,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAG3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAE/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,GAAG,MAAM,uBAAuB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAE9D,MAAM,UAAU,iCAAiC,CAAC,OAAuB;IACvE,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAClE,MAAM,mBAAmB,GAAmB,EAAE,CAAC;IAE/C,IAAI,aAAa,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC;QAC5C,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC;YAChD,IAAI,EAAE,aAAa,CAAC,SAAS,CAAC,eAAe;YAC7C,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;QACH,mBAAmB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE7C,eAAe,CAAC,aAAa,CAAC,kBAAkB,EAAE;YAChD,KAAK,EAAE,KAAK,IAAI,EAAE;gBAChB,MAAM,kBAAkB,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,4CAA4C,aAAa,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,CAAC;YACrG,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,yBAAyB,EAAE,CAAC;QACvD,MAAM,gBAAgB,GAAG,IAAI,6BAA6B,CAAC;YACzD,QAAQ,EAAE,IAAI,kBAAkB,CAAC;gBAC/B,GAAG,EAAE,aAAa,CAAC,SAAS,CAAC,yBAAyB;aACvD,CAAC;YACF,oBAAoB,EAAE,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,YAAY;SACjD,CAAC,CAAC;QAEH,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,kBAAwC,CAAC;IAC7C,MAAM,iBAAiB,GAAG,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;QACxD,kBAAkB,GAAG,OAAO,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;QAClC,KAAK,EAAE,KAAK,IAAI,EAAE;YAChB,MAAM,aAAa,GAAG,aAAa,CAAC,mBAAmB,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,sBAAsB,EAAE,CAAC;gBAChE,kBAAkB,CAAC,UAAU,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC;QACtC,CAAC,SAAS,CAAC,EAAE,WAAW;QACxB,CAAC,iBAAiB,CAAC,EAAE,GAAG,CAAC,OAAO;QAChC,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,SAAS;QAClE,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS;QACzD,CAAC,aAAa,CAAC,EAAE,iBAAiB;KACnC,CAAC,CAAC;IAEH,mFAAmF;IACnF,yEAAyE;IACzE,QAAQ,CAAC,sBAAsB,EAAE,EAAE,CAAC;IAEpC,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC;QACtC,QAAQ;QAER,OAAO,EAAE,mBAAmB;KAC7B,CAAC,CAAC;IAEH,eAAe,CAAC,aAAa,CAAC,aAAa,EAAE;QAC3C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,aAAa,CAAC,QAAQ,EAAE,CAAC;QACjC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAElD,OAAO,IAAI,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/metrics/open-telemetry/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAgB,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AACxG,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAG3D,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AACzF,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAE/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,GAAG,MAAM,uBAAuB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAE9D,MAAM,UAAU,iCAAiC,CAAC,OAAuB;IACvE,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAClE,MAAM,mBAAmB,GAAmB,EAAE,CAAC;IAE/C,IAAI,aAAa,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC;QAC5C,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC;YAChD,IAAI,EAAE,aAAa,CAAC,SAAS,CAAC,eAAe;YAC7C,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;QACH,mBAAmB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE7C,eAAe,CAAC,aAAa,CAAC,kBAAkB,EAAE;YAChD,KAAK,EAAE,KAAK,IAAI,EAAE;gBAChB,MAAM,kBAAkB,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,4CAA4C,aAAa,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,CAAC;YACrG,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,yBAAyB,EAAE,CAAC;QACvD,MAAM,YAAY,GAAG,IAAI,kBAAkB,CAAC;YAC1C,GAAG,EAAE,aAAa,CAAC,SAAS,CAAC,yBAAyB;SACvD,CAAC,CAAC;QACH,MAAM,gBAAgB,GAAG,IAAI,6BAA6B,CAAC;YACzD,yFAAyF;YACzF,oDAAoD;YACpD,QAAQ,EAAE,IAAI,gCAAgC,CAAC,YAAY,EAAE,IAAI,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;YACxF,oBAAoB,EAAE,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,YAAY;SACjD,CAAC,CAAC;QAEH,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,kBAAwC,CAAC;IAC7C,MAAM,iBAAiB,GAAG,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;QACxD,kBAAkB,GAAG,OAAO,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;QAClC,KAAK,EAAE,KAAK,IAAI,EAAE;YAChB,MAAM,aAAa,GAAG,aAAa,CAAC,mBAAmB,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,sBAAsB,EAAE,CAAC;gBAChE,kBAAkB,CAAC,UAAU,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,sBAAsB,CAAC;QACtC,CAAC,SAAS,CAAC,EAAE,WAAW;QACxB,CAAC,iBAAiB,CAAC,EAAE,GAAG,CAAC,OAAO;QAChC,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,SAAS;QAClE,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS;QACzD,CAAC,aAAa,CAAC,EAAE,iBAAiB;KACnC,CAAC,CAAC;IAEH,mFAAmF;IACnF,yEAAyE;IACzE,QAAQ,CAAC,sBAAsB,EAAE,EAAE,CAAC;IAEpC,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC;QACtC,QAAQ;QAER,OAAO,EAAE,mBAAmB;KAC7B,CAAC,CAAC;IAEH,eAAe,CAAC,aAAa,CAAC,aAAa,EAAE;QAC3C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,aAAa,CAAC,QAAQ,EAAE,CAAC;QACjC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAElD,OAAO,IAAI,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC"}

package/dist/modules/AbstractModule.d.ts

@@ -1,11 +1,15 @@
 import winston from 'winston';
-import { PersistedSyncRulesContent } from '../storage/storage-index.js';
+import { PersistedReplicationStream } from '../storage/storage-index.js';
 import { ServiceContextContainer } from '../system/ServiceContext.js';
 export interface TearDownOptions {
     /**
      *  If required, tear down any configuration/state for the specific replication stream
      */
-    syncRules?: PersistedSyncRulesContent[];
+    replicationStreams?: PersistedReplicationStream[];
+    /**
+     * @deprecated Use replicationStreams.
+     */
+    syncRules?: PersistedReplicationStream[];
 }
 export interface AbstractModuleOptions {
     name: string;

package/dist/modules/AbstractModule.js.map

@@ -1 +1 @@
-{"version":3,"file":"AbstractModule.js","sourceRoot":"","sources":["../../src/modules/AbstractModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAgB3D,MAAM,OAAgB,cAAc;IAGF;IAFtB,MAAM,CAAiB;IAEjC,YAAgC,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACjE,CAAC;IAYD,IAAW,IAAI;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;CACF"}
+{"version":3,"file":"AbstractModule.js","sourceRoot":"","sources":["../../src/modules/AbstractModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAoB3D,MAAM,OAAgB,cAAc;IAGF;IAFtB,MAAM,CAAiB;IAEjC,YAAgC,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACjE,CAAC;IAYD,IAAW,IAAI;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;CACF"}

package/dist/replication/AbstractReplicator.d.ts

@@ -19,6 +19,10 @@ export interface AbstractReplicatorOptions {
      * This limits the effect of retries when there is a persistent issue.
      */
     rateLimiter: ErrorRateLimiter;
+    /**
+     * Interval in seconds between source connection pings. Null or undefined uses the default; 0 disables pings.
+     */
+    heartbeatIntervalSeconds?: number | null;
 }
 /**
  *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
@@ -28,26 +32,31 @@ export interface AbstractReplicatorOptions {
 export declare abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
     private options;
     protected logger: winston.Logger;
-    private lockAlerted;
+    private lastReplicationStreamInfoLogs;
     /**
-     *  Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
-     *  transitioning to a new replication stream.
+     *  Map of replication jobs by replication stream job id. Usually there is only one running job, but there could be two
+     *  when transitioning to a new replication stream.
      */
     private replicationJobs;
     /**
-     * Map of replciation stream ids to promises that are clearing the replication stream.
+     * Map of replication stream ids to promises that are clearing the replication stream.
      *
-     * We primarily do this to keep track of what we're currently clearing, but don't currently
-     * use the Promise value.
+     * We primarily do this to keep track of what we're currently clearing.
      */
-    private clearingJobs;
+    protected readonly clearingJobs: Map<number, Promise<void>>;
     /**
      * Used for replication lag computation.
      */
     private activeReplicationJob;
     private lastPing;
+    private readonly heartbeatIntervalNanos;
     private abortController;
     protected constructor(options: AbstractReplicatorOptions);
+    /**
+     * Create the source-specific replication job for a locked storage stream.
+     *
+     * The job owns the per-stream snapshot and streaming work for this attempt.
+     */
     abstract createJob(options: CreateJobOptions): T;
     /**
      * Clean up any configuration or state for the specified replication stream on the datasource.
@@ -64,8 +73,23 @@ export declare abstract class AbstractReplicator<T extends AbstractReplicationJo
     stop(): Promise<void>;
     private runLoop;
     private refresh;
+    private logReplicationStreamInfoOnce;
+    private clearReplicationStreamInfoLog;
+    /**
+     * When we load sync config from the filesystem/config source, we ignore any config loaded by other processes.
+     * The idea is that if a different process loads updated config, that process should process it, not this one.
+     *
+     * This specifically helps for cases with rolling deploys.
+     *
+     * This does not apply when sync config is loaded from the database/API, instead of in the config.
+     */
+    private shouldHandleReplicationStream;
     protected createJobId(syncRuleId: number): string;
+    protected terminateStoppedReplicationStream(replicationStream: storage.PersistedReplicationStream, syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
     protected terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
+    /**
+     * Check source connectivity using the replicator's decoded connection config.
+     */
     abstract testConnection(): Promise<ConnectionTestResult>;
     /**
      * Measure replication lag in milliseconds.

package/dist/replication/AbstractReplicator.js

@@ -1,9 +1,15 @@
-import { container, ErrorCode, logger } from '@powersync/lib-services-framework';
+import { container, ErrorCode, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
 import { ReplicationMetric } from '@powersync/service-types';
 import { hrtime } from 'node:process';
+import { setTimeout as sleep } from 'node:timers/promises';
 import * as storage from '../storage/storage-index.js';
-// 1 minute
-const PING_INTERVAL = 1000000000n * 60n;
+// Default to 1 minute when no source-specific interval is configured.
+const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 60;
+// In the initial startup period, we use a short refresh interval. This helps to take over replication quickly in the case
+// of rolling deploys. After the initial period, we switch to a longer refresh interval to reduce load.
+const FAST_REFRESH_INTERVAL_MS = 500;
+const FAST_REFRESH_TIMEOUT_MS = 120_000;
+const REFRESH_INTERVAL_MS = 5_000;
 /**
  *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
  *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
@@ -12,29 +18,32 @@ const PING_INTERVAL = 1000000000n * 60n;
 export class AbstractReplicator {
     options;
     logger;
-    lockAlerted = false;
+    lastReplicationStreamInfoLogs = new Map();
     /**
-     *  Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
-     *  transitioning to a new replication stream.
+     *  Map of replication jobs by replication stream job id. Usually there is only one running job, but there could be two
+     *  when transitioning to a new replication stream.
      */
     replicationJobs = new Map();
     /**
-     * Map of replciation stream ids to promises that are clearing the replication stream.
+     * Map of replication stream ids to promises that are clearing the replication stream.
      *
-     * We primarily do this to keep track of what we're currently clearing, but don't currently
-     * use the Promise value.
+     * We primarily do this to keep track of what we're currently clearing.
      */
     clearingJobs = new Map();
     /**
      * Used for replication lag computation.
      */
     activeReplicationJob = undefined;
-    // First ping is only after 5 minutes, not when starting
+    // The first ping is only after the configured interval, not when starting.
     lastPing = hrtime.bigint();
+    heartbeatIntervalNanos;
     abortController;
     constructor(options) {
         this.options = options;
         this.logger = logger.child({ name: `Replicator:${options.id}` });
+        const heartbeatIntervalSeconds = options.heartbeatIntervalSeconds ?? DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
+        this.heartbeatIntervalNanos =
+            heartbeatIntervalSeconds === 0 ? null : BigInt(Math.round(heartbeatIntervalSeconds * 1_000_000_000));
     }
     get id() {
         return this.options.id;
@@ -84,17 +93,26 @@ export class AbstractReplicator {
         for (const job of this.replicationJobs.values()) {
             promises.push(job.stop());
         }
+        promises.push(...this.clearingJobs.values());
         await Promise.all(promises);
     }
     async runLoop() {
-        const syncRules = await this.syncRuleProvider.get();
+        const loadedSyncConfig = await this.syncRuleProvider.get();
         let configuredLock = undefined;
-        if (syncRules != null) {
-            this.logger.info('Loaded sync config');
+        if (loadedSyncConfig != null) {
+            const versionLabel = this.syncRuleProvider.versionLabel;
+            this.logger.info(`Loaded sync config${versionLabel == null ? '' : ` ${versionLabel}`}`);
             try {
                 // Configure new sync config, if they have changed.
                 // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
-                const { lock } = await this.storage.configureSyncRules(storage.updateSyncRulesFromYaml(syncRules, { lock: true, validate: this.syncRuleProvider.exitOnError }));
+                // This upfront lock is mostly superseded by the replicationStreamLoadedSyncConfigMatch() check. However, that doesn't cover old
+                // versions before that check was added, so we keep the lock for now - for where te service version and sync config is updated at
+                // the same time.
+                const { lock } = await this.storage.configureSyncRules(storage.updateSyncRulesFromYaml(loadedSyncConfig, {
+                    lock: true,
+                    validate: this.syncRuleProvider.exitOnError,
+                    version_label: versionLabel
+                }));
                 if (lock) {
                     configuredLock = lock;
                 }
@@ -109,17 +127,26 @@ export class AbstractReplicator {
         else {
             this.logger.info('No sync streams or rules configured - configure via API');
         }
+        let useFastRefresh = true;
+        const fastRefreshDeadline = Date.now() + FAST_REFRESH_TIMEOUT_MS;
         while (!this.stopped) {
             await container.probes.touch();
             try {
-                await this.refresh({ configured_lock: configuredLock });
+                const refreshResult = await this.refresh({
+                    configuredLock,
+                    loadedSyncConfig,
+                    loadedVersionLabel: this.syncRuleProvider.versionLabel
+                });
+                if (refreshResult.replicationJobStarted || Date.now() >= fastRefreshDeadline) {
+                    useFastRefresh = false;
+                }
                 // The lock is only valid on the first refresh.
                 configuredLock = undefined;
                 // Ensure that the replication jobs' connections are kept alive.
                 // We don't ping while in error retry back-off, to avoid having too failures.
-                if (this.rateLimiter.mayPing()) {
+                if (this.heartbeatIntervalNanos != null && this.rateLimiter.mayPing()) {
                     const now = hrtime.bigint();
-                    if (now - this.lastPing >= PING_INTERVAL) {
+                    if (now - this.lastPing >= this.heartbeatIntervalNanos) {
                         for (const activeJob of this.replicationJobs.values()) {
                             await activeJob.keepAlive();
                         }
@@ -130,76 +157,54 @@ export class AbstractReplicator {
             catch (e) {
                 this.logger.error('Failed to refresh replication jobs', e);
             }
-            await new Promise((resolve) => setTimeout(resolve, 5000));
+            if (Date.now() >= fastRefreshDeadline) {
+                useFastRefresh = false;
+            }
+            await sleep(useFastRefresh ? FAST_REFRESH_INTERVAL_MS : REFRESH_INTERVAL_MS);
         }
     }
     async refresh(options) {
         if (this.stopped) {
-            return;
+            return { replicationJobStarted: false };
         }
-        let configuredLock = options?.configured_lock;
+        let configuredLock = options?.configuredLock;
+        let replicationJobStarted = false;
         const existingJobs = new Map(this.replicationJobs.entries());
-        const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
+        const replicatingStreams = await this.storage.getReplicatingReplicationStreams();
         const newJobs = new Map();
+        const streamsToStart = [];
         let activeJob = undefined;
-        for (let syncRules of replicatingSyncRules) {
-            const existingJob = existingJobs.get(syncRules.id);
-            if (syncRules.active && activeJob == null) {
+        for (let replicationStream of replicatingStreams) {
+            const jobId = replicationStream.replicationJobId;
+            const existingJob = existingJobs.get(jobId);
+            const syncConfigMismatchMessage = 'Ignoring replication stream for sync config not loaded by this process';
+            if (!this.shouldHandleReplicationStream(replicationStream, options?.loadedSyncConfig, options?.loadedVersionLabel)) {
+                this.logReplicationStreamInfoOnce(replicationStream, 'sync-config-mismatch', () => {
+                    replicationStream.logger.info(syncConfigMismatchMessage);
+                });
+                continue;
+            }
+            this.clearReplicationStreamInfoLog(replicationStream, 'sync-config-mismatch');
+            if (replicationStream.state == storage.SyncRuleState.ACTIVE && activeJob == null) {
                 activeJob = existingJob;
             }
             if (existingJob && !existingJob.isStopped) {
                 // No change
-                existingJobs.delete(syncRules.id);
-                newJobs.set(syncRules.id, existingJob);
+                existingJobs.delete(jobId);
+                newJobs.set(jobId, existingJob);
             }
             else if (existingJob && existingJob.isStopped) {
                 // Stopped (e.g. fatal error).
                 // Remove from the list. Next refresh call will restart the job.
-                existingJobs.delete(syncRules.id);
+                existingJobs.delete(jobId);
             }
             else {
                 // New sync config was found (or resume after restart)
-                try {
-                    let lock;
-                    if (configuredLock?.sync_rules_id == syncRules.id) {
-                        lock = configuredLock;
-                    }
-                    else {
-                        lock = await syncRules.lock();
-                    }
-                    const storage = this.storage.getInstance(syncRules);
-                    const newJob = this.createJob({
-                        lock: lock,
-                        storage: storage
-                    });
-                    newJobs.set(syncRules.id, newJob);
-                    newJob.start();
-                    if (syncRules.active) {
-                        activeJob = newJob;
-                    }
-                    this.lockAlerted = false;
-                }
-                catch (e) {
-                    if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
-                        if (!this.lockAlerted) {
-                            syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
-                            this.lockAlerted = true;
-                        }
-                    }
-                    else {
-                        // Could be a sync config parse error,
-                        // for example from stricter validation that was added.
-                        // This will be retried every couple of seconds.
-                        // When new (valid) sync config is deployed and processed, this one be disabled.
-                        syncRules.logger.error('Failed to start replication for new sync config', e);
-                    }
-                }
+                streamsToStart.push(replicationStream);
             }
         }
-        this.replicationJobs = newJobs;
-        this.activeReplicationJob = activeJob;
-        // Stop any orphaned jobs that no longer have a replication stream.
-        // Termination happens below
+        // Stop any orphaned jobs that no longer have a replication stream before starting replacements.
+        // Termination happens below.
         for (let job of existingJobs.values()) {
             // Old - stop and clean up
             try {
@@ -210,30 +215,119 @@ export class AbstractReplicator {
                 job.storage.logger.warn('Failed to stop old replication job', e);
             }
         }
+        for (let replicationStream of streamsToStart) {
+            const jobId = replicationStream.replicationJobId;
+            try {
+                let lock;
+                if (configuredLock?.sync_rules_id == replicationStream.replicationStreamId) {
+                    lock = configuredLock;
+                }
+                else {
+                    lock = await replicationStream.lock();
+                }
+                const syncRuleStorage = this.storage.getInstance(replicationStream);
+                const newJob = this.createJob({
+                    lock: lock,
+                    storage: syncRuleStorage
+                });
+                newJobs.set(jobId, newJob);
+                newJob.start();
+                replicationJobStarted = true;
+                if (replicationStream.state == storage.SyncRuleState.ACTIVE) {
+                    activeJob = newJob;
+                }
+                this.lastReplicationStreamInfoLogs.delete(replicationStream.replicationStreamName);
+            }
+            catch (e) {
+                if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
+                    this.logReplicationStreamInfoOnce(replicationStream, 'replication-stream-locked', () => {
+                        replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
+                    });
+                }
+                else {
+                    // Could be a sync config parse error,
+                    // for example from stricter validation that was added.
+                    // This will be retried every couple of seconds.
+                    // When new (valid) sync config is deployed and processed, this one be disabled.
+                    replicationStream.logger.error('Failed to start replication for new sync config', e);
+                }
+            }
+        }
+        this.replicationJobs = newJobs;
+        this.activeReplicationJob = activeJob;
         // Replication stream stopped previously, including by a different process.
-        const stopped = await this.storage.getStoppedSyncRules();
-        for (let syncRules of stopped) {
-            if (this.clearingJobs.has(syncRules.id)) {
+        const stopped = await this.storage.getStoppedReplicationStreams();
+        for (let replicationStream of stopped) {
+            if (this.clearingJobs.has(replicationStream.replicationStreamId)) {
                 // Already in progress
                 continue;
             }
             // We clear storage asynchronously.
             // It is important to be able to continue running the refresh loop, otherwise we cannot
             // retry locked replication stream, for example.
-            const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
-            const promise = this.terminateSyncRules(syncRuleStorage)
+            const syncRuleStorage = this.storage.getInstance(replicationStream, { skipLifecycleHooks: true });
+            const promise = this.terminateStoppedReplicationStream(replicationStream, syncRuleStorage)
                 .catch((e) => {
-                syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
+                if (e instanceof ReplicationAbortedError) {
+                    // Expected when shutdown aborts an in-progress cleanup.
+                }
+                else if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
+                    this.logReplicationStreamInfoOnce(replicationStream, 'replication-stream-cleanup-locked', () => {
+                        replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
+                    });
+                }
+                else {
+                    syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
+                }
             })
                 .finally(() => {
-                this.clearingJobs.delete(syncRules.id);
+                this.clearingJobs.delete(replicationStream.replicationStreamId);
             });
-            this.clearingJobs.set(syncRules.id, promise);
+            this.clearingJobs.set(replicationStream.replicationStreamId, promise);
+        }
+        return { replicationJobStarted };
+    }
+    logReplicationStreamInfoOnce(replicationStream, category, log) {
+        if (this.lastReplicationStreamInfoLogs.get(replicationStream.replicationStreamName) == category) {
+            return;
         }
+        log();
+        this.lastReplicationStreamInfoLogs.set(replicationStream.replicationStreamName, category);
+    }
+    clearReplicationStreamInfoLog(replicationStream, category) {
+        if (this.lastReplicationStreamInfoLogs.get(replicationStream.replicationStreamName) == category) {
+            this.lastReplicationStreamInfoLogs.delete(replicationStream.replicationStreamName);
+        }
+    }
+    /**
+     * When we load sync config from the filesystem/config source, we ignore any config loaded by other processes.
+     * The idea is that if a different process loads updated config, that process should process it, not this one.
+     *
+     * This specifically helps for cases with rolling deploys.
+     *
+     * This does not apply when sync config is loaded from the database/API, instead of in the config.
+     */
+    shouldHandleReplicationStream(replicationStream, loadedSyncRules, loadedVersionLabel) {
+        if (loadedSyncRules == null) {
+            return true;
+        }
+        const processingConfig = replicationStream.syncConfigContent.find((syncConfig) => syncConfig.syncConfigState == storage.SyncRuleState.PROCESSING);
+        return (processingConfig == null ||
+            (processingConfig.sync_rules_content == loadedSyncRules && processingConfig.version_label == loadedVersionLabel));
     }
     createJobId(syncRuleId) {
         return `${this.id}-${syncRuleId}`;
     }
+    async terminateStoppedReplicationStream(replicationStream, syncRuleStorage) {
+        const lock = await replicationStream.lock();
+        this.clearReplicationStreamInfoLog(replicationStream, 'replication-stream-cleanup-locked');
+        try {
+            await this.terminateSyncRules(syncRuleStorage);
+        }
+        finally {
+            await lock.release();
+        }
+    }
     async terminateSyncRules(syncRuleStorage) {
         syncRuleStorage.logger.info(`Terminating replication stream...`);
         // This deletes postgres replication slots - should complete quickly.

package/dist/replication/AbstractReplicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACjF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAOvD,WAAW;AACX,MAAM,aAAa,GAAG,WAAc,GAAG,GAAG,CAAC;AAkB3C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IA2BR;IA1BpB,MAAM,CAAiB;IACzB,WAAW,GAAY,KAAK,CAAC;IACrC;;;OAGG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C;;;;;OAKG;IACK,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAC;IAExD;;OAEG;IACK,oBAAoB,GAAkB,SAAS,CAAC;IAExD,wDAAwD;IAChD,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAE3B,eAAe,CAA8B;IAErD,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACpC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC;IAC9C,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC,CAAC;YAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,CAAC,gBAAgB,CAAC,KAAK,IAAI,EAAE;YACrG,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC3C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,gBAAgB;gBAChB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;gBACtD,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QAEpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACvC,IAAI,CAAC;gBACH,mDAAmD;gBACnD,2GAA2G;gBAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CACpD,OAAO,CAAC,uBAAuB,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CACxG,CAAC;gBACF,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,4BAA4B;gBAC5B,4EAA4E;gBAC5E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC,CAAC,CAAC;gBACrD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;wBACzC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,IAAI,SAAS,GAAkB,SAAS,CAAC;QACzC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;gBAC1C,SAAS,GAAG,WAAW,CAAC;YAC1B,CAAC;YACD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBAClD,IAAI,GAAG,cAAc,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;oBACf,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;wBACrB,SAAS,GAAG,MAAM,CAAC;oBACrB,CAAC;oBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC;wBACjD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;4BACtB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;4BAC1E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBAC1B,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,sCAAsC;wBACtC,uDAAuD;wBACvD,gDAAgD;wBAChD,gFAAgF;wBAChF,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,CAAC,CAAC,CAAC;oBAC/E,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAC/B,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QAEtC,mEAAmE;QACnE,4BAA4B;QAC5B,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,0BAA0B;YAC1B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxC,sBAAsB;gBACtB,SAAS;YACX,CAAC;YAED,mCAAmC;YACnC,uFAAuF;YACvF,gDAAgD;YAChD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1F,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC;iBACrD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YACvE,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YACL,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QACjE,qEAAqE;QACrE,qEAAqE;QACrE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACpC,MAAM,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC5E,CAAC;IAID;;;;;;;;;;;;;;;;;OAiBG;IACH,uBAAuB;QACrB,OAAO,IAAI,CAAC,oBAAoB,EAAE,uBAAuB,EAAE,CAAC;IAC9D,CAAC;CACF"}
+{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC1G,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAOvD,sEAAsE;AACtE,MAAM,kCAAkC,GAAG,EAAE,CAAC;AAE9C,0HAA0H;AAC1H,uGAAuG;AACvG,MAAM,wBAAwB,GAAG,GAAG,CAAC;AACrC,MAAM,uBAAuB,GAAG,OAAO,CAAC;AACxC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAsBlC;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IA4BR;IA3BpB,MAAM,CAAiB;IACzB,6BAA6B,GAAG,IAAI,GAAG,EAAkB,CAAC;IAClE;;;OAGG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C;;;;OAIG;IACgB,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEnE;;OAEG;IACK,oBAAoB,GAAkB,SAAS,CAAC;IAExD,2EAA2E;IACnE,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAElB,sBAAsB,CAAgB;IAE/C,eAAe,CAA8B;IAErD,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACjE,MAAM,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,IAAI,kCAAkC,CAAC;QACxG,IAAI,CAAC,sBAAsB;YACzB,wBAAwB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,aAAa,CAAC,CAAC,CAAC;IACzG,CAAC;IAeD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACpC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC;IAC9C,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC,CAAC;YAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,CAAC,gBAAgB,CAAC,KAAK,IAAI,EAAE;YACrG,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC3C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,gBAAgB;gBAChB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;gBACtD,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7C,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QAE3D,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,EAAE,CAAC,CAAC;YACxF,IAAI,CAAC;gBACH,mDAAmD;gBACnD,2GAA2G;gBAC3G,gIAAgI;gBAChI,iIAAiI;gBACjI,iBAAiB;gBAEjB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CACpD,OAAO,CAAC,uBAAuB,CAAC,gBAAgB,EAAE;oBAChD,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW;oBAC3C,aAAa,EAAE,YAAY;iBAC5B,CAAC,CACH,CAAC;gBACF,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,4BAA4B;gBAC5B,4EAA4E;gBAC5E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC,CAAC,CAAC;gBACrD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,cAAc,GAAG,IAAI,CAAC;QAC1B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,uBAAuB,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;oBACvC,cAAc;oBACd,gBAAgB;oBAChB,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,YAAY;iBACvD,CAAC,CAAC;gBACH,IAAI,aAAa,CAAC,qBAAqB,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,mBAAmB,EAAE,CAAC;oBAC7E,cAAc,GAAG,KAAK,CAAC;gBACzB,CAAC;gBACD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBACtE,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;wBACvD,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,mBAAmB,EAAE,CAAC;gBACtC,cAAc,GAAG,KAAK,CAAC;YACzB,CAAC;YACD,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAIrB;QACC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;QAC7C,IAAI,qBAAqB,GAAG,KAAK,CAAC;QAElC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,gCAAgC,EAAE,CAAC;QACjF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,MAAM,cAAc,GAAyC,EAAE,CAAC;QAChE,IAAI,SAAS,GAAkB,SAAS,CAAC;QACzC,KAAK,IAAI,iBAAiB,IAAI,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;YACjD,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,yBAAyB,GAAG,wEAAwE,CAAC;YAC3G,IACE,CAAC,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,kBAAkB,CAAC,EAC9G,CAAC;gBACD,IAAI,CAAC,4BAA4B,CAAC,iBAAiB,EAAE,sBAAsB,EAAE,GAAG,EAAE;oBAChF,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;gBAC3D,CAAC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,CAAC;YAC9E,IAAI,iBAAiB,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;gBACjF,SAAS,GAAG,WAAW,CAAC;YAC1B,CAAC;YACD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,gGAAgG;QAChG,6BAA6B;QAC7B,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,0BAA0B;YAC1B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAED,KAAK,IAAI,iBAAiB,IAAI,cAAc,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;YACjD,IAAI,CAAC;gBACH,IAAI,IAA6B,CAAC;gBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;oBAC3E,IAAI,GAAG,cAAc,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC;gBACxC,CAAC;gBACD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;gBACpE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC5B,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBAEH,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC3B,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,qBAAqB,GAAG,IAAI,CAAC;gBAC7B,IAAI,iBAAiB,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;oBAC5D,SAAS,GAAG,MAAM,CAAC;gBACrB,CAAC;gBACD,IAAI,CAAC,6BAA6B,CAAC,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,CAAC;YACrF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC;oBACjD,IAAI,CAAC,4BAA4B,CAAC,iBAAiB,EAAE,2BAA2B,EAAE,GAAG,EAAE;wBACrF,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;oBACpF,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,sCAAsC;oBACtC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,CAAC,CAAC,CAAC;gBACvF,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAC/B,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QAEtC,2EAA2E;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE,CAAC;QAClE,KAAK,IAAI,iBAAiB,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBACjE,sBAAsB;gBACtB,SAAS;YACX,CAAC;YAED,mCAAmC;YACnC,uFAAuF;YACvF,gDAAgD;YAChD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,iBAAiB,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;YAClG,MAAM,OAAO,GAAG,IAAI,CAAC,iCAAiC,CAAC,iBAAiB,EAAE,eAAe,CAAC;iBACvF,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,IAAI,CAAC,YAAY,uBAAuB,EAAE,CAAC;oBACzC,wDAAwD;gBAC1D,CAAC;qBAAM,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC;oBACxD,IAAI,CAAC,4BAA4B,CAAC,iBAAiB,EAAE,mCAAmC,EAAE,GAAG,EAAE;wBAC7F,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;oBACpF,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;YAClE,CAAC,CAAC,CAAC;YACL,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACxE,CAAC;QAED,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACnC,CAAC;IAEO,4BAA4B,CAClC,iBAAqD,EACrD,QAAgB,EAChB,GAAe;QAEf,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,QAAQ,EAAE,CAAC;YAChG,OAAO;QACT,CAAC;QAED,GAAG,EAAE,CAAC;QACN,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAEO,6BAA6B,CAAC,iBAAqD,EAAE,QAAgB;QAC3G,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,QAAQ,EAAE,CAAC;YAChG,IAAI,CAAC,6BAA6B,CAAC,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,6BAA6B,CACnC,iBAAqD,EACrD,eAAmC,EACnC,kBAAsC;QAEtC,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,iBAAiB,CAAC,IAAI,CAC/D,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,eAAe,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,CAC/E,CAAC;QACF,OAAO,CACL,gBAAgB,IAAI,IAAI;YACxB,CAAC,gBAAgB,CAAC,kBAAkB,IAAI,eAAe,IAAI,gBAAgB,CAAC,aAAa,IAAI,kBAAkB,CAAC,CACjH,CAAC;IACJ,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,iCAAiC,CAC/C,iBAAqD,EACrD,eAA+C;QAE/C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,EAAE,mCAAmC,CAAC,CAAC;QAC3F,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;QACjD,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QACjE,qEAAqE;QACrE,qEAAqE;QACrE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACpC,MAAM,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC5E,CAAC;IAOD;;;;;;;;;;;;;;;;;OAiBG;IACH,uBAAuB;QACrB,OAAO,IAAI,CAAC,oBAAoB,EAAE,uBAAuB,EAAE,CAAC;IAC9D,CAAC;CACF"}