@powersync/service-core 1.10.2 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/dist/api/api-index.d.ts +1 -0
  3. package/dist/api/api-index.js +1 -0
  4. package/dist/api/api-index.js.map +1 -1
  5. package/dist/api/api-metrics.d.ts +11 -0
  6. package/dist/api/api-metrics.js +30 -0
  7. package/dist/api/api-metrics.js.map +1 -0
  8. package/dist/index.d.ts +2 -2
  9. package/dist/index.js +2 -2
  10. package/dist/index.js.map +1 -1
  11. package/dist/metrics/MetricsEngine.d.ts +21 -0
  12. package/dist/metrics/MetricsEngine.js +79 -0
  13. package/dist/metrics/MetricsEngine.js.map +1 -0
  14. package/dist/metrics/metrics-index.d.ts +5 -0
  15. package/dist/metrics/metrics-index.js +6 -0
  16. package/dist/metrics/metrics-index.js.map +1 -0
  17. package/dist/metrics/metrics-interfaces.d.ts +36 -0
  18. package/dist/metrics/metrics-interfaces.js +6 -0
  19. package/dist/metrics/metrics-interfaces.js.map +1 -0
  20. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.d.ts +10 -0
  21. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +51 -0
  22. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -0
  23. package/dist/metrics/open-telemetry/util.d.ts +6 -0
  24. package/dist/metrics/open-telemetry/util.js +62 -0
  25. package/dist/metrics/open-telemetry/util.js.map +1 -0
  26. package/dist/metrics/register-metrics.d.ts +11 -0
  27. package/dist/metrics/register-metrics.js +44 -0
  28. package/dist/metrics/register-metrics.js.map +1 -0
  29. package/dist/replication/AbstractReplicationJob.d.ts +2 -0
  30. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  31. package/dist/replication/AbstractReplicator.d.ts +3 -0
  32. package/dist/replication/AbstractReplicator.js +3 -0
  33. package/dist/replication/AbstractReplicator.js.map +1 -1
  34. package/dist/replication/ReplicationModule.d.ts +7 -0
  35. package/dist/replication/ReplicationModule.js +1 -0
  36. package/dist/replication/ReplicationModule.js.map +1 -1
  37. package/dist/replication/replication-index.d.ts +1 -0
  38. package/dist/replication/replication-index.js +1 -0
  39. package/dist/replication/replication-index.js.map +1 -1
  40. package/dist/replication/replication-metrics.d.ts +11 -0
  41. package/dist/replication/replication-metrics.js +39 -0
  42. package/dist/replication/replication-metrics.js.map +1 -0
  43. package/dist/routes/endpoints/socket-route.js +5 -5
  44. package/dist/routes/endpoints/socket-route.js.map +1 -1
  45. package/dist/routes/endpoints/sync-stream.js +6 -6
  46. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  47. package/dist/storage/storage-index.d.ts +1 -0
  48. package/dist/storage/storage-index.js +1 -0
  49. package/dist/storage/storage-index.js.map +1 -1
  50. package/dist/storage/storage-metrics.d.ts +4 -0
  51. package/dist/storage/storage-metrics.js +56 -0
  52. package/dist/storage/storage-metrics.js.map +1 -0
  53. package/dist/sync/RequestTracker.d.ts +3 -0
  54. package/dist/sync/RequestTracker.js +8 -3
  55. package/dist/sync/RequestTracker.js.map +1 -1
  56. package/dist/system/ServiceContext.d.ts +3 -3
  57. package/dist/system/ServiceContext.js +7 -3
  58. package/dist/system/ServiceContext.js.map +1 -1
  59. package/dist/util/config/compound-config-collector.js +1 -0
  60. package/dist/util/config/compound-config-collector.js.map +1 -1
  61. package/dist/util/config/types.d.ts +1 -0
  62. package/dist/util/env.d.ts +0 -1
  63. package/dist/util/env.js +0 -4
  64. package/dist/util/env.js.map +1 -1
  65. package/package.json +7 -7
  66. package/src/api/api-index.ts +1 -0
  67. package/src/api/api-metrics.ts +35 -0
  68. package/src/index.ts +2 -2
  69. package/src/metrics/MetricsEngine.ts +98 -0
  70. package/src/metrics/metrics-index.ts +5 -0
  71. package/src/metrics/metrics-interfaces.ts +41 -0
  72. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +66 -0
  73. package/src/metrics/open-telemetry/util.ts +80 -0
  74. package/src/metrics/register-metrics.ts +56 -0
  75. package/src/replication/AbstractReplicationJob.ts +2 -0
  76. package/src/replication/AbstractReplicator.ts +7 -0
  77. package/src/replication/ReplicationModule.ts +10 -0
  78. package/src/replication/replication-index.ts +1 -0
  79. package/src/replication/replication-metrics.ts +45 -0
  80. package/src/routes/endpoints/socket-route.ts +6 -5
  81. package/src/routes/endpoints/sync-stream.ts +7 -6
  82. package/src/storage/storage-index.ts +1 -0
  83. package/src/storage/storage-metrics.ts +67 -0
  84. package/src/sync/RequestTracker.ts +9 -3
  85. package/src/system/ServiceContext.ts +9 -4
  86. package/src/util/config/compound-config-collector.ts +1 -0
  87. package/src/util/config/types.ts +1 -0
  88. package/src/util/env.ts +0 -4
  89. package/tsconfig.tsbuildinfo +1 -1
  90. package/dist/metrics/Metrics.d.ts +0 -30
  91. package/dist/metrics/Metrics.js +0 -202
  92. package/dist/metrics/Metrics.js.map +0 -1
  93. package/src/metrics/Metrics.ts +0 -255
@@ -1,30 +0,0 @@
1
- import { Attributes, Counter, ObservableGauge, UpDownCounter } from '@opentelemetry/api';
2
- import * as storage from '../storage/storage-index.js';
3
- export interface MetricsOptions {
4
- disable_telemetry_sharing: boolean;
5
- powersync_instance_id: string;
6
- internal_metrics_endpoint: string;
7
- }
8
- export declare class Metrics {
9
- private static instance;
10
- private prometheusExporter;
11
- private meterProvider;
12
- data_replicated_bytes: Counter<Attributes>;
13
- data_synced_bytes: Counter<Attributes>;
14
- rows_replicated_total: Counter<Attributes>;
15
- transactions_replicated_total: Counter<Attributes>;
16
- chunks_replicated_total: Counter<Attributes>;
17
- operations_synced_total: Counter<Attributes>;
18
- replication_storage_size_bytes: ObservableGauge<Attributes>;
19
- operation_storage_size_bytes: ObservableGauge<Attributes>;
20
- parameter_storage_size_bytes: ObservableGauge<Attributes>;
21
- concurrent_connections: UpDownCounter<Attributes>;
22
- private constructor();
23
- resetCounters(): void;
24
- static getInstance(): Metrics;
25
- static initialise(options: MetricsOptions): Promise<void>;
26
- shutdown(): Promise<void>;
27
- configureApiMetrics(): void;
28
- configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory): void;
29
- getMetricValueForTests(name: string): Promise<number | undefined>;
30
- }
@@ -1,202 +0,0 @@
1
- import { ValueType } from '@opentelemetry/api';
2
- import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
3
- import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
4
- import { Resource } from '@opentelemetry/resources';
5
- import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
6
- import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
7
- import * as util from '../util/util-index.js';
8
- export class Metrics {
9
- static instance;
10
- prometheusExporter;
11
- meterProvider;
12
- // Metrics
13
- // 1. Data processing / month
14
- // 1a. Postgres -> PowerSync
15
- // Record on replication pod
16
- data_replicated_bytes;
17
- // 1b. PowerSync -> clients
18
- // Record on API pod
19
- data_synced_bytes;
20
- // Unused for pricing
21
- // Record on replication pod
22
- rows_replicated_total;
23
- // Unused for pricing
24
- // Record on replication pod
25
- transactions_replicated_total;
26
- // Unused for pricing
27
- // Record on replication pod
28
- chunks_replicated_total;
29
- // 2. Sync operations / month
30
- // Record on API pod
31
- operations_synced_total;
32
- // 3. Data hosted on PowerSync sync service
33
- // Record on replication pod
34
- // 3a. Replication storage -> raw data as received from Postgres.
35
- replication_storage_size_bytes;
36
- // 3b. Operations storage -> transformed history, as will be synced to clients
37
- operation_storage_size_bytes;
38
- // 3c. Parameter storage -> used for parameter queries
39
- parameter_storage_size_bytes;
40
- // 4. Peak concurrent connections
41
- // Record on API pod
42
- concurrent_connections;
43
- constructor(meterProvider, prometheusExporter) {
44
- this.meterProvider = meterProvider;
45
- this.prometheusExporter = prometheusExporter;
46
- const meter = meterProvider.getMeter('powersync');
47
- this.data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', {
48
- description: 'Uncompressed size of replicated data',
49
- unit: 'bytes',
50
- valueType: ValueType.INT
51
- });
52
- this.data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', {
53
- description: 'Uncompressed size of synced data',
54
- unit: 'bytes',
55
- valueType: ValueType.INT
56
- });
57
- this.rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', {
58
- description: 'Total number of replicated rows',
59
- valueType: ValueType.INT
60
- });
61
- this.transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', {
62
- description: 'Total number of replicated transactions',
63
- valueType: ValueType.INT
64
- });
65
- this.chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', {
66
- description: 'Total number of replication chunks',
67
- valueType: ValueType.INT
68
- });
69
- this.operations_synced_total = meter.createCounter('powersync_operations_synced_total', {
70
- description: 'Number of operations synced',
71
- valueType: ValueType.INT
72
- });
73
- this.replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', {
74
- description: 'Size of current data stored in PowerSync',
75
- unit: 'bytes',
76
- valueType: ValueType.INT
77
- });
78
- this.operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', {
79
- description: 'Size of operations stored in PowerSync',
80
- unit: 'bytes',
81
- valueType: ValueType.INT
82
- });
83
- this.parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', {
84
- description: 'Size of parameter data stored in PowerSync',
85
- unit: 'bytes',
86
- valueType: ValueType.INT
87
- });
88
- this.concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', {
89
- description: 'Number of concurrent sync connections',
90
- valueType: ValueType.INT
91
- });
92
- }
93
- // Generally only useful for tests. Note: gauges are ignored here.
94
- resetCounters() {
95
- this.data_replicated_bytes.add(0);
96
- this.data_synced_bytes.add(0);
97
- this.rows_replicated_total.add(0);
98
- this.transactions_replicated_total.add(0);
99
- this.chunks_replicated_total.add(0);
100
- this.operations_synced_total.add(0);
101
- this.concurrent_connections.add(0);
102
- }
103
- static getInstance() {
104
- if (!Metrics.instance) {
105
- throw new ServiceAssertionError('Metrics have not been initialized');
106
- }
107
- return Metrics.instance;
108
- }
109
- static async initialise(options) {
110
- if (Metrics.instance) {
111
- return;
112
- }
113
- logger.info('Configuring telemetry.');
114
- logger.info(`
115
- Attention:
116
- PowerSync collects completely anonymous telemetry regarding usage.
117
- This information is used to shape our roadmap to better serve our customers.
118
- You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
119
- https://docs.powersync.com/self-hosting/telemetry
120
- Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
121
- `.trim());
122
- const configuredExporters = [];
123
- const port = util.env.METRICS_PORT ?? 0;
124
- const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true });
125
- configuredExporters.push(prometheusExporter);
126
- if (!options.disable_telemetry_sharing) {
127
- logger.info('Sharing anonymous telemetry');
128
- const periodicExporter = new PeriodicExportingMetricReader({
129
- exporter: new OTLPMetricExporter({
130
- url: options.internal_metrics_endpoint
131
- }),
132
- exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
133
- });
134
- configuredExporters.push(periodicExporter);
135
- }
136
- const meterProvider = new MeterProvider({
137
- resource: new Resource({
138
- ['service']: 'PowerSync',
139
- ['instance_id']: options.powersync_instance_id
140
- }),
141
- readers: configuredExporters
142
- });
143
- if (port > 0) {
144
- await prometheusExporter.startServer();
145
- }
146
- Metrics.instance = new Metrics(meterProvider, prometheusExporter);
147
- logger.info('Telemetry configuration complete.');
148
- }
149
- async shutdown() {
150
- await this.meterProvider.shutdown();
151
- }
152
- configureApiMetrics() {
153
- // Initialize the metric, so that it reports a value before connections
154
- // have been opened.
155
- this.concurrent_connections.add(0);
156
- }
157
- configureReplicationMetrics(bucketStorage) {
158
- // Rate limit collection of these stats, since it may be an expensive query
159
- const MINIMUM_INTERVAL = 60_000;
160
- let cachedRequest = undefined;
161
- let cacheTimestamp = 0;
162
- function getMetrics() {
163
- if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
164
- cachedRequest = bucketStorage.getStorageMetrics().catch((e) => {
165
- logger.error(`Failed to get storage metrics`, e);
166
- return null;
167
- });
168
- cacheTimestamp = Date.now();
169
- }
170
- return cachedRequest;
171
- }
172
- this.operation_storage_size_bytes.addCallback(async (result) => {
173
- const metrics = await getMetrics();
174
- if (metrics) {
175
- result.observe(metrics.operations_size_bytes);
176
- }
177
- });
178
- this.parameter_storage_size_bytes.addCallback(async (result) => {
179
- const metrics = await getMetrics();
180
- if (metrics) {
181
- result.observe(metrics.parameters_size_bytes);
182
- }
183
- });
184
- this.replication_storage_size_bytes.addCallback(async (result) => {
185
- const metrics = await getMetrics();
186
- if (metrics) {
187
- result.observe(metrics.replication_size_bytes);
188
- }
189
- });
190
- }
191
- async getMetricValueForTests(name) {
192
- const metrics = await this.prometheusExporter.collect();
193
- const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics;
194
- const metric = scoped.find((metric) => metric.descriptor.name == name);
195
- if (metric == null) {
196
- throw new Error(`Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}`);
197
- }
198
- const point = metric.dataPoints[metric.dataPoints.length - 1];
199
- return point?.value;
200
- }
201
- }
202
- //# sourceMappingURL=Metrics.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"Metrics.js","sourceRoot":"","sources":["../../src/metrics/Metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuD,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpG,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,aAAa,EAAgB,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AACxG,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAElF,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAQ9C,MAAM,OAAO,OAAO;IACV,MAAM,CAAC,QAAQ,CAAU;IAEzB,kBAAkB,CAAqB;IACvC,aAAa,CAAgB;IAErC,UAAU;IACV,6BAA6B;IAE7B,4BAA4B;IAC5B,4BAA4B;IACrB,qBAAqB,CAAsB;IAClD,2BAA2B;IAC3B,oBAAoB;IACb,iBAAiB,CAAsB;IAC9C,qBAAqB;IACrB,4BAA4B;IACrB,qBAAqB,CAAsB;IAClD,qBAAqB;IACrB,4BAA4B;IACrB,6BAA6B,CAAsB;IAC1D,qBAAqB;IACrB,4BAA4B;IACrB,uBAAuB,CAAsB;IAEpD,6BAA6B;IAE7B,oBAAoB;IACb,uBAAuB,CAAsB;IAEpD,2CAA2C;IAE3C,4BAA4B;IAC5B,iEAAiE;IAC1D,8BAA8B,CAA8B;IACnE,8EAA8E;IACvE,4BAA4B,CAA8B;IACjE,sDAAsD;IAC/C,4BAA4B,CAA8B;IAEjE,iCAAiC;IAEjC,oBAAoB;IACb,sBAAsB,CAA4B;IAEzD,YAAoB,aAA4B,EAAE,kBAAsC;QACtF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAElD,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC,aAAa,CAAC,uCAAuC,EAAE;YACxF,WAAW,EAAE,sCAAsC;YACnD,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC,aAAa,CAAC,mCAAmC,EAAE;YAChF,WAAW,EAAE,kCAAkC;YAC/C,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC,aAAa,CAAC,iCAAiC,EAAE;YAClF,WAAW,EAAE,iCAAiC;YAC9C,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,6BAA6B,GAAG,KAAK,CAAC,aAAa,CAAC,yCAAyC,EAAE;YAClG,WAAW,EAAE,yCAAyC;YACtD,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC,aAAa,CAAC,mCAAmC,EAAE;YACtF,WAAW,EAAE,oCAAoC;YACjD,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC,aAAa,CAAC,mCAAmC,EAAE;YACtF,WAAW,EAAE,6BAA6B;YAC1C,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC,qBAAqB,CAAC,0CAA0C,EAAE;YAC5G,WAAW,EAAE,0CAA0C;YACvD,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,4BAA4B,GAAG,KAAK,CAAC,qBAAqB,CAAC,wCAAwC,EAAE;YACxG,WAAW,EAAE,wCAAwC;YACrD,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,4BAA4B,GAAG,KAAK,CAAC,qBAAqB,CAAC,wCAAwC,EAAE;YACxG,WAAW,EAAE,4CAA4C;YACzD,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,sBAAsB,GAAG,KAAK,CAAC,mBAAmB,CAAC,kCAAkC,EAAE;YAC1F,WAAW,EAAE,uCAAuC;YACpD,SAAS,EAAE,SAAS,CAAC,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,kEAAkE;IAClE,aAAa;QACX,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAEM,MAAM,CAAC,WAAW;QACvB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAAC,mCAAmC,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAuB;QACpD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAEtC,MAAM,CAAC,IAAI,CACT;;;;;;oCAM8B,OAAO,CAAC,yBAAyB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;KACzF,CAAC,IAAI,EAAE,CACP,CAAC;QAEF,MAAM,mBAAmB,GAAmB,EAAE,CAAC;QAE/C,MAAM,IAAI,GAAW,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC;QAChD,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5F,mBAAmB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE7C,IAAI,CAAC,OAAO,CAAC,yBAAyB,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAC3C,MAAM,gBAAgB,GAAG,IAAI,6BAA6B,CAAC;gBACzD,QAAQ,EAAE,IAAI,kBAAkB,CAAC;oBAC/B,GAAG,EAAE,OAAO,CAAC,yBAAyB;iBACvC,CAAC;gBACF,oBAAoB,EAAE,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,YAAY;aACjD,CAAC,CAAC;YAEH,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC;YACtC,QAAQ,EAAE,IAAI,QAAQ,CAAC;gBACrB,CAAC,SAAS,CAAC,EAAE,WAAW;gBACxB,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC,qBAAqB;aAC/C,CAAC;YACF,OAAO,EAAE,mBAAmB;SAC7B,CAAC,CAAC;QAEH,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,kBAAkB,CAAC,WAAW,EAAE,CAAC;QACzC,CAAC;QAED,OAAO,CAAC,QAAQ,GAAG,IAAI,OAAO,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;QAElE,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IACnD,CAAC;IAEM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;IACtC,CAAC;IAEM,mBAAmB;QACxB,uEAAuE;QACvE,oBAAoB;QACpB,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAEM,2BAA2B,CAAC,aAA2C;QAC5E,2EAA2E;QAC3E,MAAM,gBAAgB,GAAG,MAAM,CAAC;QAEhC,IAAI,aAAa,GAAuD,SAAS,CAAC;QAClF,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,SAAS,UAAU;YACjB,IAAI,aAAa,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;gBAC5E,aAAa,GAAG,aAAa,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC5D,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;oBACjD,OAAO,IAAI,CAAC;gBACd,CAAC,CAAC,CAAC;gBACH,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9B,CAAC;YACD,OAAO,aAAa,CAAC;QACvB,CAAC;QAED,IAAI,CAAC,4BAA4B,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC7D,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;YACnC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,4BAA4B,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC7D,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;YACnC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,8BAA8B,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC/D,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;YACnC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,IAAY;QAC9C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;QACvE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,cAAc,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9D,OAAO,KAAK,EAAE,KAAe,CAAC;IAChC,CAAC;CACF"}
@@ -1,255 +0,0 @@
1
- import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
2
- import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
3
- import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
4
- import { Resource } from '@opentelemetry/resources';
5
- import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
6
- import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
7
- import * as storage from '../storage/storage-index.js';
8
- import * as util from '../util/util-index.js';
9
-
10
- export interface MetricsOptions {
11
- disable_telemetry_sharing: boolean;
12
- powersync_instance_id: string;
13
- internal_metrics_endpoint: string;
14
- }
15
-
16
- export class Metrics {
17
- private static instance: Metrics;
18
-
19
- private prometheusExporter: PrometheusExporter;
20
- private meterProvider: MeterProvider;
21
-
22
- // Metrics
23
- // 1. Data processing / month
24
-
25
- // 1a. Postgres -> PowerSync
26
- // Record on replication pod
27
- public data_replicated_bytes: Counter<Attributes>;
28
- // 1b. PowerSync -> clients
29
- // Record on API pod
30
- public data_synced_bytes: Counter<Attributes>;
31
- // Unused for pricing
32
- // Record on replication pod
33
- public rows_replicated_total: Counter<Attributes>;
34
- // Unused for pricing
35
- // Record on replication pod
36
- public transactions_replicated_total: Counter<Attributes>;
37
- // Unused for pricing
38
- // Record on replication pod
39
- public chunks_replicated_total: Counter<Attributes>;
40
-
41
- // 2. Sync operations / month
42
-
43
- // Record on API pod
44
- public operations_synced_total: Counter<Attributes>;
45
-
46
- // 3. Data hosted on PowerSync sync service
47
-
48
- // Record on replication pod
49
- // 3a. Replication storage -> raw data as received from Postgres.
50
- public replication_storage_size_bytes: ObservableGauge<Attributes>;
51
- // 3b. Operations storage -> transformed history, as will be synced to clients
52
- public operation_storage_size_bytes: ObservableGauge<Attributes>;
53
- // 3c. Parameter storage -> used for parameter queries
54
- public parameter_storage_size_bytes: ObservableGauge<Attributes>;
55
-
56
- // 4. Peak concurrent connections
57
-
58
- // Record on API pod
59
- public concurrent_connections: UpDownCounter<Attributes>;
60
-
61
- private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
62
- this.meterProvider = meterProvider;
63
- this.prometheusExporter = prometheusExporter;
64
- const meter = meterProvider.getMeter('powersync');
65
-
66
- this.data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', {
67
- description: 'Uncompressed size of replicated data',
68
- unit: 'bytes',
69
- valueType: ValueType.INT
70
- });
71
-
72
- this.data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', {
73
- description: 'Uncompressed size of synced data',
74
- unit: 'bytes',
75
- valueType: ValueType.INT
76
- });
77
-
78
- this.rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', {
79
- description: 'Total number of replicated rows',
80
- valueType: ValueType.INT
81
- });
82
-
83
- this.transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', {
84
- description: 'Total number of replicated transactions',
85
- valueType: ValueType.INT
86
- });
87
-
88
- this.chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', {
89
- description: 'Total number of replication chunks',
90
- valueType: ValueType.INT
91
- });
92
-
93
- this.operations_synced_total = meter.createCounter('powersync_operations_synced_total', {
94
- description: 'Number of operations synced',
95
- valueType: ValueType.INT
96
- });
97
-
98
- this.replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', {
99
- description: 'Size of current data stored in PowerSync',
100
- unit: 'bytes',
101
- valueType: ValueType.INT
102
- });
103
-
104
- this.operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', {
105
- description: 'Size of operations stored in PowerSync',
106
- unit: 'bytes',
107
- valueType: ValueType.INT
108
- });
109
-
110
- this.parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', {
111
- description: 'Size of parameter data stored in PowerSync',
112
- unit: 'bytes',
113
- valueType: ValueType.INT
114
- });
115
-
116
- this.concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', {
117
- description: 'Number of concurrent sync connections',
118
- valueType: ValueType.INT
119
- });
120
- }
121
-
122
- // Generally only useful for tests. Note: gauges are ignored here.
123
- resetCounters() {
124
- this.data_replicated_bytes.add(0);
125
- this.data_synced_bytes.add(0);
126
- this.rows_replicated_total.add(0);
127
- this.transactions_replicated_total.add(0);
128
- this.chunks_replicated_total.add(0);
129
- this.operations_synced_total.add(0);
130
- this.concurrent_connections.add(0);
131
- }
132
-
133
- public static getInstance(): Metrics {
134
- if (!Metrics.instance) {
135
- throw new ServiceAssertionError('Metrics have not been initialized');
136
- }
137
-
138
- return Metrics.instance;
139
- }
140
-
141
- public static async initialise(options: MetricsOptions): Promise<void> {
142
- if (Metrics.instance) {
143
- return;
144
- }
145
- logger.info('Configuring telemetry.');
146
-
147
- logger.info(
148
- `
149
- Attention:
150
- PowerSync collects completely anonymous telemetry regarding usage.
151
- This information is used to shape our roadmap to better serve our customers.
152
- You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
153
- https://docs.powersync.com/self-hosting/telemetry
154
- Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
155
- `.trim()
156
- );
157
-
158
- const configuredExporters: MetricReader[] = [];
159
-
160
- const port: number = util.env.METRICS_PORT ?? 0;
161
- const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true });
162
- configuredExporters.push(prometheusExporter);
163
-
164
- if (!options.disable_telemetry_sharing) {
165
- logger.info('Sharing anonymous telemetry');
166
- const periodicExporter = new PeriodicExportingMetricReader({
167
- exporter: new OTLPMetricExporter({
168
- url: options.internal_metrics_endpoint
169
- }),
170
- exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
171
- });
172
-
173
- configuredExporters.push(periodicExporter);
174
- }
175
-
176
- const meterProvider = new MeterProvider({
177
- resource: new Resource({
178
- ['service']: 'PowerSync',
179
- ['instance_id']: options.powersync_instance_id
180
- }),
181
- readers: configuredExporters
182
- });
183
-
184
- if (port > 0) {
185
- await prometheusExporter.startServer();
186
- }
187
-
188
- Metrics.instance = new Metrics(meterProvider, prometheusExporter);
189
-
190
- logger.info('Telemetry configuration complete.');
191
- }
192
-
193
- public async shutdown(): Promise<void> {
194
- await this.meterProvider.shutdown();
195
- }
196
-
197
- public configureApiMetrics() {
198
- // Initialize the metric, so that it reports a value before connections
199
- // have been opened.
200
- this.concurrent_connections.add(0);
201
- }
202
-
203
- public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) {
204
- // Rate limit collection of these stats, since it may be an expensive query
205
- const MINIMUM_INTERVAL = 60_000;
206
-
207
- let cachedRequest: Promise<storage.StorageMetrics | null> | undefined = undefined;
208
- let cacheTimestamp = 0;
209
-
210
- function getMetrics() {
211
- if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
212
- cachedRequest = bucketStorage.getStorageMetrics().catch((e) => {
213
- logger.error(`Failed to get storage metrics`, e);
214
- return null;
215
- });
216
- cacheTimestamp = Date.now();
217
- }
218
- return cachedRequest;
219
- }
220
-
221
- this.operation_storage_size_bytes.addCallback(async (result) => {
222
- const metrics = await getMetrics();
223
- if (metrics) {
224
- result.observe(metrics.operations_size_bytes);
225
- }
226
- });
227
-
228
- this.parameter_storage_size_bytes.addCallback(async (result) => {
229
- const metrics = await getMetrics();
230
- if (metrics) {
231
- result.observe(metrics.parameters_size_bytes);
232
- }
233
- });
234
-
235
- this.replication_storage_size_bytes.addCallback(async (result) => {
236
- const metrics = await getMetrics();
237
- if (metrics) {
238
- result.observe(metrics.replication_size_bytes);
239
- }
240
- });
241
- }
242
-
243
- public async getMetricValueForTests(name: string): Promise<number | undefined> {
244
- const metrics = await this.prometheusExporter.collect();
245
- const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics;
246
- const metric = scoped.find((metric) => metric.descriptor.name == name);
247
- if (metric == null) {
248
- throw new Error(
249
- `Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}`
250
- );
251
- }
252
- const point = metric.dataPoints[metric.dataPoints.length - 1];
253
- return point?.value as number;
254
- }
255
- }