@powersync/service-core 0.0.2 → 0.1.0

package/dist/metrics/metrics.d.ts

@@ -1,16 +0,0 @@
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-export declare const exporter: PrometheusExporter;
-export declare const data_replicated_bytes: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
-export declare const data_synced_bytes: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
-export declare const rows_replicated_total: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
-export declare const transactions_replicated_total: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
-export declare const chunks_replicated_total: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
-export declare const operations_synced_total: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
-export declare const replication_storage_size_bytes: import("@opentelemetry/api").ObservableGauge<import("@opentelemetry/api").Attributes>;
-export declare const operation_storage_size_bytes: import("@opentelemetry/api").ObservableGauge<import("@opentelemetry/api").Attributes>;
-export declare const parameter_storage_size_bytes: import("@opentelemetry/api").ObservableGauge<import("@opentelemetry/api").Attributes>;
-export declare const concurrent_connections: import("@opentelemetry/api").UpDownCounter<import("@opentelemetry/api").Attributes>;
-export declare function configureApiMetrics(): void;
-export declare function configureReplicationMetrics(system: CorePowerSyncSystem): void;
-export declare function getMetricValueForTests(name: string): Promise<number | undefined>;

package/dist/metrics/metrics.js

@@ -1,139 +0,0 @@
-import * as micro from '@journeyapps-platform/micro';
-import { ValueType } from '@opentelemetry/api';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { MeterProvider } from '@opentelemetry/sdk-metrics';
-import * as jpgwire from '@powersync/service-jpgwire';
-import * as util from '../util/util-index.js';
-const meterProvider = new MeterProvider();
-const port = util.env.METRICS_PORT ?? 0;
-// Expose a pull-based metrics exporter on a prometheus endpoint.
-// This is not quite the standard approach of opentelemetry or journey-micro.
-// However, it avoids the need for running a separate sidecar that uses additional resources.
-// We can consider switching to the standard push-based architecture if needed later.
-export const exporter = new PrometheusExporter({ port: port, preventServerStart: true });
-meterProvider.addMetricReader(exporter);
-if (port > 0) {
-    exporter.startServer();
-}
-const meter = meterProvider.getMeter('powersync');
-// 1. Data processing / month
-// 1a. Postgres -> PowerSync
-// Record on replication pod
-export const data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', {
-    description: 'Uncompressed size of replicated data',
-    unit: 'bytes',
-    valueType: ValueType.INT
-});
-// 1b. PowerSync -> clients
-// Record on API pod
-export const data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', {
-    description: 'Uncompressed size of synced data',
-    unit: 'bytes',
-    valueType: ValueType.INT
-});
-// Unused for pricing
-// Record on replication pod
-export const rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', {
-    description: 'Total number of replicated rows',
-    valueType: ValueType.INT
-});
-// Unused for pricing
-// Record on replication pod
-export const transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', {
-    description: 'Total number of replicated transactions',
-    valueType: ValueType.INT
-});
-// Unused for pricing
-// Record on replication pod
-export const chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', {
-    description: 'Total number of replication chunks',
-    valueType: ValueType.INT
-});
-// 2. Sync operations / month
-// Record on API pod
-export const operations_synced_total = meter.createCounter('powersync_operations_synced_total', {
-    description: 'Number of operations synced',
-    valueType: ValueType.INT
-});
-// 3. Data hosted on PowerSync sync service
-// Record on replication pod
-// 3a. Replication storage -> raw data as received from Postgres.
-export const replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', {
-    description: 'Size of current data stored in PowerSync',
-    unit: 'bytes',
-    valueType: ValueType.INT
-});
-// 3b. Operations storage -> transformed history, as will be synced to clients
-export const operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', {
-    description: 'Size of operations stored in PowerSync',
-    unit: 'bytes',
-    valueType: ValueType.INT
-});
-// 3c. Parameter storage -> used for parameter queries
-export const parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', {
-    description: 'Size of parameter data stored in PowerSync',
-    unit: 'bytes',
-    valueType: ValueType.INT
-});
-// 4. Peak concurrent connections
-// Record on API pod
-export const concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', {
-    description: 'Number of concurrent sync connections',
-    valueType: ValueType.INT
-});
-export function configureApiMetrics() {
-    // Initialize the metric, so that it reports a value before connections
-    // have been opened.
-    concurrent_connections.add(0);
-}
-export function configureReplicationMetrics(system) {
-    // Rate limit collection of these stats, since it may be an expensive query
-    const MINIMUM_INTERVAL = 60000;
-    let cachedRequest = undefined;
-    let cacheTimestamp = 0;
-    function getMetrics() {
-        if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
-            cachedRequest = system.storage.getStorageMetrics().catch((e) => {
-                micro.logger.error(`Failed to get storage metrics`, e);
-                return null;
-            });
-            cacheTimestamp = Date.now();
-        }
-        return cachedRequest;
-    }
-    operation_storage_size_bytes.addCallback(async (result) => {
-        const metrics = await getMetrics();
-        if (metrics) {
-            result.observe(metrics.operations_size_bytes);
-        }
-    });
-    parameter_storage_size_bytes.addCallback(async (result) => {
-        const metrics = await getMetrics();
-        if (metrics) {
-            result.observe(metrics.parameters_size_bytes);
-        }
-    });
-    replication_storage_size_bytes.addCallback(async (result) => {
-        const metrics = await getMetrics();
-        if (metrics) {
-            result.observe(metrics.replication_size_bytes);
-        }
-    });
-    // Record replicated bytes using global jpgwire metrics.
-    jpgwire.setMetricsRecorder({
-        addBytesRead(bytes) {
-            data_replicated_bytes.add(bytes);
-        }
-    });
-}
-export async function getMetricValueForTests(name) {
-    const metrics = await exporter.collect();
-    const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics;
-    const metric = scoped.find((metric) => metric.descriptor.name == name);
-    if (metric == null) {
-        throw new Error(`Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}`);
-    }
-    const point = metric.dataPoints[metric.dataPoints.length - 1];
-    return point?.value;
-}
-//# sourceMappingURL=metrics.js.map

package/dist/metrics/metrics.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/metrics/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AAEtD,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAI7C,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;AAE1C,MAAM,IAAI,GAAW,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC;AAEhD,iEAAiE;AACjE,6EAA6E;AAC7E,6FAA6F;AAC7F,qFAAqF;AAErF,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;AACzF,aAAa,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;AAExC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;IACb,QAAQ,CAAC,WAAW,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAElD,6BAA6B;AAE7B,4BAA4B;AAC5B,4BAA4B;AAC5B,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC,aAAa,CAAC,uCAAuC,EAAE;IAChG,WAAW,EAAE,sCAAsC;IACnD,IAAI,EAAE,OAAO;IACb,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,2BAA2B;AAC3B,oBAAoB;AACpB,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAC,aAAa,CAAC,mCAAmC,EAAE;IACxF,WAAW,EAAE,kCAAkC;IAC/C,IAAI,EAAE,OAAO;IACb,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,qBAAqB;AACrB,4BAA4B;AAC5B,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC,aAAa,CAAC,iCAAiC,EAAE;IAC1F,WAAW,EAAE,iCAAiC;IAC9C,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,qBAAqB;AACrB,4BAA4B;AAC5B,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,CAAC,aAAa,CAAC,yCAAyC,EAAE;IAC1G,WAAW,EAAE,yCAAyC;IACtD,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,qBAAqB;AACrB,4BAA4B;AAC5B,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,CAAC,aAAa,CAAC,mCAAmC,EAAE;IAC9F,WAAW,EAAE,oCAAoC;IACjD,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,6BAA6B;AAC7B,oBAAoB;AACpB,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,CAAC,aAAa,CAAC,mCAAmC,EAAE;IAC9F,WAAW,EAAE,6BAA6B;IAC1C,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,2CAA2C;AAC3C,4BAA4B;AAE5B,iEAAiE;AACjE,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,CAAC,qBAAqB,CAAC,0CAA0C,EAAE;IACpH,WAAW,EAAE,0CAA0C;IACvD,IAAI,EAAE,OAAO;IACb,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,8EAA8E;AAC9E,MAAM,CAAC,MAAM,4BAA4B,GAAG,KAAK,CAAC,qBAAqB,CAAC,wCAAwC,EAAE;IAChH,WAAW,EAAE,wCAAwC;IACrD,IAAI,EAAE,OAAO;IACb,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,sDAAsD;AACtD,MAAM,CAAC,MAAM,4BAA4B,GAAG,KAAK,CAAC,qBAAqB,CAAC,wCAAwC,EAAE;IAChH,WAAW,EAAE,4CAA4C;IACzD,IAAI,EAAE,OAAO;IACb,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,iCAAiC;AACjC,oBAAoB;AAEpB,MAAM,CAAC,MAAM,sBAAsB,GAAG,KAAK,CAAC,mBAAmB,CAAC,kCAAkC,EAAE;IAClG,WAAW,EAAE,uCAAuC;IACpD,SAAS,EAAE,SAAS,CAAC,GAAG;CACzB,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB;IACjC,uEAAuE;IACvE,oBAAoB;IACpB,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAA2B;IACrE,2EAA2E;IAC3E,MAAM,gBAAgB,GAAG,KAAM,CAAC;IAEhC,IAAI,aAAa,GAAuD,SAAS,CAAC;IAClF,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,SAAS,UAAU;QACjB,IAAI,aAAa,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YAC5E,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC7D,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;YACH,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,4BAA4B,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACxD,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,4BAA4B,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACxD,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,8BAA8B,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC1D,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,wDAAwD;IACxD,OAAO,CAAC,kBAAkB,CAAC;QACzB,YAAY,CAAC,KAAK;YAChB,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAY;IACvD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;IACvE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,cAAc,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9D,OAAO,KAAK,EAAE,KAAe,CAAC;AAChC,CAAC"}

package/src/metrics/metrics.ts

@@ -1,169 +0,0 @@
-import * as micro from '@journeyapps-platform/micro';
-import { ValueType } from '@opentelemetry/api';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { MeterProvider } from '@opentelemetry/sdk-metrics';
-import * as jpgwire from '@powersync/service-jpgwire';
-
-import * as util from '@/util/util-index.js';
-import * as storage from '@/storage/storage-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-
-const meterProvider = new MeterProvider();
-
-const port: number = util.env.METRICS_PORT ?? 0;
-
-// Expose a pull-based metrics exporter on a prometheus endpoint.
-// This is not quite the standard approach of opentelemetry or journey-micro.
-// However, it avoids the need for running a separate sidecar that uses additional resources.
-// We can consider switching to the standard push-based architecture if needed later.
-
-export const exporter = new PrometheusExporter({ port: port, preventServerStart: true });
-meterProvider.addMetricReader(exporter);
-
-if (port > 0) {
-  exporter.startServer();
-}
-
-const meter = meterProvider.getMeter('powersync');
-
-// 1. Data processing / month
-
-// 1a. Postgres -> PowerSync
-// Record on replication pod
-export const data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', {
-  description: 'Uncompressed size of replicated data',
-  unit: 'bytes',
-  valueType: ValueType.INT
-});
-
-// 1b. PowerSync -> clients
-// Record on API pod
-export const data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', {
-  description: 'Uncompressed size of synced data',
-  unit: 'bytes',
-  valueType: ValueType.INT
-});
-
-// Unused for pricing
-// Record on replication pod
-export const rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', {
-  description: 'Total number of replicated rows',
-  valueType: ValueType.INT
-});
-
-// Unused for pricing
-// Record on replication pod
-export const transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', {
-  description: 'Total number of replicated transactions',
-  valueType: ValueType.INT
-});
-
-// Unused for pricing
-// Record on replication pod
-export const chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', {
-  description: 'Total number of replication chunks',
-  valueType: ValueType.INT
-});
-
-// 2. Sync operations / month
-// Record on API pod
-export const operations_synced_total = meter.createCounter('powersync_operations_synced_total', {
-  description: 'Number of operations synced',
-  valueType: ValueType.INT
-});
-
-// 3. Data hosted on PowerSync sync service
-// Record on replication pod
-
-// 3a. Replication storage -> raw data as received from Postgres.
-export const replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', {
-  description: 'Size of current data stored in PowerSync',
-  unit: 'bytes',
-  valueType: ValueType.INT
-});
-
-// 3b. Operations storage -> transformed history, as will be synced to clients
-export const operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', {
-  description: 'Size of operations stored in PowerSync',
-  unit: 'bytes',
-  valueType: ValueType.INT
-});
-
-// 3c. Parameter storage -> used for parameter queries
-export const parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', {
-  description: 'Size of parameter data stored in PowerSync',
-  unit: 'bytes',
-  valueType: ValueType.INT
-});
-
-// 4. Peak concurrent connections
-// Record on API pod
-
-export const concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', {
-  description: 'Number of concurrent sync connections',
-  valueType: ValueType.INT
-});
-
-export function configureApiMetrics() {
-  // Initialize the metric, so that it reports a value before connections
-  // have been opened.
-  concurrent_connections.add(0);
-}
-
-export function configureReplicationMetrics(system: CorePowerSyncSystem) {
-  // Rate limit collection of these stats, since it may be an expensive query
-  const MINIMUM_INTERVAL = 60_000;
-
-  let cachedRequest: Promise<storage.StorageMetrics | null> | undefined = undefined;
-  let cacheTimestamp = 0;
-
-  function getMetrics() {
-    if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
-      cachedRequest = system.storage.getStorageMetrics().catch((e) => {
-        micro.logger.error(`Failed to get storage metrics`, e);
-        return null;
-      });
-      cacheTimestamp = Date.now();
-    }
-    return cachedRequest;
-  }
-
-  operation_storage_size_bytes.addCallback(async (result) => {
-    const metrics = await getMetrics();
-    if (metrics) {
-      result.observe(metrics.operations_size_bytes);
-    }
-  });
-
-  parameter_storage_size_bytes.addCallback(async (result) => {
-    const metrics = await getMetrics();
-    if (metrics) {
-      result.observe(metrics.parameters_size_bytes);
-    }
-  });
-
-  replication_storage_size_bytes.addCallback(async (result) => {
-    const metrics = await getMetrics();
-    if (metrics) {
-      result.observe(metrics.replication_size_bytes);
-    }
-  });
-
-  // Record replicated bytes using global jpgwire metrics.
-  jpgwire.setMetricsRecorder({
-    addBytesRead(bytes) {
-      data_replicated_bytes.add(bytes);
-    }
-  });
-}
-
-export async function getMetricValueForTests(name: string): Promise<number | undefined> {
-  const metrics = await exporter.collect();
-  const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics;
-  const metric = scoped.find((metric) => metric.descriptor.name == name);
-  if (metric == null) {
-    throw new Error(`Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}`);
-  }
-  const point = metric.dataPoints[metric.dataPoints.length - 1];
-  return point?.value as number;
-}