@powersync/service-module-postgres 0.19.3 → 0.19.4

package/src/types/registry.ts

@@ -1,275 +0,0 @@
-import {
-  applyValueContext,
-  CompatibilityContext,
-  CompatibilityOption,
-  CustomSqliteValue,
-  DatabaseInputValue,
-  SqliteValue,
-  SqliteValueType,
-  toSyncRulesValue
-} from '@powersync/service-sync-rules';
-import * as pgwire from '@powersync/service-jpgwire';
-
-interface BaseType {
-  sqliteType: () => SqliteValueType;
-}
-
-/** A type natively supported by {@link pgwire.PgType.decode}. */
-interface BuiltinType extends BaseType {
-  type: 'builtin';
-  oid: number;
-}
-
-/**
- * An array type.
- */
-interface ArrayType extends BaseType {
-  type: 'array';
-  innerId: number;
-  separatorCharCode: number;
-}
-
-/**
- * A domain type, like `CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`
- *
- * This type gets decoded and synced as the inner type (`FLOAT` in the example above).
- */
-interface DomainType extends BaseType {
-  type: 'domain';
-  innerId: number;
-}
-
-/**
- * A composite type as created by `CREATE TYPE AS`.
- *
- * These types are encoded as a tuple of values, so we recover attribute names to restore them as a JSON object.
- */
-interface CompositeType extends BaseType {
-  type: 'composite';
-  members: { name: string; typeId: number }[];
-}
-
-/**
- * A type created with `CREATE TYPE AS RANGE`.
- *
- * Ranges are represented as {@link pgwire.Range}. Multiranges are represented as arrays thereof.
- */
-interface RangeType extends BaseType {
-  type: 'range' | 'multirange';
-  innerId: number;
-}
-
-type KnownType = BuiltinType | ArrayType | DomainType | CompositeType | RangeType;
-
-interface UnknownType extends BaseType {
-  type: 'unknown';
-}
-
-type MaybeKnownType = KnownType | UnknownType;
-
-const UNKNOWN_TYPE: UnknownType = {
-  type: 'unknown',
-  sqliteType: () => 'text'
-};
-
-class CustomTypeValue extends CustomSqliteValue {
-  constructor(
-    readonly oid: number,
-    readonly cache: CustomTypeRegistry,
-    readonly rawValue: string
-  ) {
-    super();
-  }
-
-  private lookup(): KnownType | UnknownType {
-    return this.cache.lookupType(this.oid);
-  }
-
-  private decodeToDatabaseInputValue(context: CompatibilityContext): DatabaseInputValue {
-    if (context.isEnabled(CompatibilityOption.customTypes)) {
-      try {
-        return this.cache.decodeWithCustomTypes(this.rawValue, this.oid);
-      } catch (_e) {
-        return this.rawValue;
-      }
-    } else {
-      return pgwire.PgType.decode(this.rawValue, this.oid);
-    }
-  }
-
-  toSqliteValue(context: CompatibilityContext): SqliteValue {
-    const value = toSyncRulesValue(this.decodeToDatabaseInputValue(context));
-    return applyValueContext(value, context);
-  }
-
-  get sqliteType(): SqliteValueType {
-    return this.lookup().sqliteType();
-  }
-}
-
-/**
- * A registry of custom types.
- *
- * These extend the builtin decoding behavior in {@link pgwire.PgType.decode} for user-defined types like `DOMAIN`s or
- * composite types.
- */
-export class CustomTypeRegistry {
-  private readonly byOid: Map<number, KnownType>;
-
-  constructor() {
-    this.byOid = new Map();
-
-    for (const builtin of Object.values(pgwire.PgTypeOid)) {
-      if (typeof builtin == 'number') {
-        // We need to know the SQLite type of builtins to implement CustomSqliteValue.sqliteType for DOMAIN types.
-        let sqliteType: SqliteValueType;
-        switch (builtin) {
-          case pgwire.PgTypeOid.TEXT:
-          case pgwire.PgTypeOid.UUID:
-          case pgwire.PgTypeOid.VARCHAR:
-          case pgwire.PgTypeOid.DATE:
-          case pgwire.PgTypeOid.TIMESTAMP:
-          case pgwire.PgTypeOid.TIMESTAMPTZ:
-          case pgwire.PgTypeOid.TIME:
-          case pgwire.PgTypeOid.JSON:
-          case pgwire.PgTypeOid.JSONB:
-          case pgwire.PgTypeOid.PG_LSN:
-            sqliteType = 'text';
-            break;
-          case pgwire.PgTypeOid.BYTEA:
-            sqliteType = 'blob';
-            break;
-          case pgwire.PgTypeOid.BOOL:
-          case pgwire.PgTypeOid.INT2:
-          case pgwire.PgTypeOid.INT4:
-          case pgwire.PgTypeOid.OID:
-          case pgwire.PgTypeOid.INT8:
-            sqliteType = 'integer';
-            break;
-          case pgwire.PgTypeOid.FLOAT4:
-          case pgwire.PgTypeOid.FLOAT8:
-            sqliteType = 'real';
-            break;
-          default:
-            sqliteType = 'text';
-        }
-
-        this.byOid.set(builtin, {
-          type: 'builtin',
-          oid: builtin,
-          sqliteType: () => sqliteType
-        });
-      }
-    }
-
-    for (const [arrayId, innerId] of pgwire.ARRAY_TO_ELEM_OID.entries()) {
-      // We can just use the default decoder, except for box[] because those use a different delimiter. We don't fix
-      // this in PgType._decodeArray for backwards-compatibility.
-      if (innerId == 603) {
-        this.byOid.set(arrayId, {
-          type: 'array',
-          innerId,
-          sqliteType: () => 'text', // these get encoded as JSON arrays
-          separatorCharCode: 0x3b // ";"
-        });
-      } else {
-        this.byOid.set(arrayId, {
-          type: 'builtin',
-          oid: arrayId,
-          sqliteType: () => 'text' // these get encoded as JSON arrays
-        });
-      }
-    }
-  }
-
-  knows(oid: number): boolean {
-    return this.byOid.has(oid);
-  }
-
-  set(oid: number, value: KnownType) {
-    this.byOid.set(oid, value);
-  }
-
-  setDomainType(oid: number, inner: number) {
-    this.set(oid, {
-      type: 'domain',
-      innerId: inner,
-      sqliteType: () => this.lookupType(inner).sqliteType()
-    });
-  }
-
-  decodeWithCustomTypes(raw: string, oid: number): DatabaseInputValue {
-    const resolved = this.lookupType(oid);
-    switch (resolved.type) {
-      case 'builtin':
-      case 'unknown':
-        return pgwire.PgType.decode(raw, oid);
-      case 'domain':
-        return this.decodeWithCustomTypes(raw, resolved.innerId);
-      case 'composite': {
-        const parsed: [string, any][] = [];
-
-        new pgwire.StructureParser(raw).parseComposite((raw) => {
-          const nextMember = resolved.members[parsed.length];
-          if (nextMember) {
-            const value = raw == null ? null : this.decodeWithCustomTypes(raw, nextMember.typeId);
-            parsed.push([nextMember.name, value]);
-          }
-        });
-        return Object.fromEntries(parsed);
-      }
-      case 'array': {
-        // Nornalize "array of array of T" types into just "array of T", because Postgres arrays are natively multi-
-        // dimensional. This may be required when we have a DOMAIN wrapper around an array followed by another array
-        // around that domain.
-        let innerId = resolved.innerId;
-        while (true) {
-          const resolvedInner = this.lookupType(innerId);
-          if (resolvedInner.type == 'domain') {
-            innerId = resolvedInner.innerId;
-          } else if (resolvedInner.type == 'array') {
-            innerId = resolvedInner.innerId;
-          } else {
-            break;
-          }
-        }
-
-        return new pgwire.StructureParser(raw).parseArray(
-          (source) => this.decodeWithCustomTypes(source, innerId),
-          resolved.separatorCharCode
-        );
-      }
-      case 'range':
-        return new pgwire.StructureParser(raw).parseRange((s) => this.decodeWithCustomTypes(s, resolved.innerId));
-      case 'multirange':
-        return new pgwire.StructureParser(raw).parseMultiRange((s) => this.decodeWithCustomTypes(s, resolved.innerId));
-    }
-  }
-
-  lookupType(type: number): KnownType | UnknownType {
-    return this.byOid.get(type) ?? UNKNOWN_TYPE;
-  }
-
-  private isParsedWithoutCustomTypesSupport(type: MaybeKnownType): boolean {
-    switch (type.type) {
-      case 'builtin':
-      case 'unknown':
-        return true;
-      case 'array':
-        return type.separatorCharCode == pgwire.CHAR_CODE_COMMA && pgwire.ARRAY_TO_ELEM_OID.has(type.innerId);
-      default:
-        return false;
-    }
-  }
-
-  decodeDatabaseValue(value: string, oid: number): DatabaseInputValue {
-    const resolved = this.lookupType(oid);
-    // For backwards-compatibility, some types are only properly parsed with a compatibility option. Others are synced
-    // in the raw text representation by default, and are only parsed as JSON values when necessary.
-    if (this.isParsedWithoutCustomTypesSupport(resolved)) {
-      return pgwire.PgType.decode(value, oid);
-    } else {
-      return new CustomTypeValue(oid, this, value);
-    }
-  }
-}

package/src/types/resolver.ts

@@ -1,227 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules';
-import semver from 'semver';
-import { getServerVersion } from '../utils/postgres_version.js';
-import { CustomTypeRegistry } from './registry.js';
-
-/**
- * Resolves descriptions used to decode values for custom postgres types.
- *
- * Custom types are resolved from the source database, which also involves crawling inner types (e.g. for composites).
- */
-export class PostgresTypeResolver {
-  private cachedVersion: semver.SemVer | null = null;
-  readonly registry: CustomTypeRegistry;
-
-  constructor(private readonly pool: pgwire.PgClient) {
-    this.registry = new CustomTypeRegistry();
-  }
-
-  private async fetchVersion(): Promise<semver.SemVer> {
-    if (this.cachedVersion == null) {
-      this.cachedVersion = (await getServerVersion(this.pool)) ?? semver.parse('0.0.1');
-    }
-
-    return this.cachedVersion!;
-  }
-
-  /**
-   * @returns Whether the Postgres instance this type cache is connected to has support for the multirange type (which
-   * is the case for Postgres 14 and later).
-   */
-  async supportsMultiRanges() {
-    const version = await this.fetchVersion();
-    return version.compare(PostgresTypeResolver.minVersionForMultirange) >= 0;
-  }
-
-  /**
-   * Fetches information about indicated types.
-   *
-   * If a type references another custom type (e.g. because it's a composite type with a custom field), these are
-   * automatically crawled as well.
-   */
-  public async fetchTypes(oids: number[]) {
-    const multiRangeSupport = await this.supportsMultiRanges();
-
-    let pending = oids.filter((id) => !this.registry.knows(id));
-    // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html
-    const multiRangeDesc = `WHEN 'm' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngmultitypid = t.oid))`;
-    const statement = `
-SELECT oid, t.typtype,
-    CASE t.typtype
-        WHEN 'b' THEN json_build_object('element_type', t.typelem, 'delim', (SELECT typdelim FROM pg_type i WHERE i.oid = t.typelem))
-        WHEN 'd' THEN json_build_object('type', t.typbasetype)
-        WHEN 'c' THEN json_build_object(
-            'elements',
-            (SELECT json_agg(json_build_object('name', a.attname, 'type', a.atttypid))
-                FROM pg_attribute a
-                WHERE a.attrelid = t.typrelid)
-        )
-        WHEN 'r' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngtypid = t.oid))
-        ${multiRangeSupport ? multiRangeDesc : ''}
-        ELSE NULL
-    END AS desc
-FROM pg_type t
-WHERE t.oid = ANY($1)
-`;
-
-    while (pending.length != 0) {
-      // 1016: int8 array
-      const query = await this.pool.query({ statement, params: [{ type: 1016, value: pending }] });
-      const stillPending: number[] = [];
-
-      const requireType = (oid: number) => {
-        if (!this.registry.knows(oid) && !pending.includes(oid) && !stillPending.includes(oid)) {
-          stillPending.push(oid);
-        }
-      };
-
-      for (const row of pgwire.pgwireRows(query)) {
-        const oid = Number(row.oid);
-        const desc = JSON.parse(row.desc);
-
-        switch (row.typtype) {
-          case 'b':
-            const { element_type, delim } = desc;
-
-            if (!this.registry.knows(oid)) {
-              // This type is an array of another custom type.
-              const inner = Number(element_type);
-              if (inner != 0) {
-                // Some array types like macaddr[] don't seem to have their inner type set properly - skip!
-                requireType(inner);
-                this.registry.set(oid, {
-                  type: 'array',
-                  innerId: inner,
-                  separatorCharCode: (delim as string).charCodeAt(0),
-                  sqliteType: () => 'text' // Since it's JSON
-                });
-              }
-            }
-            break;
-          case 'c':
-            // For composite types, we sync the JSON representation.
-            const elements: { name: string; typeId: number }[] = [];
-            for (const { name, type } of desc.elements) {
-              const typeId = Number(type);
-              elements.push({ name, typeId });
-              requireType(typeId);
-            }
-
-            this.registry.set(oid, {
-              type: 'composite',
-              members: elements,
-              sqliteType: () => 'text' // Since it's JSON
-            });
-            break;
-          case 'd':
-            // For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync
-            // the inner type (pg_type.typbasetype).
-            const inner = Number(desc.type);
-            this.registry.setDomainType(oid, inner);
-            requireType(inner);
-            break;
-          case 'r':
-          case 'm': {
-            const inner = Number(desc.inner);
-            this.registry.set(oid, {
-              type: row.typtype == 'r' ? 'range' : 'multirange',
-              innerId: inner,
-              sqliteType: () => 'text' // Since it's JSON
-            });
-          }
-        }
-      }
-
-      pending = stillPending;
-    }
-  }
-
-  /**
-   * Crawls all custom types referenced by table columns in the current database.
-   */
-  public async fetchTypesForSchema() {
-    const sql = `
-SELECT DISTINCT a.atttypid AS type_oid
-FROM pg_attribute a
-JOIN pg_class c ON c.oid = a.attrelid
-JOIN pg_namespace cn ON cn.oid = c.relnamespace
-JOIN pg_type t ON t.oid = a.atttypid
-JOIN pg_namespace tn ON tn.oid = t.typnamespace
-WHERE a.attnum > 0
-  AND NOT a.attisdropped
-  AND cn.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
-    `;
-
-    const query = await this.pool.query(sql);
-    let ids: number[] = [];
-    for (const row of pgwire.pgwireRows(query)) {
-      ids.push(Number(row.type_oid));
-    }
-
-    await this.fetchTypes(ids);
-  }
-
-  /**
-   * pgwire message -> SQLite row.
-   * @param message
-   */
-  constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteInputRow {
-    const rawData = (message as any).afterRaw;
-
-    const record = this.decodeTuple(message.relation, rawData);
-    return toSyncRulesRow(record);
-  }
-
-  /**
-   * pgwire message -> SQLite row.
-   * @param message
-   */
-  constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteInputRow | undefined {
-    const rawData = (message as any).beforeRaw;
-    if (rawData == null) {
-      return undefined;
-    }
-    const record = this.decodeTuple(message.relation, rawData);
-    return toSyncRulesRow(record);
-  }
-
-  /**
-   * We need a high level of control over how values are decoded, to make sure there is no loss
-   * of precision in the process.
-   */
-  decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record<string, any>): DatabaseInputRow {
-    let result: Record<string, any> = {};
-    for (const column of relation.columns) {
-      const rawval = tupleRaw[column.name];
-      result[column.name] =
-        rawval == null
-          ? // We can't decode null values, but it's important that null and undefined stay distinct because undefined
-            // represents a TOASTed value.
-            rawval
-          : this.registry.decodeDatabaseValue(rawval, column.typeOid);
-    }
-
-    return result;
-  }
-
-  /**
-   * We need a high level of control over how values are decoded, to make sure there is no loss
-   * of precision in the process.
-   */
-  private decodeTupleForTable(columnMap: Record<string, number>, tupleRaw: Record<string, any>): DatabaseInputRow {
-    let result: Record<string, any> = {};
-    for (let columnName in tupleRaw) {
-      const rawval = tupleRaw[columnName];
-      const typeOid = columnMap[columnName];
-      if (typeof rawval == 'string' && typeOid) {
-        result[columnName] = this.registry.decodeDatabaseValue(rawval, typeOid);
-      } else {
-        result[columnName] = rawval;
-      }
-    }
-    return result;
-  }
-
-  private static minVersionForMultirange: semver.SemVer = semver.parse('14.0.0')!;
-}

package/src/types/types.ts

@@ -1,44 +0,0 @@
-import * as lib_postgres from '@powersync/lib-service-postgres';
-import * as service_types from '@powersync/service-types';
-import * as t from 'ts-codec';
-
-// Maintain backwards compatibility by exporting these
-export const validatePort = lib_postgres.validatePort;
-export const baseUri = lib_postgres.baseUri;
-export type NormalizedPostgresConnectionConfig = lib_postgres.NormalizedBasePostgresConnectionConfig;
-export const POSTGRES_CONNECTION_TYPE = lib_postgres.POSTGRES_CONNECTION_TYPE;
-
-export const PostgresConnectionConfig = service_types.configFile.DataSourceConfig.and(
-  lib_postgres.BasePostgresConnectionConfig
-).and(
-  t.object({
-    // Add any replication connection specific config here in future
-  })
-);
-
-/**
- * Config input specified when starting services
- */
-export type PostgresConnectionConfig = t.Decoded<typeof PostgresConnectionConfig>;
-
-/**
- * Resolved version of {@link PostgresConnectionConfig}
- */
-export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig;
-
-export function isPostgresConfig(
-  config: service_types.configFile.DataSourceConfig
-): config is PostgresConnectionConfig {
-  return config.type == lib_postgres.POSTGRES_CONNECTION_TYPE;
-}
-
-/**
- * Validate and normalize connection options.
- *
- * Returns destructured options.
- */
-export function normalizeConnectionConfig(options: PostgresConnectionConfig) {
-  return {
-    ...lib_postgres.normalizeConnectionConfig(options)
-  } satisfies NormalizedPostgresConnectionConfig;
-}

package/src/utils/application-name.ts

@@ -1,8 +0,0 @@
-import { POWERSYNC_VERSION } from '@powersync/service-core';
-
-/**
- * application_name for PostgreSQL connections to the source database
- */
-export function getApplicationName() {
-  return `powersync/${POWERSYNC_VERSION}`;
-}

package/src/utils/migration_lib.ts

@@ -1,80 +0,0 @@
-import { ServiceAssertionError } from '@powersync/lib-services-framework';
-import * as pgwire from '@powersync/service-jpgwire';
-
-export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
-
-interface Migration {
-  id: number;
-  name: string;
-  up: MigrationFunction;
-}
-
-// Very loosely based on https://github.com/porsager/postgres-shift/
-export class Migrations {
-  private migrations: Migration[] = [];
-
-  add(id: number, name: string, up: MigrationFunction) {
-    if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
-      throw new ServiceAssertionError('Migration ids must be strictly incrementing');
-    }
-    this.migrations.push({ id, up, name });
-  }
-
-  async up(db: pgwire.PgConnection) {
-    await db.query('BEGIN');
-    try {
-      await this.ensureMigrationsTable(db);
-      const current = await this.getCurrentMigration(db);
-      let currentId = current ? current.id : 0;
-
-      for (let migration of this.migrations) {
-        if (migration.id <= currentId) {
-          continue;
-        }
-        await migration.up(db);
-
-        await db.query({
-          statement: `
-      insert into migrations (
-        migration_id,
-        name
-      ) values (
-        $1,
-        $2
-      )
-    `,
-          params: [
-            { type: 'int4', value: migration.id },
-            { type: 'varchar', value: migration.name }
-          ]
-        });
-      }
-
-      await db.query('COMMIT');
-    } catch (e) {
-      await db.query('ROLLBACK');
-      throw e;
-    }
-  }
-
-  getCurrentMigration(db: pgwire.PgConnection) {
-    return db
-      .query(
-        `
-      select migration_id as id from migrations
-      order by migration_id desc
-      limit 1
-    `
-      )
-      .then((results) => ({ id: results.rows[0].decodeWithoutCustomTypes(0) as number }));
-  }
-
-  async ensureMigrationsTable(db: pgwire.PgConnection) {
-    await db.query(`create table if not exists migrations (
-        migration_id serial primary key,
-        created_at timestamp with time zone not null default now(),
-        name text
-      )
-    `);
-  }
-}

package/src/utils/populate_test_data.ts

@@ -1,37 +0,0 @@
-import { Worker } from 'node:worker_threads';
-
-import * as pgwire from '@powersync/service-jpgwire';
-
-// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
-
-export interface PopulateDataOptions {
-  connection: pgwire.NormalizedConnectionConfig;
-  num_transactions: number;
-  per_transaction: number;
-  size: number;
-}
-
-export async function populateData(options: PopulateDataOptions) {
-  const WORKER_TIMEOUT = 30_000;
-
-  const worker = new Worker(new URL('./populate_test_data_worker.js', import.meta.url), {
-    workerData: options
-  });
-  const timeout = setTimeout(() => {
-    // Exits with code 1 below
-    worker.terminate();
-  }, WORKER_TIMEOUT);
-  try {
-    return await new Promise<number>((resolve, reject) => {
-      worker.on('message', resolve);
-      worker.on('error', reject);
-      worker.on('exit', (code) => {
-        if (code !== 0) {
-          reject(new Error(`Populating data failed with exit code ${code}`));
-        }
-      });
-    });
-  } finally {
-    clearTimeout(timeout);
-  }
-}