@powersync/service-module-postgres 0.16.1 → 0.16.3

package/src/types/registry.ts

@@ -0,0 +1,278 @@
+import {
+  applyValueContext,
+  CompatibilityContext,
+  CompatibilityOption,
+  CustomSqliteValue,
+  DatabaseInputValue,
+  SqliteValue,
+  SqliteValueType,
+  toSyncRulesValue
+} from '@powersync/service-sync-rules';
+import * as pgwire from '@powersync/service-jpgwire';
+
+interface BaseType {
+  sqliteType: () => SqliteValueType;
+}
+
+/** A type natively supported by {@link pgwire.PgType.decode}. */
+interface BuiltinType extends BaseType {
+  type: 'builtin';
+  oid: number;
+}
+
+/**
+ * An array type.
+ */
+interface ArrayType extends BaseType {
+  type: 'array';
+  innerId: number;
+  separatorCharCode: number;
+}
+
+/**
+ * A domain type, like `CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`
+ *
+ * This type gets decoded and synced as the inner type (`FLOAT` in the example above).
+ */
+interface DomainType extends BaseType {
+  type: 'domain';
+  innerId: number;
+}
+
+/**
+ * A composite type as created by `CREATE TYPE AS`.
+ *
+ * These types are encoded as a tuple of values, so we recover attribute names to restore them as a JSON object.
+ */
+interface CompositeType extends BaseType {
+  type: 'composite';
+  members: { name: string; typeId: number }[];
+}
+
+/**
+ * A type created with `CREATE TYPE AS RANGE`.
+ *
+ * Ranges are represented as {@link pgwire.Range}. Multiranges are represented as arrays thereof.
+ */
+interface RangeType extends BaseType {
+  type: 'range' | 'multirange';
+  innerId: number;
+}
+
+type KnownType = BuiltinType | ArrayType | DomainType | DomainType | CompositeType | RangeType;
+
+interface UnknownType extends BaseType {
+  type: 'unknown';
+}
+
+type MaybeKnownType = KnownType | UnknownType;
+
+const UNKNOWN_TYPE: UnknownType = {
+  type: 'unknown',
+  sqliteType: () => 'text'
+};
+
+class CustomTypeValue extends CustomSqliteValue {
+  constructor(
+    readonly oid: number,
+    readonly cache: CustomTypeRegistry,
+    readonly rawValue: string
+  ) {
+    super();
+  }
+
+  private lookup(): KnownType | UnknownType {
+    return this.cache.lookupType(this.oid);
+  }
+
+  private decodeToDatabaseInputValue(context: CompatibilityContext): DatabaseInputValue {
+    if (context.isEnabled(CompatibilityOption.customTypes)) {
+      try {
+        return this.cache.decodeWithCustomTypes(this.rawValue, this.oid);
+      } catch (_e) {
+        return this.rawValue;
+      }
+    } else {
+      return pgwire.PgType.decode(this.rawValue, this.oid);
+    }
+  }
+
+  toSqliteValue(context: CompatibilityContext): SqliteValue {
+    const value = toSyncRulesValue(this.decodeToDatabaseInputValue(context));
+    return applyValueContext(value, context);
+  }
+
+  get sqliteType(): SqliteValueType {
+    return this.lookup().sqliteType();
+  }
+}
+
+/**
+ * A registry of custom types.
+ *
+ * These extend the builtin decoding behavior in {@link pgwire.PgType.decode} for user-defined types like `DOMAIN`s or
+ * composite types.
+ */
+export class CustomTypeRegistry {
+  private readonly byOid: Map<number, KnownType>;
+
+  constructor() {
+    this.byOid = new Map();
+
+    for (const builtin of Object.values(pgwire.PgTypeOid)) {
+      if (typeof builtin == 'number') {
+        // We need to know the SQLite type of builtins to implement CustomSqliteValue.sqliteType for DOMAIN types.
+        let sqliteType: SqliteValueType;
+        switch (builtin) {
+          case pgwire.PgTypeOid.TEXT:
+          case pgwire.PgTypeOid.UUID:
+          case pgwire.PgTypeOid.VARCHAR:
+          case pgwire.PgTypeOid.DATE:
+          case pgwire.PgTypeOid.TIMESTAMP:
+          case pgwire.PgTypeOid.TIMESTAMPTZ:
+          case pgwire.PgTypeOid.TIME:
+          case pgwire.PgTypeOid.JSON:
+          case pgwire.PgTypeOid.JSONB:
+          case pgwire.PgTypeOid.PG_LSN:
+            sqliteType = 'text';
+            break;
+          case pgwire.PgTypeOid.BYTEA:
+            sqliteType = 'blob';
+            break;
+          case pgwire.PgTypeOid.BOOL:
+          case pgwire.PgTypeOid.INT2:
+          case pgwire.PgTypeOid.INT4:
+          case pgwire.PgTypeOid.OID:
+          case pgwire.PgTypeOid.INT8:
+            sqliteType = 'integer';
+            break;
+          case pgwire.PgTypeOid.FLOAT4:
+          case pgwire.PgTypeOid.FLOAT8:
+            sqliteType = 'real';
+            break;
+          default:
+            sqliteType = 'text';
+        }
+
+        this.byOid.set(builtin, {
+          type: 'builtin',
+          oid: builtin,
+          sqliteType: () => sqliteType
+        });
+      }
+    }
+
+    for (const [arrayId, innerId] of pgwire.ARRAY_TO_ELEM_OID.entries()) {
+      // We can just use the default decoder, except for box[] because those use a different delimiter. We don't fix
+      // this in PgType._decodeArray for backwards-compatibility.
+      if (innerId == 603) {
+        this.byOid.set(arrayId, {
+          type: 'array',
+          innerId,
+          sqliteType: () => 'text', // these get encoded as JSON arrays
+          separatorCharCode: 0x3b // ";"
+        });
+      } else {
+        this.byOid.set(arrayId, {
+          type: 'builtin',
+          oid: arrayId,
+          sqliteType: () => 'text' // these get encoded as JSON arrays
+        });
+      }
+    }
+  }
+
+  knows(oid: number): boolean {
+    return this.byOid.has(oid);
+  }
+
+  set(oid: number, value: KnownType) {
+    this.byOid.set(oid, value);
+  }
+
+  setDomainType(oid: number, inner: number) {
+    this.set(oid, {
+      type: 'domain',
+      innerId: inner,
+      sqliteType: () => this.lookupType(inner).sqliteType()
+    });
+  }
+
+  decodeWithCustomTypes(raw: string, oid: number): DatabaseInputValue {
+    const resolved = this.lookupType(oid);
+    switch (resolved.type) {
+      case 'builtin':
+      case 'unknown':
+        return pgwire.PgType.decode(raw, oid);
+      case 'domain':
+        return this.decodeWithCustomTypes(raw, resolved.innerId);
+      case 'composite': {
+        const parsed: [string, any][] = [];
+
+        new pgwire.StructureParser(raw).parseComposite((raw) => {
+          const nextMember = resolved.members[parsed.length];
+          if (nextMember) {
+            const value = raw == null ? null : this.decodeWithCustomTypes(raw, nextMember.typeId);
+            parsed.push([nextMember.name, value]);
+          }
+        });
+        return Object.fromEntries(parsed);
+      }
+      case 'array': {
+        // Nornalize "array of array of T" types into just "array of T", because Postgres arrays are natively multi-
+        // dimensional. This may be required when we have a DOMAIN wrapper around an array followed by another array
+        // around that domain.
+        let innerId = resolved.innerId;
+        while (true) {
+          const resolvedInner = this.lookupType(innerId);
+          if (resolvedInner.type == 'domain') {
+            innerId = resolvedInner.innerId;
+          } else if (resolvedInner.type == 'array') {
+            innerId = resolvedInner.innerId;
+          } else {
+            break;
+          }
+        }
+
+        return new pgwire.StructureParser(raw).parseArray(
+          (source) => this.decodeWithCustomTypes(source, innerId),
+          resolved.separatorCharCode
+        );
+      }
+      case 'range':
+        return new pgwire.StructureParser(raw).parseRange((s) => this.decodeWithCustomTypes(s, resolved.innerId));
+      case 'multirange':
+        return new pgwire.StructureParser(raw).parseMultiRange((s) => this.decodeWithCustomTypes(s, resolved.innerId));
+    }
+  }
+
+  lookupType(type: number): KnownType | UnknownType {
+    return this.byOid.get(type) ?? UNKNOWN_TYPE;
+  }
+
+  private isParsedWithoutCustomTypesSupport(type: MaybeKnownType): boolean {
+    switch (type.type) {
+      case 'builtin':
+      case 'unknown':
+        return true;
+      case 'array':
+        return (
+          type.separatorCharCode == pgwire.CHAR_CODE_COMMA &&
+          this.isParsedWithoutCustomTypesSupport(this.lookupType(type.innerId))
+        );
+      default:
+        return false;
+    }
+  }
+
+  decodeDatabaseValue(value: string, oid: number): DatabaseInputValue {
+    const resolved = this.lookupType(oid);
+    // For backwards-compatibility, some types are only properly parsed with a compatibility option. Others are synced
+    // in the raw text representation by default, and are only parsed as JSON values when necessary.
+    if (this.isParsedWithoutCustomTypesSupport(resolved)) {
+      return pgwire.PgType.decode(value, oid);
+    } else {
+      return new CustomTypeValue(oid, this, value);
+    }
+  }
+}

package/src/types/resolver.ts

@@ -0,0 +1,210 @@
+import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules';
+import * as pgwire from '@powersync/service-jpgwire';
+import { CustomTypeRegistry } from './registry.js';
+import semver from 'semver';
+import { getServerVersion } from '../utils/postgres_version.js';
+
+/**
+ * Resolves descriptions used to decode values for custom postgres types.
+ *
+ * Custom types are resolved from the source database, which also involves crawling inner types (e.g. for composites).
+ */
+export class PostgresTypeResolver {
+  private cachedVersion: semver.SemVer | null = null;
+
+  constructor(
+    readonly registry: CustomTypeRegistry,
+    private readonly pool: pgwire.PgClient
+  ) {
+    this.registry = new CustomTypeRegistry();
+  }
+
+  private async fetchVersion(): Promise<semver.SemVer> {
+    if (this.cachedVersion == null) {
+      this.cachedVersion = (await getServerVersion(this.pool)) ?? semver.parse('0.0.1');
+    }
+
+    return this.cachedVersion!;
+  }
+
+  /**
+   * @returns Whether the Postgres instance this type cache is connected to has support for the multirange type (which
+   * is the case for Postgres 14 and later).
+   */
+  async supportsMultiRanges() {
+    const version = await this.fetchVersion();
+    return version.compare(PostgresTypeResolver.minVersionForMultirange) >= 0;
+  }
+
+  /**
+   * Fetches information about indicated types.
+   *
+   * If a type references another custom type (e.g. because it's a composite type with a custom field), these are
+   * automatically crawled as well.
+   */
+  public async fetchTypes(oids: number[]) {
+    const multiRangeSupport = await this.supportsMultiRanges();
+
+    let pending = oids.filter((id) => !this.registry.knows(id));
+    // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html
+    const multiRangeDesc = `WHEN 'm' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngmultitypid = t.oid))`;
+    const statement = `
+SELECT oid, t.typtype,
+    CASE t.typtype
+        WHEN 'b' THEN json_build_object('element_type', t.typelem, 'delim', (SELECT typdelim FROM pg_type i WHERE i.oid = t.typelem))
+        WHEN 'd' THEN json_build_object('type', t.typbasetype)
+        WHEN 'c' THEN json_build_object(
+            'elements',
+            (SELECT json_agg(json_build_object('name', a.attname, 'type', a.atttypid))
+                FROM pg_attribute a
+                WHERE a.attrelid = t.typrelid)
+        )
+        WHEN 'r' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngtypid = t.oid))
+        ${multiRangeSupport ? multiRangeDesc : ''}
+        ELSE NULL
+    END AS desc
+FROM pg_type t
+WHERE t.oid = ANY($1)
+`;
+
+    while (pending.length != 0) {
+      // 1016: int8 array
+      const query = await this.pool.query({ statement, params: [{ type: 1016, value: pending }] });
+      const stillPending: number[] = [];
+
+      const requireType = (oid: number) => {
+        if (!this.registry.knows(oid) && !pending.includes(oid) && !stillPending.includes(oid)) {
+          stillPending.push(oid);
+        }
+      };
+
+      for (const row of pgwire.pgwireRows(query)) {
+        const oid = Number(row.oid);
+        const desc = JSON.parse(row.desc);
+
+        switch (row.typtype) {
+          case 'b':
+            const { element_type, delim } = desc;
+
+            if (!this.registry.knows(oid)) {
+              // This type is an array of another custom type.
+              const inner = Number(element_type);
+              if (inner != 0) {
+                // Some array types like macaddr[] don't seem to have their inner type set properly - skip!
+                requireType(inner);
+                this.registry.set(oid, {
+                  type: 'array',
+                  innerId: inner,
+                  separatorCharCode: (delim as string).charCodeAt(0),
+                  sqliteType: () => 'text' // Since it's JSON
+                });
+              }
+            }
+            break;
+          case 'c':
+            // For composite types, we sync the JSON representation.
+            const elements: { name: string; typeId: number }[] = [];
+            for (const { name, type } of desc.elements) {
+              const typeId = Number(type);
+              elements.push({ name, typeId });
+              requireType(typeId);
+            }
+
+            this.registry.set(oid, {
+              type: 'composite',
+              members: elements,
+              sqliteType: () => 'text' // Since it's JSON
+            });
+            break;
+          case 'd':
+            // For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync
+            // the inner type (pg_type.typbasetype).
+            const inner = Number(desc.type);
+            this.registry.setDomainType(oid, inner);
+            requireType(inner);
+            break;
+          case 'r':
+          case 'm': {
+            const inner = Number(desc.inner);
+            this.registry.set(oid, {
+              type: row.typtype == 'r' ? 'range' : 'multirange',
+              innerId: inner,
+              sqliteType: () => 'text' // Since it's JSON
+            });
+          }
+        }
+      }
+
+      pending = stillPending;
+    }
+  }
+
+  /**
+   * Crawls all custom types referenced by table columns in the current database.
+   */
+  public async fetchTypesForSchema() {
+    const sql = `
+SELECT DISTINCT a.atttypid AS type_oid
+FROM pg_attribute a
+JOIN pg_class c ON c.oid = a.attrelid
+JOIN pg_namespace cn ON cn.oid = c.relnamespace
+JOIN pg_type t ON t.oid = a.atttypid
+JOIN pg_namespace tn ON tn.oid = t.typnamespace
+WHERE a.attnum > 0
+  AND NOT a.attisdropped
+  AND cn.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
+    `;
+
+    const query = await this.pool.query({ statement: sql });
+    let ids: number[] = [];
+    for (const row of pgwire.pgwireRows(query)) {
+      ids.push(Number(row.type_oid));
+    }
+
+    await this.fetchTypes(ids);
+  }
+
+  /**
+   * pgwire message -> SQLite row.
+   * @param message
+   */
+  constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteInputRow {
+    const rawData = (message as any).afterRaw;
+
+    const record = this.decodeTuple(message.relation, rawData);
+    return toSyncRulesRow(record);
+  }
+
+  /**
+   * pgwire message -> SQLite row.
+   * @param message
+   */
+  constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteInputRow | undefined {
+    const rawData = (message as any).beforeRaw;
+    if (rawData == null) {
+      return undefined;
+    }
+    const record = this.decodeTuple(message.relation, rawData);
+    return toSyncRulesRow(record);
+  }
+
+  /**
+   * We need a high level of control over how values are decoded, to make sure there is no loss
+   * of precision in the process.
+   */
+  decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record<string, any>): DatabaseInputRow {
+    let result: Record<string, any> = {};
+    for (let columnName in tupleRaw) {
+      const rawval = tupleRaw[columnName];
+      const typeOid = (relation as any)._tupleDecoder._typeOids.get(columnName);
+      if (typeof rawval == 'string' && typeOid) {
+        result[columnName] = this.registry.decodeDatabaseValue(rawval, typeOid);
+      } else {
+        result[columnName] = rawval;
+      }
+    }
+    return result;
+  }
+
+  private static minVersionForMultirange: semver.SemVer = semver.parse('14.0.0')!;
+}

package/src/types/types.ts

@@ -1,6 +1,7 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
+import { CustomTypeRegistry } from './registry.js';
 
 // Maintain backwards compatibility by exporting these
 export const validatePort = lib_postgres.validatePort;
@@ -24,7 +25,10 @@ export type PostgresConnectionConfig = t.Decoded<typeof PostgresConnectionConfig
 /**
  * Resolved version of {@link PostgresConnectionConfig}
  */
-export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig;
+export type ResolvedConnectionConfig = PostgresConnectionConfig &
+  NormalizedPostgresConnectionConfig & {
+    typeRegistry: CustomTypeRegistry;
+  };
 
 export function isPostgresConfig(
   config: service_types.configFile.DataSourceConfig

package/src/utils/postgres_version.ts

@@ -0,0 +1,8 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import semver, { type SemVer } from 'semver';
+
+export async function getServerVersion(db: pgwire.PgClient): Promise<SemVer | null> {
+  const result = await db.query(`SHOW server_version;`);
+  // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
+  return semver.coerce(result.rows[0][0].split(' ')[0]);
+}

package/test/src/checkpoints.test.ts

@@ -23,11 +23,13 @@ const checkpointTests = (factory: TestStorageFactory) => {
     const { pool } = context;
     const api = new PostgresRouteAPIAdapter(pool);
     const serverVersion = await context.connectionManager.getServerVersion();
-    if (serverVersion!.compareMain('13.0.0') < 0) {
+    if (serverVersion!.compareMain('14.0.0') < 0) {
       // The test is not stable on Postgres 11 or 12. See the notes on
       // PostgresRouteAPIAdapter.createReplicationHead() for details.
       // Postgres 12 is already EOL, so not worth finding a fix - just skip the tests.
-      console.log('Skipping write checkpoint test on Postgres < 13.0.0');
+      // Postgres 13 fares a little better, but even there the test is still unstable.
+      // Postgres 14+ appears to have no issue.
+      console.log('Skipping write checkpoint test on Postgres < 14.0.0');
       return;
     }