@powersync/service-module-postgres 0.16.15 → 0.16.16

package/dist/utils/migration_lib.js

@@ -49,7 +49,7 @@ export class Migrations {
       order by migration_id desc
       limit 1
     `)
-            .then((results) => ({ id: results.rows[0][0] }));
+            .then((results) => ({ id: results.rows[0].decodeWithoutCustomTypes(0) }));
     }
     async ensureMigrationsTable(db) {
         await db.query(`create table if not exists migrations (

package/dist/utils/migration_lib.js.map

@@ -1 +1 @@
-{"version":3,"file":"migration_lib.js","sourceRoot":"","sources":["../../src/utils/migration_lib.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAW1E,oEAAoE;AACpE,MAAM,OAAO,UAAU;IACb,UAAU,GAAgB,EAAE,CAAC;IAErC,GAAG,CAAC,EAAU,EAAE,IAAY,EAAE,EAAqB;QACjD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;YACvF,MAAM,IAAI,qBAAqB,CAAC,6CAA6C,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,EAAuB;QAC9B,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAEzC,KAAK,IAAI,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACtC,IAAI,SAAS,CAAC,EAAE,IAAI,SAAS,EAAE,CAAC;oBAC9B,SAAS;gBACX,CAAC;gBACD,MAAM,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAEvB,MAAM,EAAE,CAAC,KAAK,CAAC;oBACb,SAAS,EAAE;;;;;;;;KAQhB;oBACK,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE;wBACrC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE;qBAC3C;iBACF,CAAC,CAAC;YACL,CAAC;YAED,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC3B,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,mBAAmB,CAAC,EAAuB;QACzC,OAAO,EAAE;aACN,KAAK,CACJ;;;;KAIH,CACE;aACA,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAW,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,EAAuB;QACjD,MAAM,EAAE,CAAC,KAAK,CAAC;;;;;KAKd,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"file":"migration_lib.js","sourceRoot":"","sources":["../../src/utils/migration_lib.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAW1E,oEAAoE;AACpE,MAAM,OAAO,UAAU;IACb,UAAU,GAAgB,EAAE,CAAC;IAErC,GAAG,CAAC,EAAU,EAAE,IAAY,EAAE,EAAqB;QACjD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;YACvF,MAAM,IAAI,qBAAqB,CAAC,6CAA6C,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,EAAuB;QAC9B,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAEzC,KAAK,IAAI,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACtC,IAAI,SAAS,CAAC,EAAE,IAAI,SAAS,EAAE,CAAC;oBAC9B,SAAS;gBACX,CAAC;gBACD,MAAM,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAEvB,MAAM,EAAE,CAAC,KAAK,CAAC;oBACb,SAAS,EAAE;;;;;;;;KAQhB;oBACK,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE;wBACrC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE;qBAC3C;iBACF,CAAC,CAAC;YACL,CAAC;YAED,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC3B,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,mBAAmB,CAAC,EAAuB;QACzC,OAAO,EAAE;aACN,KAAK,CACJ;;;;KAIH,CACE;aACA,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAW,EAAE,CAAC,CAAC,CAAC;IACxF,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,EAAuB;QACjD,MAAM,EAAE,CAAC,KAAK,CAAC;;;;;KAKd,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/utils/postgres_version.js

@@ -2,6 +2,6 @@ import semver from 'semver';
 export async function getServerVersion(db) {
     const result = await db.query(`SHOW server_version;`);
     // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
-    return semver.coerce(result.rows[0][0].split(' ')[0]);
+    return semver.coerce(result.rows[0].decodeWithoutCustomTypes(0).split(' ')[0]);
 }
 //# sourceMappingURL=postgres_version.js.map

package/dist/utils/postgres_version.js.map

@@ -1 +1 @@
-{"version":3,"file":"postgres_version.js","sourceRoot":"","sources":["../../src/utils/postgres_version.ts"],"names":[],"mappings":"AACA,OAAO,MAAuB,MAAM,QAAQ,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAmB;IACxD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtD,qEAAqE;IACrE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC"}
+{"version":3,"file":"postgres_version.js","sourceRoot":"","sources":["../../src/utils/postgres_version.ts"],"names":[],"mappings":"AACA,OAAO,MAAuB,MAAM,QAAQ,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAmB;IACxD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtD,qEAAqE;IACrE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC"}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.16.15",
+  "version": "0.16.16",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -28,20 +28,20 @@
     "ts-codec": "^1.3.0",
     "uri-js": "^4.4.1",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-postgres": "0.4.17",
-    "@powersync/lib-services-framework": "0.7.13",
-    "@powersync/service-core": "1.18.1",
-    "@powersync/service-jpgwire": "0.21.8",
+    "@powersync/lib-service-postgres": "0.4.18",
+    "@powersync/lib-services-framework": "0.7.14",
+    "@powersync/service-core": "1.18.2",
+    "@powersync/service-jpgwire": "0.21.9",
     "@powersync/service-jsonbig": "0.17.12",
-    "@powersync/service-sync-rules": "0.29.9",
+    "@powersync/service-sync-rules": "0.29.10",
     "@powersync/service-types": "0.13.3"
   },
   "devDependencies": {
     "@types/semver": "^7.5.4",
-    "@powersync/service-core-tests": "0.12.15",
-    "@powersync/service-module-mongodb-storage": "0.12.15",
-    "@powersync/lib-service-postgres": "0.4.17",
-    "@powersync/service-module-postgres-storage": "0.10.15"
+    "@powersync/service-core-tests": "0.12.16",
+    "@powersync/service-module-mongodb-storage": "0.12.16",
+    "@powersync/lib-service-postgres": "0.4.18",
+    "@powersync/service-module-postgres-storage": "0.10.16"
   },
   "scripts": {
     "build": "tsc -b",

package/src/api/PostgresRouteAPIAdapter.ts

@@ -108,7 +108,8 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
         results: {
           columns: result.columns.map((c) => c.name),
           rows: result.rows.map((row) => {
-            return row.map((value) => {
+            return row.raw.map((raw, i) => {
+              const value = pgwire.PgType.decode(raw, row.columns[i].typeOid);
               const sqlValue = sync_rules.applyValueContext(
                 sync_rules.toSyncRulesValue(value),
                 sync_rules.CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY
@@ -252,7 +253,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
     // For those, we need to use pg_current_wal_lsn() instead.
     const { results } = await lib_postgres.retriedQuery(this.pool, `SELECT pg_current_wal_lsn() as lsn`);
 
-    const lsn = results[0].rows[0][0];
+    const lsn = results[0].rows[0].decodeWithoutCustomTypes(0);
     return String(lsn);
   }
 

package/src/replication/SnapshotQuery.ts

@@ -127,19 +127,19 @@ export class ChunkedSnapshotQuery implements SnapshotQuery {
       if (this.key.typeId == null) {
         throw new Error(`typeId required for primary key ${this.key.name}`);
       }
-      let type: StatementParam['type'] = Number(this.key.typeId);
+      const type = Number(this.key.typeId);
       stream = this.connection.stream({
         statement: `SELECT * FROM ${this.table.qualifiedName} WHERE ${escapedKeyName} > $1 ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}`,
         params: [{ value: this.lastKey, type }]
       });
     }
     let primaryKeyIndex: number = -1;
+    let typeOid = 0;
 
     for await (let chunk of stream) {
       if (chunk.tag == 'RowDescription') {
         // We get a RowDescription for each FETCH call, but they should
         // all be the same.
-        let i = 0;
         const pk = chunk.payload.findIndex((c) => c.name == this.key.name);
         if (pk < 0) {
           throw new Error(
@@ -147,10 +147,14 @@ export class ChunkedSnapshotQuery implements SnapshotQuery {
           );
         }
         primaryKeyIndex = pk;
+        typeOid = chunk.payload[pk].typeOid;
       }
 
       if (chunk.rows.length > 0) {
-        this.lastKey = chunk.rows[chunk.rows.length - 1][primaryKeyIndex];
+        this.lastKey = PgType.decode(chunk.rows[chunk.rows.length - 1].raw[primaryKeyIndex], typeOid) as
+          | string
+          | bigint
+          | null;
       }
       yield chunk;
     }

package/src/replication/WalStream.ts

@@ -20,16 +20,19 @@ import {
 } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import {
+  applyRowContext,
   applyValueContext,
   CompatibilityContext,
   DatabaseInputRow,
   SqliteInputRow,
   SqliteInputValue,
   SqliteRow,
+  SqliteValue,
   SqlSyncRules,
   TablePattern,
   ToastableSqliteRow,
-  toSyncRulesRow
+  toSyncRulesRow,
+  toSyncRulesValue
 } from '@powersync/service-sync-rules';
 
 import { ReplicationMetric } from '@powersync/service-types';
@@ -44,6 +47,7 @@ import {
   SimpleSnapshotQuery,
   SnapshotQuery
 } from './SnapshotQuery.js';
+import { PostgresTypeResolver } from '../types/resolver.js';
 
 export interface WalStreamOptions {
   logger?: Logger;
@@ -361,11 +365,7 @@ WHERE  oid = $1::regclass`,
       params: [{ value: table.qualifiedName, type: 'varchar' }]
     });
     const row = results.rows[0];
-    if ((row?.[0] ?? -1n) == -1n) {
-      return -1;
-    } else {
-      return Number(row[0]);
-    }
+    return Number(row?.decodeWithoutCustomTypes(0) ?? -1n);
   }
 
   /**
@@ -466,11 +466,25 @@ WHERE  oid = $1::regclass`,
     }
   }
 
-  static *getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteInputRow> {
-    for (let row of results) {
-      yield toSyncRulesRow(row);
-    }
+  static decodeRow(row: pgwire.PgRow, types: PostgresTypeResolver): SqliteInputRow {
+    let result: SqliteInputRow = {};
+
+    row.raw.forEach((rawValue, i) => {
+      const column = row.columns[i];
+      let mappedValue: SqliteInputValue;
+
+      if (typeof rawValue == 'string') {
+        mappedValue = toSyncRulesValue(types.registry.decodeDatabaseValue(rawValue, column.typeOid), false, true);
+      } else {
+        // Binary format, expose as-is.
+        mappedValue = rawValue;
+      }
+
+      result[column.name] = mappedValue;
+    });
+    return result;
   }
+
   private async snapshotTableInTx(
     batch: storage.BucketStorageBatch,
     db: pgwire.PgConnection,
@@ -499,7 +513,7 @@ WHERE  oid = $1::regclass`,
       // 2. Wait until logical replication has caught up with all the change between A and B.
       // Calling `markSnapshotDone(LSN B)` covers that.
       const rs = await db.query(`select pg_current_wal_lsn() as lsn`);
-      tableLsnNotBefore = rs.rows[0][0];
+      tableLsnNotBefore = rs.rows[0].decodeWithoutCustomTypes(0);
       // Side note: A ROLLBACK would probably also be fine here, since we only read in this transaction.
       await db.query('COMMIT');
       const [resultTable] = await batch.markSnapshotDone([table], tableLsnNotBefore);
@@ -545,8 +559,6 @@ WHERE  oid = $1::regclass`,
     }
     await q.initialize();
 
-    let columns: { i: number; name: string }[] = [];
-    let columnMap: Record<string, number> = {};
     let hasRemainingData = true;
     while (hasRemainingData) {
       // Fetch 10k at a time.
@@ -560,31 +572,16 @@ WHERE  oid = $1::regclass`,
       // There are typically 100-200 rows per chunk.
       for await (let chunk of cursor) {
         if (chunk.tag == 'RowDescription') {
-          // We get a RowDescription for each FETCH call, but they should
-          // all be the same.
-          let i = 0;
-          columns = chunk.payload.map((c) => {
-            return { i: i++, name: c.name };
-          });
-          for (let column of chunk.payload) {
-            columnMap[column.name] = column.typeOid;
-          }
           continue;
         }
 
-        const rows = chunk.rows.map((row) => {
-          let q: DatabaseInputRow = {};
-          for (let c of columns) {
-            q[c.name] = row[c.i];
-          }
-          return q;
-        });
-        if (rows.length > 0) {
+        if (chunk.rows.length > 0) {
           hasRemainingData = true;
         }
 
-        for (const inputRecord of WalStream.getQueryData(rows)) {
-          const record = this.syncRulesRecord(this.connections.types.constructRowRecord(columnMap, inputRecord));
+        for (const rawRow of chunk.rows) {
+          const record = this.sync_rules.applyRowContext<never>(WalStream.decodeRow(rawRow, this.connections.types));
+
           // This auto-flushes when the batch reaches its size limit
           await batch.save({
             tag: storage.SaveOperationTag.INSERT,
@@ -596,8 +593,8 @@ WHERE  oid = $1::regclass`,
           });
         }
 
-        at += rows.length;
-        this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(rows.length);
+        at += chunk.rows.length;
+        this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(chunk.rows.length);
 
         this.touch();
       }

package/src/replication/replication-utils.ts

@@ -32,8 +32,8 @@ export async function getPrimaryKeyColumns(
 
   return attrRows.rows.map((row) => {
     return {
-      name: row[0] as string,
-      typeId: row[1] as number
+      name: row.decodeWithoutCustomTypes(0) as string,
+      typeId: row.decodeWithoutCustomTypes(1) as number
     } satisfies storage.ColumnDescriptor;
   });
 }
@@ -50,8 +50,8 @@ export async function getAllColumns(db: pgwire.PgClient, relationId: number): Pr
   });
   return attrRows.rows.map((row) => {
     return {
-      name: row[0] as string,
-      typeId: row[1] as number
+      name: row.decodeWithoutCustomTypes(0) as string,
+      typeId: row.decodeWithoutCustomTypes(1) as number
     } satisfies storage.ColumnDescriptor;
   });
 }
@@ -71,7 +71,7 @@ FROM pg_class
 WHERE oid = $1::oid LIMIT 1`,
     params: [{ type: 'int8', value: relationId }]
   });
-  const idType: string = rows.rows[0]?.[0];
+  const idType: string = rows.rows[0]?.decodeWithoutCustomTypes(0);
   if (idType == 'nothing' || idType == null) {
     return { replicationIdentity: 'nothing', replicationColumns: [] };
   } else if (idType == 'full') {

package/src/types/registry.ts

@@ -256,10 +256,7 @@ export class CustomTypeRegistry {
       case 'unknown':
         return true;
       case 'array':
-        return (
-          type.separatorCharCode == pgwire.CHAR_CODE_COMMA &&
-          this.isParsedWithoutCustomTypesSupport(this.lookupType(type.innerId))
-        );
+        return type.separatorCharCode == pgwire.CHAR_CODE_COMMA && pgwire.ARRAY_TO_ELEM_OID.has(type.innerId);
       default:
         return false;
     }

package/src/types/resolver.ts

@@ -153,7 +153,7 @@ WHERE a.attnum > 0
   AND cn.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
     `;
 
-    const query = await this.pool.query({ statement: sql });
+    const query = await this.pool.query(sql);
     let ids: number[] = [];
     for (const row of pgwire.pgwireRows(query)) {
       ids.push(Number(row.type_oid));
@@ -186,26 +186,22 @@ WHERE a.attnum > 0
     return toSyncRulesRow(record);
   }
 
-  constructRowRecord(columnMap: Record<string, number>, tupleRaw: Record<string, any>): SqliteInputRow {
-    const record = this.decodeTupleForTable(columnMap, tupleRaw);
-    return toSyncRulesRow(record);
-  }
-
   /**
    * We need a high level of control over how values are decoded, to make sure there is no loss
    * of precision in the process.
    */
   decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record<string, any>): DatabaseInputRow {
     let result: Record<string, any> = {};
-    for (let columnName in tupleRaw) {
-      const rawval = tupleRaw[columnName];
-      const typeOid = (relation as any)._tupleDecoder._typeOids.get(columnName);
-      if (typeof rawval == 'string' && typeOid) {
-        result[columnName] = this.registry.decodeDatabaseValue(rawval, typeOid);
-      } else {
-        result[columnName] = rawval;
-      }
+    for (const column of relation.columns) {
+      const rawval = tupleRaw[column.name];
+      result[column.name] =
+        rawval == null
+          ? // We can't decode null values, but it's important that null and undefined stay distinct because undefined
+            // represents a TOASTed value.
+            rawval
+          : this.registry.decodeDatabaseValue(rawval, column.typeOid);
     }
+
     return result;
   }
 

package/src/utils/migration_lib.ts

@@ -66,7 +66,7 @@ export class Migrations {
       limit 1
     `
       )
-      .then((results) => ({ id: results.rows[0][0] as number }));
+      .then((results) => ({ id: results.rows[0].decodeWithoutCustomTypes(0) as number }));
   }
 
   async ensureMigrationsTable(db: pgwire.PgConnection) {

package/src/utils/postgres_version.ts

@@ -4,5 +4,5 @@ import semver, { type SemVer } from 'semver';
 export async function getServerVersion(db: pgwire.PgClient): Promise<SemVer | null> {
   const result = await db.query(`SHOW server_version;`);
   // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
-  return semver.coerce(result.rows[0][0].split(' ')[0]);
+  return semver.coerce(result.rows[0].decodeWithoutCustomTypes(0).split(' ')[0]);
 }