@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.20

package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAiB,KAAK,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAY7E;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA4MD,eAAO,MAAM,sBAAsB,QAMwB,CAAC"}
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAiB,KAAK,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAY7E;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,2BAA2B,UAAO,GACjC,OAAO,CAAC,IAAI,CAAC,CAiBf;AAgOD,eAAO,MAAM,sBAAsB,QASwB,CAAC"}

package/out/zero-cache/src/services/change-source/pg/schema/init.js

@@ -10,13 +10,13 @@ import { runSchemaMigrations } from "../../../../db/migration.js";
 /**
 * Ensures that a shard is set up for initial sync.
 */
-async function ensureShardSchema(lc, db, shard) {
+async function ensureShardSchema(lc, db, shard, installPartialIndexTriggers = true) {
 	await runSchemaMigrations(lc, `upstream-shard-${shard.appID}`, upstreamSchema(shard), db, {
-		migrateSchema: (lc, tx) => setupTablesAndReplication(lc, tx, shard),
+		migrateSchema: (lc, tx) => setupTablesAndReplication(lc, tx, shard, installPartialIndexTriggers),
 		minSafeVersion: 1
-	}, getIncrementalMigrations(shard));
+	}, getIncrementalMigrations(shard, installPartialIndexTriggers));
 }
-function getIncrementalMigrations(shard) {
+function getIncrementalMigrations(shard, installPartialIndexTriggers) {
 	const shardConfigTable = `${upstreamSchema(shard)}.shardConfig`;
 	return {
 		6: { migrateSchema: () => {
@@ -129,14 +129,23 @@ function getIncrementalMigrations(shard) {
           ALTER TABLE ${sql(upstreamSchema(shard))}.replicas 
             ADD COLUMN "backupV5" BOOL DEFAULT false;
         `;
-		} }
+		} },
+		...installPartialIndexTriggers ? { 26: { migrateSchema: async (lc, sql) => {
+			const [{ publications }] = await sql`
+                SELECT publications FROM ${sql(shardConfigTable)}`;
+			await setupTriggers(lc, sql, {
+				...shard,
+				publications
+			});
+			lc.info?.(`Upgraded DDL event triggers`);
+		} } } : {}
 	};
 }
 Object.keys(getIncrementalMigrations({
 	appID: "unused",
 	shardNum: 0,
 	publications: ["foo"]
-})).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);
+}, true)).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);
 //#endregion
 export { ensureShardSchema };
 

package/out/zero-cache/src/services/change-source/pg/schema/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../../db/migration.ts';\nimport type {PostgresDB} from '../../../../types/pg.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {publishedSchema} from './published.ts';\nimport {\n  getMutationsTableDefinition,\n  legacyReplicationSlot,\n  metadataPublicationName,\n  setupTablesAndReplication,\n  setupTriggers,\n} from './shard.ts';\n\n/**\n * Ensures that a shard is set up for initial sync.\n */\nexport async function ensureShardSchema(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n): Promise<void> {\n  const initialSetup: Migration = {\n    migrateSchema: (lc, tx) => setupTablesAndReplication(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n  await runSchemaMigrations(\n    lc,\n    `upstream-shard-${shard.appID}`,\n    upstreamSchema(shard),\n    db,\n    initialSetup,\n    // The incremental migration of any existing replicas will be replaced by\n    // the incoming replica being synced, so the replicaVersion here is\n    // unnecessary.\n    getIncrementalMigrations(shard),\n  );\n}\n\nfunction getIncrementalMigrations(shard: ShardConfig): IncrementalMigrationMap {\n  const shardConfigTable = `${upstreamSchema(shard)}.shardConfig`;\n\n  return {\n    6: {\n      migrateSchema: () => {\n        throw new AutoResetSignal('resetting to upgrade shard schema');\n      },\n    },\n\n    // v7: Updates the DDL event trigger protocol to v2, and adds support for\n    //     ALTER SCHEMA x RENAME TO y\n    //     (migration logic subsumbed by v23)\n\n    // v8: Adds support for non-disruptive resyncs, which tracks multiple\n    //     replicas with different slot names.\n    8: {\n      migrateSchema: async (lc, sql) => {\n        const legacyShardConfigSchema = v.object({\n          replicaVersion: v.string().nullable(),\n          initialSchema: publishedSchema.nullable(),\n        });\n        const result = await sql`\n          SELECT \"replicaVersion\", \"initialSchema\" FROM ${sql(shardConfigTable)}`;\n        assert(\n          result.length === 1,\n          () => `Expected exactly one shardConfig row, got ${result.length}`,\n        );\n        const {replicaVersion, initialSchema} = v.parse(\n          result[0],\n          legacyShardConfigSchema,\n          'passthrough',\n        );\n\n        await Promise.all([\n          sql`\n          CREATE TABLE ${sql(upstreamSchema(shard))}.replicas (\n            \"slot\"          TEXT PRIMARY KEY,\n            \"version\"       TEXT NOT NULL,\n            \"initialSchema\" JSON NOT NULL\n          );\n          `,\n          sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.replicas ${sql({\n            slot: legacyReplicationSlot(shard),\n            version: replicaVersion,\n            initialSchema,\n          })}\n          `,\n          sql`\n          ALTER TABLE ${sql(shardConfigTable)} DROP \"replicaVersion\", DROP \"initialSchema\"\n          `,\n        ]);\n        lc.info?.(`Upgraded schema to support non-disruptive resyncs`);\n      },\n    },\n\n    // v9: Fixes field ordering of compound indexes. This incremental migration\n    // only fixes indexes resulting from new schema changes. A full resync is\n    // required to fix existing indexes.\n    //\n    // The migration has been subsumed by the identical logic for migrating\n    // to v12 (i.e. a trigger upgrade).\n\n    // Adds the `mutations` table used to track mutation results.\n    10: {\n      migrateSchema: async (lc, sql) => {\n        await sql.unsafe(/*sql*/ `\n          ${getMutationsTableDefinition(upstreamSchema(shard))}\n          ALTER PUBLICATION ${id(metadataPublicationName(shard.appID, shard.shardNum))} ADD TABLE ${id(upstreamSchema(shard))}.\"mutations\";\n        `);\n        lc.info?.('Upgraded schema with new mutations table');\n      },\n    },\n\n    // v11: Formerly dropped the schemaVersions table, but restored in the v13\n    // migration for rollback safety.\n\n    // v12: Upgrade DDL trigger to query schemaOID, needed information for auto-backfill.\n    // (subsumed by v14)\n\n    // Recreates the legacy schemaVersions table that was prematurely dropped\n    // in the (former) v11 migration. It needs to remain present for at least one\n    // release in order to be rollback safe.\n    //\n    // TODO: Drop the table once a release that no longer reads the table has\n    // been rolled out.\n    13: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          CREATE TABLE IF NOT EXISTS ${sql(upstreamSchema(shard))}.\"schemaVersions\" (\n            \"minSupportedVersion\" INT4,\n            \"maxSupportedVersion\" INT4,\n            \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n        );`;\n        await sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.\"schemaVersions\" \n            (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n            VALUES (true, 1, 1)\n            ON CONFLICT DO NOTHING;\n        `;\n      },\n    },\n\n    // v14: Upgrade DDL trigger to log more info to PG logs.\n    // (subsumed by v16)\n\n    // Add initialSyncContext column to replicas table.\n    15: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas\n            ADD COLUMN \"initialSyncContext\" JSON,\n            ADD COLUMN \"subscriberContext\" JSON\n        `;\n      },\n    },\n\n    // v16: Upgrade DDL trigger to fire on all ALTER TABLE statements\n    // to catch the *removal* of a table from the published set.\n    // v17 (1.0.0): Upgrade DDL triggers to support the COMMENT ON PUBLICATION hook for\n    // working around the lack of event trigger support for PUBLICATION\n    // changes in supabase.\n    // This also adds forwards-compatible support for hierarchical logical\n    // message prefixes and unknown ddl event types.\n    // v18: Pure refactoring of event trigger code.\n    // v19 (1.4.0): Correctly handle concurrently issued DDL statements.\n    // v20 (1.4.0): Handle nested DDL triggers\n    // v21 (1.5.0): Handle cross-transaction DDL operations (i.e. concurrent\n    // index operations), and support manual invocation of update_schemas().\n\n    22: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ALTER \"initialSchema\" DROP NOT NULL;\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            DROP CONSTRAINT replicas_pkey;\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN id TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', '');\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN rank BIGSERIAL;\n        `;\n      },\n    },\n\n    // v23 (1.6.0): Event triggers: Handle the case where current_query()\n    // returns NULL. Add more logging to debug unexpected messages\n    23: {\n      migrateSchema: async (lc, sql) => {\n        const [{publications}] = await sql<{publications: string[]}[]>`\n          SELECT publications FROM ${sql(shardConfigTable)}`;\n        await setupTriggers(lc, sql, {...shard, publications});\n        lc.info?.(`Upgraded DDL event triggers`);\n      },\n    },\n\n    // v24 (1.10.0):\n    // - adds the \"backupPath\" column for per-replica backups.\n    // - adds the \"generation\" column to replace \"version\".\n    24: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN \"backupPath\" TEXT;\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN \"generation\" TEXT;\n        `;\n      },\n\n      migrateData: async (_, sql) => {\n        // Backfill the \"generation\" column to eventually replace \"version\"\n        await sql`\n          UPDATE ${sql(upstreamSchema(shard))}.replicas SET \"generation\" = \"version\";\n        `;\n      },\n    },\n\n    // v25 (1.10.0):\n    // - adds the \"backupV5\" column to signal the current semantics\n    //   of the replication slot's confirmed_flush_lsn with respect to what\n    //   has been backed up.\n    25: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN \"backupV5\" BOOL DEFAULT false;\n        `;\n      },\n    },\n  };\n}\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  getIncrementalMigrations({\n    appID: 'unused',\n    shardNum: 0,\n    publications: ['foo'],\n  }),\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;;;AAwBA,eAAsB,kBACpB,IACA,IACA,OACe;CAKf,MAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,KAAK,GACpB,IACA;EARA,gBAAgB,IAAI,OAAO,0BAA0B,IAAI,IAAI,KAAK;EAClE,gBAAgB;CAOhB,GAIA,yBAAyB,KAAK,CAChC;AACF;AAEA,SAAS,yBAAyB,OAA6C;CAC7E,MAAM,mBAAmB,GAAG,eAAe,KAAK,EAAE;CAElD,OAAO;EACL,GAAG,EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,mCAAmC;EAC/D,EACF;EAQA,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,0BAA0B,eAAE,OAAO;IACvC,gBAAgB,eAAE,OAAO,EAAE,SAAS;IACpC,eAAe,gBAAgB,SAAS;GAC1C,CAAC;GACD,MAAM,SAAS,MAAM,GAAG;0DAC0B,IAAI,gBAAgB;GACtE,OACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,QAC5D;GACA,MAAM,EAAC,gBAAgB,kBAAiB,MACtC,OAAO,IACP,yBACA,aACF;GAEA,MAAM,QAAQ,IAAI;IAChB,GAAG;yBACY,IAAI,eAAe,KAAK,CAAC,EAAE;;;;;;IAM1C,GAAG;wBACW,IAAI,eAAe,KAAK,CAAC,EAAE,YAAY,IAAI;KACvD,MAAM,sBAAsB,KAAK;KACjC,SAAS;KACT;IACF,CAAC,EAAE;;IAEH,GAAG;wBACW,IAAI,gBAAgB,EAAE;;GAEtC,CAAC;GACD,GAAG,OAAO,mDAAmD;EAC/D,EACF;EAUA,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,IAAI,OAAe;YACrB,4BAA4B,eAAe,KAAK,CAAC,EAAE;8BACjC,GAAG,wBAAwB,MAAM,OAAO,MAAM,QAAQ,CAAC,EAAE,aAAa,GAAG,eAAe,KAAK,CAAC,EAAE;SACrH;GACD,GAAG,OAAO,0CAA0C;EACtD,EACF;EAcA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;uCACsB,IAAI,eAAe,KAAK,CAAC,EAAE;;;;;GAK1D,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;;;EAK7C,EACF;EAMA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;;EAI7C,EACF;EAeA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;EAG7C,EACF;EAIA,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,gBAAgB;GACjD,MAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;GAAY,CAAC;GACrD,GAAG,OAAO,6BAA6B;EACzC,EACF;EAKA,IAAI;GACF,eAAe,OAAO,GAAG,QAAQ;IAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;IAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG7C;GAEA,aAAa,OAAO,GAAG,QAAQ;IAE7B,MAAM,GAAG;mBACE,IAAI,eAAe,KAAK,CAAC,EAAE;;GAExC;EACF;EAMA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;EAG7C,EACF;CACF;AACF;AAGsC,OAAO,KAC3C,yBAAyB;CACvB,OAAO;CACP,UAAU;CACV,cAAc,CAAC,KAAK;AACtB,CAAC,CACH,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
+{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../../db/migration.ts';\nimport type {PostgresDB} from '../../../../types/pg.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {publishedSchema} from './published.ts';\nimport {\n  getMutationsTableDefinition,\n  legacyReplicationSlot,\n  metadataPublicationName,\n  setupTablesAndReplication,\n  setupTriggers,\n} from './shard.ts';\n\n/**\n * Ensures that a shard is set up for initial sync.\n */\nexport async function ensureShardSchema(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n  installPartialIndexTriggers = true,\n): Promise<void> {\n  const initialSetup: Migration = {\n    migrateSchema: (lc, tx) =>\n      setupTablesAndReplication(lc, tx, shard, installPartialIndexTriggers),\n    minSafeVersion: 1,\n  };\n  await runSchemaMigrations(\n    lc,\n    `upstream-shard-${shard.appID}`,\n    upstreamSchema(shard),\n    db,\n    initialSetup,\n    // The incremental migration of any existing replicas will be replaced by\n    // the incoming replica being synced, so the replicaVersion here is\n    // unnecessary.\n    getIncrementalMigrations(shard, installPartialIndexTriggers),\n  );\n}\n\nfunction getIncrementalMigrations(\n  shard: ShardConfig,\n  installPartialIndexTriggers: boolean,\n): IncrementalMigrationMap {\n  const shardConfigTable = `${upstreamSchema(shard)}.shardConfig`;\n\n  return {\n    6: {\n      migrateSchema: () => {\n        throw new AutoResetSignal('resetting to upgrade shard schema');\n      },\n    },\n\n    // v7: Updates the DDL event trigger protocol to v2, and adds support for\n    //     ALTER SCHEMA x RENAME TO y\n    //     (migration logic subsumbed by v23)\n\n    // v8: Adds support for non-disruptive resyncs, which tracks multiple\n    //     replicas with different slot names.\n    8: {\n      migrateSchema: async (lc, sql) => {\n        const legacyShardConfigSchema = v.object({\n          replicaVersion: v.string().nullable(),\n          initialSchema: publishedSchema.nullable(),\n        });\n        const result = await sql`\n          SELECT \"replicaVersion\", \"initialSchema\" FROM ${sql(shardConfigTable)}`;\n        assert(\n          result.length === 1,\n          () => `Expected exactly one shardConfig row, got ${result.length}`,\n        );\n        const {replicaVersion, initialSchema} = v.parse(\n          result[0],\n          legacyShardConfigSchema,\n          'passthrough',\n        );\n\n        await Promise.all([\n          sql`\n          CREATE TABLE ${sql(upstreamSchema(shard))}.replicas (\n            \"slot\"          TEXT PRIMARY KEY,\n            \"version\"       TEXT NOT NULL,\n            \"initialSchema\" JSON NOT NULL\n          );\n          `,\n          sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.replicas ${sql({\n            slot: legacyReplicationSlot(shard),\n            version: replicaVersion,\n            initialSchema,\n          })}\n          `,\n          sql`\n          ALTER TABLE ${sql(shardConfigTable)} DROP \"replicaVersion\", DROP \"initialSchema\"\n          `,\n        ]);\n        lc.info?.(`Upgraded schema to support non-disruptive resyncs`);\n      },\n    },\n\n    // v9: Fixes field ordering of compound indexes. This incremental migration\n    // only fixes indexes resulting from new schema changes. A full resync is\n    // required to fix existing indexes.\n    //\n    // The migration has been subsumed by the identical logic for migrating\n    // to v12 (i.e. a trigger upgrade).\n\n    // Adds the `mutations` table used to track mutation results.\n    10: {\n      migrateSchema: async (lc, sql) => {\n        await sql.unsafe(/*sql*/ `\n          ${getMutationsTableDefinition(upstreamSchema(shard))}\n          ALTER PUBLICATION ${id(metadataPublicationName(shard.appID, shard.shardNum))} ADD TABLE ${id(upstreamSchema(shard))}.\"mutations\";\n        `);\n        lc.info?.('Upgraded schema with new mutations table');\n      },\n    },\n\n    // v11: Formerly dropped the schemaVersions table, but restored in the v13\n    // migration for rollback safety.\n\n    // v12: Upgrade DDL trigger to query schemaOID, needed information for auto-backfill.\n    // (subsumed by v14)\n\n    // Recreates the legacy schemaVersions table that was prematurely dropped\n    // in the (former) v11 migration. It needs to remain present for at least one\n    // release in order to be rollback safe.\n    //\n    // TODO: Drop the table once a release that no longer reads the table has\n    // been rolled out.\n    13: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          CREATE TABLE IF NOT EXISTS ${sql(upstreamSchema(shard))}.\"schemaVersions\" (\n            \"minSupportedVersion\" INT4,\n            \"maxSupportedVersion\" INT4,\n            \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n        );`;\n        await sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.\"schemaVersions\" \n            (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n            VALUES (true, 1, 1)\n            ON CONFLICT DO NOTHING;\n        `;\n      },\n    },\n\n    // v14: Upgrade DDL trigger to log more info to PG logs.\n    // (subsumed by v16)\n\n    // Add initialSyncContext column to replicas table.\n    15: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas\n            ADD COLUMN \"initialSyncContext\" JSON,\n            ADD COLUMN \"subscriberContext\" JSON\n        `;\n      },\n    },\n\n    // v16: Upgrade DDL trigger to fire on all ALTER TABLE statements\n    // to catch the *removal* of a table from the published set.\n    // v17 (1.0.0): Upgrade DDL triggers to support the COMMENT ON PUBLICATION hook for\n    // working around the lack of event trigger support for PUBLICATION\n    // changes in supabase.\n    // This also adds forwards-compatible support for hierarchical logical\n    // message prefixes and unknown ddl event types.\n    // v18: Pure refactoring of event trigger code.\n    // v19 (1.4.0): Correctly handle concurrently issued DDL statements.\n    // v20 (1.4.0): Handle nested DDL triggers\n    // v21 (1.5.0): Handle cross-transaction DDL operations (i.e. concurrent\n    // index operations), and support manual invocation of update_schemas().\n\n    22: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ALTER \"initialSchema\" DROP NOT NULL;\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            DROP CONSTRAINT replicas_pkey;\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN id TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', '');\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN rank BIGSERIAL;\n        `;\n      },\n    },\n\n    // v23 (1.6.0): Event triggers: Handle the case where current_query()\n    // returns NULL. Add more logging to debug unexpected messages\n    23: {\n      migrateSchema: async (lc, sql) => {\n        const [{publications}] = await sql<{publications: string[]}[]>`\n          SELECT publications FROM ${sql(shardConfigTable)}`;\n        await setupTriggers(lc, sql, {...shard, publications});\n        lc.info?.(`Upgraded DDL event triggers`);\n      },\n    },\n\n    // v24 (1.10.0):\n    // - adds the \"backupPath\" column for per-replica backups.\n    // - adds the \"generation\" column to replace \"version\".\n    24: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN \"backupPath\" TEXT;\n        `;\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN \"generation\" TEXT;\n        `;\n      },\n\n      migrateData: async (_, sql) => {\n        // Backfill the \"generation\" column to eventually replace \"version\"\n        await sql`\n          UPDATE ${sql(upstreamSchema(shard))}.replicas SET \"generation\" = \"version\";\n        `;\n      },\n    },\n\n    // v25 (1.10.0):\n    // - adds the \"backupV5\" column to signal the current semantics\n    //   of the replication slot's confirmed_flush_lsn with respect to what\n    //   has been backed up.\n    25: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n            ADD COLUMN \"backupV5\" BOOL DEFAULT false;\n        `;\n      },\n    },\n\n    ...(installPartialIndexTriggers\n      ? {\n          // v26: Upgrade the DDL event triggers to include partial indexes in\n          // schema snapshots. Note that setupTriggers() also refreshes the\n          // stored \"publishedSchema\" so that the change in format does not\n          // manifest as a spurious schema change.\n          26: {\n            migrateSchema: async (lc, sql) => {\n              const [{publications}] = await sql<{publications: string[]}[]>`\n                SELECT publications FROM ${sql(shardConfigTable)}`;\n              await setupTriggers(lc, sql, {...shard, publications});\n              lc.info?.(`Upgraded DDL event triggers`);\n            },\n          },\n        }\n      : {}),\n  };\n}\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  getIncrementalMigrations(\n    {\n      appID: 'unused',\n      shardNum: 0,\n      publications: ['foo'],\n    },\n    true,\n  ),\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;;;AAwBA,eAAsB,kBACpB,IACA,IACA,OACA,8BAA8B,MACf;CAMf,MAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,KAAK,GACpB,IACA;EATA,gBAAgB,IAAI,OAClB,0BAA0B,IAAI,IAAI,OAAO,2BAA2B;EACtE,gBAAgB;CAOhB,GAIA,yBAAyB,OAAO,2BAA2B,CAC7D;AACF;AAEA,SAAS,yBACP,OACA,6BACyB;CACzB,MAAM,mBAAmB,GAAG,eAAe,KAAK,EAAE;CAElD,OAAO;EACL,GAAG,EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,mCAAmC;EAC/D,EACF;EAQA,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,0BAA0B,eAAE,OAAO;IACvC,gBAAgB,eAAE,OAAO,EAAE,SAAS;IACpC,eAAe,gBAAgB,SAAS;GAC1C,CAAC;GACD,MAAM,SAAS,MAAM,GAAG;0DAC0B,IAAI,gBAAgB;GACtE,OACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,QAC5D;GACA,MAAM,EAAC,gBAAgB,kBAAiB,MACtC,OAAO,IACP,yBACA,aACF;GAEA,MAAM,QAAQ,IAAI;IAChB,GAAG;yBACY,IAAI,eAAe,KAAK,CAAC,EAAE;;;;;;IAM1C,GAAG;wBACW,IAAI,eAAe,KAAK,CAAC,EAAE,YAAY,IAAI;KACvD,MAAM,sBAAsB,KAAK;KACjC,SAAS;KACT;IACF,CAAC,EAAE;;IAEH,GAAG;wBACW,IAAI,gBAAgB,EAAE;;GAEtC,CAAC;GACD,GAAG,OAAO,mDAAmD;EAC/D,EACF;EAUA,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,IAAI,OAAe;YACrB,4BAA4B,eAAe,KAAK,CAAC,EAAE;8BACjC,GAAG,wBAAwB,MAAM,OAAO,MAAM,QAAQ,CAAC,EAAE,aAAa,GAAG,eAAe,KAAK,CAAC,EAAE;SACrH;GACD,GAAG,OAAO,0CAA0C;EACtD,EACF;EAcA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;uCACsB,IAAI,eAAe,KAAK,CAAC,EAAE;;;;;GAK1D,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;;;EAK7C,EACF;EAMA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;;EAI7C,EACF;EAeA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;EAG7C,EACF;EAIA,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,gBAAgB;GACjD,MAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;GAAY,CAAC;GACrD,GAAG,OAAO,6BAA6B;EACzC,EACF;EAKA,IAAI;GACF,eAAe,OAAO,GAAG,QAAQ;IAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;IAG3C,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;GAG7C;GAEA,aAAa,OAAO,GAAG,QAAQ;IAE7B,MAAM,GAAG;mBACE,IAAI,eAAe,KAAK,CAAC,EAAE;;GAExC;EACF;EAMA,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;GAC/B,MAAM,GAAG;wBACO,IAAI,eAAe,KAAK,CAAC,EAAE;;;EAG7C,EACF;EAEA,GAAI,8BACA,EAKE,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;2CACjC,IAAI,gBAAgB;GACjD,MAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;GAAY,CAAC;GACrD,GAAG,OAAO,6BAA6B;EACzC,EACF,EACF,IACA,CAAC;CACP;AACF;AAGsC,OAAO,KAC3C,yBACE;CACE,OAAO;CACP,UAAU;CACV,cAAc,CAAC,KAAK;AACtB,GACA,IACF,CACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}

package/out/zero-cache/src/services/change-source/pg/schema/published.d.ts

@@ -1,38 +1,155 @@
+import type { LogContext } from '@rocicorp/logger';
 import type postgres from 'postgres';
 import * as v from '../../../../../../shared/src/valita.ts';
-export declare function publishedSchemaQuery(publications: readonly string[]): string;
+import { type UnsupportedPredicateReason } from './index-predicate.ts';
+export declare function publishedSchemaQuery(publications: readonly string[], includePartialIndexes?: boolean): string;
+export type SkippedIndexDiagnostic = {
+    readonly schema: string;
+    readonly index: string;
+    readonly predicate: string;
+    readonly reason: UnsupportedPredicateReason;
+};
+export declare function getSkippedIndexDiagnostics(schema: PublishedSchema): readonly SkippedIndexDiagnostic[];
+export declare function warnForSkippedIndexes(lc: LogContext, schema: PublishedSchema, warned?: Set<string>): void;
 export declare const publishedSchema: v.Type<{
     indexes: {
-        isReplicaIdentity?: boolean | undefined;
-        isPrimaryKey?: boolean | undefined;
-        isImmediate?: boolean | undefined;
-        name: string;
-        columns: Record<string, "ASC" | "DESC">;
-        schema: string;
-        tableName: string;
-        unique: boolean;
+        readonly predicate?: {
+            readonly type: "and" | "or";
+            readonly conditions: readonly (/*elided*/ any | {
+                readonly type: "not";
+                readonly condition: /*elided*/ any | /*elided*/ any | {
+                    readonly type: "null-test";
+                    readonly column: string;
+                    readonly op: "IS NULL" | "IS NOT NULL";
+                } | {
+                    readonly type: "comparison";
+                    readonly column: string;
+                    readonly op: "=" | "<>" | "<" | "<=" | ">" | ">=";
+                    readonly value: {
+                        readonly type: "boolean";
+                        readonly value: boolean;
+                    } | {
+                        readonly type: "integer";
+                        readonly value: string;
+                    } | {
+                        readonly type: "string";
+                        readonly value: string;
+                    };
+                };
+            } | {
+                readonly type: "null-test";
+                readonly column: string;
+                readonly op: "IS NULL" | "IS NOT NULL";
+            } | {
+                readonly type: "comparison";
+                readonly column: string;
+                readonly op: "=" | "<>" | "<" | "<=" | ">" | ">=";
+                readonly value: {
+                    readonly type: "boolean";
+                    readonly value: boolean;
+                } | {
+                    readonly type: "integer";
+                    readonly value: string;
+                } | {
+                    readonly type: "string";
+                    readonly value: string;
+                };
+            })[];
+        } | {
+            readonly type: "not";
+            readonly condition: {
+                readonly type: "and" | "or";
+                readonly conditions: readonly (/*elided*/ any | /*elided*/ any | {
+                    readonly type: "null-test";
+                    readonly column: string;
+                    readonly op: "IS NULL" | "IS NOT NULL";
+                } | {
+                    readonly type: "comparison";
+                    readonly column: string;
+                    readonly op: "=" | "<>" | "<" | "<=" | ">" | ">=";
+                    readonly value: {
+                        readonly type: "boolean";
+                        readonly value: boolean;
+                    } | {
+                        readonly type: "integer";
+                        readonly value: string;
+                    } | {
+                        readonly type: "string";
+                        readonly value: string;
+                    };
+                })[];
+            } | /*elided*/ any | {
+                readonly type: "null-test";
+                readonly column: string;
+                readonly op: "IS NULL" | "IS NOT NULL";
+            } | {
+                readonly type: "comparison";
+                readonly column: string;
+                readonly op: "=" | "<>" | "<" | "<=" | ">" | ">=";
+                readonly value: {
+                    readonly type: "boolean";
+                    readonly value: boolean;
+                } | {
+                    readonly type: "integer";
+                    readonly value: string;
+                } | {
+                    readonly type: "string";
+                    readonly value: string;
+                };
+            };
+        } | {
+            readonly type: "null-test";
+            readonly column: string;
+            readonly op: "IS NULL" | "IS NOT NULL";
+        } | {
+            readonly type: "comparison";
+            readonly column: string;
+            readonly op: "=" | "<>" | "<" | "<=" | ">" | ">=";
+            readonly value: {
+                readonly type: "boolean";
+                readonly value: boolean;
+            } | {
+                readonly type: "integer";
+                readonly value: string;
+            } | {
+                readonly type: "string";
+                readonly value: string;
+            };
+        } | undefined;
+        readonly isReplicaIdentity?: boolean | undefined;
+        readonly isPrimaryKey?: boolean | undefined;
+        readonly isImmediate?: boolean | undefined;
+        readonly name: string;
+        readonly columns: {
+            readonly [x: string]: "ASC" | "DESC";
+        };
+        readonly schema: string;
+        readonly tableName: string;
+        readonly unique: boolean;
     }[];
     tables: {
         replicaIdentityColumns: string[];
+        columns: {
+            [k: string]: {
+                pgTypeClass?: "e" | "d" | "b" | "c" | "p" | "r" | "m" | undefined;
+                elemPgTypeClass?: "e" | "d" | "b" | "c" | "p" | "r" | "m" | null | undefined;
+                characterMaximumLength?: number | null | undefined;
+                notNull?: boolean | null | undefined;
+                dflt?: string | null | undefined;
+                pos: number;
+                dataType: string;
+                typeOID: number;
+            };
+        };
         primaryKey?: string[] | undefined;
         schemaOID?: number | undefined;
         replicaIdentity?: "n" | "d" | "f" | "i" | undefined;
         name: string;
         schema: string;
-        oid: number;
-        columns: Record<string, {
-            pgTypeClass?: "e" | "d" | "b" | "c" | "p" | "r" | "m" | undefined;
-            elemPgTypeClass?: "e" | "d" | "b" | "c" | "p" | "r" | "m" | null | undefined;
-            characterMaximumLength?: number | null | undefined;
-            notNull?: boolean | null | undefined;
-            dflt?: string | null | undefined;
-            pos: number;
-            dataType: string;
-            typeOID: number;
-        }>;
         publications: Record<string, {
             rowFilter: string | null;
         }>;
+        oid: number;
     }[];
 }>;
 export type PublishedSchema = v.Infer<typeof publishedSchema>;

package/out/zero-cache/src/services/change-source/pg/schema/published.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"published.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/published.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAIrC,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,UAmKnE;AAED,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsDxB,CAAC;AAEL,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D,MAAM,MAAM,iCAAiC,GAC3C,eAAe,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC;AAUpC,QAAA,MAAM,wBAAwB;;;;;;cAA6B,CAAC;AAE5D,MAAM,MAAM,eAAe,GAAG,eAAe,GAAG;IAC9C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;CACxD,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,QAAQ,CAAC,GAAG,EACjB,YAAY,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC,CAmD1B"}
+{"version":3,"file":"published.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/published.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAIrC,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAc5D,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,sBAAsB,CAAC;AAE9B,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,qBAAqB,UAAO,UA+K7B;AAcD,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,0BAA0B,CAAC;CAC7C,CAAC;AAMF,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,eAAe,GACtB,SAAS,sBAAsB,EAAE,CAEnC;AAED,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,eAAe,EACvB,MAAM,GAAE,GAAG,CAAC,MAAM,CAAa,GAC9B,IAAI,CAWN;AAED,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4FxB,CAAC;AAEL,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D,MAAM,MAAM,iCAAiC,GAC3C,eAAe,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC;AAUpC,QAAA,MAAM,wBAAwB;;;;;;cAA6B,CAAC;AAE5D,MAAM,MAAM,eAAe,GAAG,eAAe,GAAG;IAC9C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;CACxD,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,QAAQ,CAAC,GAAG,EACjB,YAAY,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC,CAsD1B"}

package/out/zero-cache/src/services/change-source/pg/schema/published.js

@@ -2,13 +2,15 @@ import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { equals } from "../../../../../../shared/src/set-utils.js";
 import { BigIntJSON } from "../../../../../../shared/src/bigint-json.js";
-import { publishedIndexSpec, publishedTableSpec } from "../../../../db/specs.js";
+import { publishedColumnSpec, publishedIndexSpec, publishedTableSpec } from "../../../../db/specs.js";
 import { liteTableName } from "../../../../types/names.js";
 import { mapPostgresToLite, mapPostgresToLiteIndex } from "../../../../db/pg-to-lite.js";
 import { computeZqlSpecsFromLiteSpecs } from "../../../../db/lite-tables.js";
+import { getOrCreateCounter } from "../../../../observability/metrics.js";
+import { translatePostgresIndexPredicate } from "./index-predicate.js";
 import { literal } from "pg-format";
 //#region ../zero-cache/src/services/change-source/pg/schema/published.ts
-function publishedSchemaQuery(publications) {
+function publishedSchemaQuery(publications, includePartialIndexes = true) {
 	return `
 WITH published_columns AS (SELECT 
   pc.oid::int8 AS "oid",
@@ -22,6 +24,7 @@ WITH published_columns AS (SELECT
   atttypid::int8 AS "typeOID", 
   pt.typtype,
   elem_pt.typtype AS "elemTyptype",
+  coll.collisdeterministic AS "collationIsDeterministic",
   NULLIF(atttypmod, -1) AS "maxLen", 
   attndims "arrayDims", 
   attnotnull AS "notNull",
@@ -34,6 +37,7 @@ JOIN pg_class pc ON pc.oid = attrelid
 JOIN pg_namespace pns ON pns.oid = relnamespace
 JOIN pg_type pt ON atttypid = pt.oid
 LEFT JOIN pg_type elem_pt ON elem_pt.oid = pt.typelem
+LEFT JOIN pg_collation coll ON coll.oid = pg_attribute.attcollation
 JOIN pg_publication_tables as pb ON 
   pb.schemaname = nspname AND 
   pb.tablename = pc.relname AND
@@ -61,6 +65,7 @@ tables AS (SELECT json_build_object(
       'pgTypeClass', "typtype",
       'elemPgTypeClass', "elemTyptype",
       'typeOID', "typeOID",
+      'collationIsDeterministic', "collationIsDeterministic",
       -- https://stackoverflow.com/a/52376230
       'characterMaximumLength', CASE WHEN "typeOID" = 1043 OR "typeOID" = 1042 
                                      THEN "maxLen" - 4 
@@ -90,10 +95,17 @@ tables AS (SELECT json_build_object(
       pg_indexes.indexname as "name",
       index_column.name as "col",
       CASE WHEN pg_index.indoption[index_column.pos-1] & 1 = 1 THEN 'DESC' ELSE 'ASC' END as "dir",
-      pg_index.indisunique as "unique",
+      -- Partial indexes are always reported as non-unique. Nothing on the
+      -- replica relies on the uniqueness of a partial index (it cannot serve
+      -- as a row key), and reporting it as non-unique keeps the ddl event
+      -- format compatible with older versions of the zero-cache, which
+      -- ignore "predicateSQL" and would otherwise create a full UNIQUE index
+      -- on the replica.
+      (pg_index.indisunique AND pg_index.indpred IS NULL) as "unique",
       pg_index.indisprimary as "isPrimaryKey",
       pg_index.indisreplident as "isReplicaIdentity",
-      pg_index.indimmediate as "isImmediate"
+      pg_index.indimmediate as "isImmediate",
+      pg_get_expr(pg_index.indpred, pg_index.indrelid, false) as "predicateSQL"
     FROM pg_indexes
     JOIN pg_namespace ON pg_indexes.schemaname = pg_namespace.nspname
     JOIN pg_class pc ON
@@ -117,7 +129,7 @@ tables AS (SELECT json_build_object(
     LEFT JOIN pg_constraint ON pg_constraint.conindid = pc.oid
     WHERE pb.pubname IN (${literal(publications)})
       AND pg_index.indexprs IS NULL
-      AND pg_index.indpred IS NULL
+      AND (${includePartialIndexes} OR pg_index.indpred IS NULL)
       AND (pg_constraint.contype IS NULL OR pg_constraint.contype IN ('p', 'u'))
       AND indexed.attnames <@ pb.attnames
       AND (current_setting('server_version_num')::int >= 160000 OR false = ALL(indexed.generated))
@@ -135,10 +147,11 @@ tables AS (SELECT json_build_object(
       'isPrimaryKey', "isPrimaryKey",
       'isReplicaIdentity', "isReplicaIdentity",
       'isImmediate', "isImmediate",
+      'predicateSQL', "predicateSQL",
       'columns', json_object_agg("col", "dir")
     ) AS index FROM indexed_columns 
       GROUP BY "schema", "tableName", "name", "unique", 
-         "isPrimaryKey", "isReplicaIdentity", "isImmediate")
+         "isPrimaryKey", "isReplicaIdentity", "isImmediate", "predicateSQL")
 
     SELECT json_build_object(
       'tables', COALESCE((SELECT json_agg("table") FROM tables), '[]'::json),
@@ -146,12 +159,63 @@ tables AS (SELECT json_build_object(
     ) as "publishedSchema"
   `;
 }
+var rawPublishedColumnSpec = publishedColumnSpec.extend({ collationIsDeterministic: valita_exports.boolean().nullable().optional() });
+var rawPublishedTableSpec = publishedTableSpec.extend({ columns: valita_exports.record(rawPublishedColumnSpec) });
+var rawPublishedIndexSpec = publishedIndexSpec.extend({ predicateSQL: valita_exports.string().nullable().optional() });
+var skippedIndexDiagnostics = /* @__PURE__ */ new WeakMap();
+function getSkippedIndexDiagnostics(schema) {
+	return skippedIndexDiagnostics.get(schema) ?? [];
+}
+function warnForSkippedIndexes(lc, schema, warned = /* @__PURE__ */ new Set()) {
+	for (const diagnostic of getSkippedIndexDiagnostics(schema)) {
+		const key = JSON.stringify(diagnostic);
+		if (warned.has(key)) continue;
+		warned.add(key);
+		lc.warn?.("Skipping unsupported PostgreSQL partial index", diagnostic);
+		getOrCreateCounter("replication", "skipped_index", {
+			description: "Number of PostgreSQL indexes skipped during replication.",
+			unit: "{index}"
+		}).add(1, {
+			reason: diagnostic.reason,
+			partial: true
+		});
+	}
+}
 var publishedSchema = valita_exports.object({
-	tables: valita_exports.array(publishedTableSpec),
-	indexes: valita_exports.array(publishedIndexSpec)
-}).map(({ tables, indexes }) => {
+	tables: valita_exports.array(rawPublishedTableSpec),
+	indexes: valita_exports.array(rawPublishedIndexSpec)
+}).map(({ tables: rawTables, indexes: rawIndexes }) => {
+	const diagnostics = [];
+	const indexes = [];
+	for (const rawIndex of rawIndexes) {
+		const { predicateSQL, ...index } = rawIndex;
+		if (predicateSQL !== null && predicateSQL !== void 0) {
+			const table = rawTables.find((candidate) => candidate.schema === index.schema && candidate.name === index.tableName);
+			const translation = table ? translatePostgresIndexPredicate(predicateSQL, table) : {
+				supported: false,
+				reason: "unpublished-column"
+			};
+			if (!translation.supported) {
+				diagnostics.push({
+					schema: index.schema,
+					index: index.name,
+					predicate: predicateSQL,
+					reason: translation.reason
+				});
+				continue;
+			}
+			indexes.push({
+				...index,
+				predicate: translation.predicate
+			});
+		} else indexes.push(index);
+	}
+	const tables = rawTables.map((table) => ({
+		...table,
+		columns: Object.fromEntries(Object.entries(table.columns).map(([name, { collationIsDeterministic: _, ...column }]) => [name, column]))
+	}));
 	const zqlSpecs = computeZqlSpecsFromLiteSpecs(tables.map((t) => mapPostgresToLite(t)), indexes.map(mapPostgresToLiteIndex), { includeBackfillingColumns: true });
-	return {
+	const result = {
 		indexes,
 		tables: tables.map((table) => {
 			const replicaIdentityColumns = [];
@@ -174,6 +238,8 @@ var publishedSchema = valita_exports.object({
 			};
 		})
 	};
+	skippedIndexDiagnostics.set(result, diagnostics);
+	return result;
 });
 var publicationSchema = valita_exports.object({
 	pubname: valita_exports.string(),
@@ -212,12 +278,15 @@ async function getPublicationInfo(sql, publications) {
 		});
 	}
 	assert(result[2][0].publishedSchema, () => `Invalid publishedSchema result ${BigIntJSON.stringify(result[2])}`);
-	return {
+	const schema = parse(result[2][0].publishedSchema, publishedSchema);
+	const info = {
 		publications: parse(result[1], publicationsResultSchema),
-		...parse(result[2][0].publishedSchema, publishedSchema)
+		...schema
 	};
+	skippedIndexDiagnostics.set(info, getSkippedIndexDiagnostics(schema));
+	return info;
 }
 //#endregion
-export { getPublicationInfo, publishedSchema, publishedSchemaQuery };
+export { getPublicationInfo, publishedSchema, publishedSchemaQuery, warnForSkippedIndexes };
 
 //# sourceMappingURL=published.js.map

package/out/zero-cache/src/services/change-source/pg/schema/published.js.map

@@ -1 +1 @@
-{"version":3,"file":"published.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/published.ts"],"sourcesContent":["import {literal} from 'pg-format';\nimport type postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../../shared/src/bigint-json.ts';\nimport {equals} from '../../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {computeZqlSpecsFromLiteSpecs} from '../../../../db/lite-tables.ts';\nimport {\n  mapPostgresToLite,\n  mapPostgresToLiteIndex,\n} from '../../../../db/pg-to-lite.ts';\nimport {publishedIndexSpec, publishedTableSpec} from '../../../../db/specs.ts';\nimport {liteTableName} from '../../../../types/names.ts';\n\nexport function publishedSchemaQuery(publications: readonly string[]) {\n  // Notes:\n  // * There's a bug in PG15 in which generated columns are incorrectly\n  //   included in pg_publication_tables.attnames, (even though the generated\n  //   column values are not be included in the replication stream).\n  //   The WHERE condition `attgenerated = ''` fixes this by explicitly excluding\n  //   generated columns from the list.\n  return (\n    /*sql*/ `\nWITH published_columns AS (SELECT \n  pc.oid::int8 AS \"oid\",\n  nspname AS \"schema\",\n  pc.relnamespace::int8 AS \"schemaOID\" ,\n  pc.relname AS \"name\", \n  pc.relreplident AS \"replicaIdentity\",\n  attnum AS \"pos\", \n  attname AS \"col\", \n  pt.typname AS \"type\", \n  atttypid::int8 AS \"typeOID\", \n  pt.typtype,\n  elem_pt.typtype AS \"elemTyptype\",\n  NULLIF(atttypmod, -1) AS \"maxLen\", \n  attndims \"arrayDims\", \n  attnotnull AS \"notNull\",\n  pg_get_expr(pd.adbin, pd.adrelid) as \"dflt\",\n  NULLIF(ARRAY_POSITION(conkey, attnum), -1) AS \"keyPos\", \n  pb.rowfilter as \"rowFilter\",\n  pb.pubname as \"publication\"\nFROM pg_attribute\nJOIN pg_class pc ON pc.oid = attrelid\nJOIN pg_namespace pns ON pns.oid = relnamespace\nJOIN pg_type pt ON atttypid = pt.oid\nLEFT JOIN pg_type elem_pt ON elem_pt.oid = pt.typelem\nJOIN pg_publication_tables as pb ON \n  pb.schemaname = nspname AND \n  pb.tablename = pc.relname AND\n  attname = ANY(pb.attnames)\nLEFT JOIN pg_constraint pk ON pk.contype = 'p' AND pk.connamespace = relnamespace AND pk.conrelid = attrelid\nLEFT JOIN pg_attrdef pd ON pd.adrelid = attrelid AND pd.adnum = attnum\nWHERE pb.pubname IN (${literal(publications)}) AND \n      (current_setting('server_version_num')::int >= 160000 OR attgenerated = '')\nORDER BY nspname, pc.relname),\n\ntables AS (SELECT json_build_object(\n  'oid', \"oid\",\n  'schema', \"schema\", \n  'schemaOID', \"schemaOID\",\n  'name', \"name\", \n  'replicaIdentity', \"replicaIdentity\",\n  'columns', json_object_agg(\n    DISTINCT\n    col,\n    jsonb_build_object(\n      'pos', \"pos\",\n      'dataType', CASE WHEN \"arrayDims\" = 0 \n                       THEN \"type\" \n                       ELSE substring(\"type\" from 2) || repeat('[]', \"arrayDims\") END,\n      'pgTypeClass', \"typtype\",\n      'elemPgTypeClass', \"elemTyptype\",\n      'typeOID', \"typeOID\",\n      -- https://stackoverflow.com/a/52376230\n      'characterMaximumLength', CASE WHEN \"typeOID\" = 1043 OR \"typeOID\" = 1042 \n                                     THEN \"maxLen\" - 4 \n                                     ELSE \"maxLen\" END,\n      'notNull', \"notNull\",\n      'dflt', \"dflt\"\n    )\n  ),\n  'primaryKey', ARRAY( SELECT json_object_keys(\n    json_strip_nulls(\n      json_object_agg(\n        DISTINCT \"col\", \"keyPos\" ORDER BY \"keyPos\"\n      )\n    )\n  )),\n  'publications', json_object_agg(\n    DISTINCT \n    \"publication\", \n    jsonb_build_object('rowFilter', \"rowFilter\")\n  )\n) AS \"table\" FROM published_columns \n  GROUP BY \"schema\", \"schemaOID\", \"name\", \"oid\", \"replicaIdentity\"),\n  ` +\n    // Note: pg_attribute contains column names for tables and for indexes.\n    // However, the latter does not get updated when a column in a table is\n    // renamed.\n    //\n    // https://www.postgresql.org/message-id/5860814f-c91d-4ab0-b771-ded90d7b9c55%40www.fastmail.com\n    //\n    // To address this, the pg_attribute rows are looked up for the index's\n    // table rather than the index itself, using the pg_index.indkey array\n    // to determine the set and order of columns to include.\n    //\n    // Notes:\n    // * The first bit of indoption is 1 for DESC and 0 for ASC:\n    //   https://github.com/postgres/postgres/blob/4e1fad37872e49a711adad5d9870516e5c71a375/src/include/catalog/pg_index.h#L89\n    // * pg_index.indkey is an int2vector which is 0-based instead of 1-based.\n    // * The additional check for attgenerated is required for the aforementioned\n    //   (in publishedTableQuery) bug in PG15 in which generated columns are\n    //   incorrectly included in pg_publication_tables.attnames\n    /*sql*/ `\n  indexed_columns AS (SELECT\n      pg_indexes.schemaname as \"schema\",\n      pg_indexes.tablename as \"tableName\",\n      pg_indexes.indexname as \"name\",\n      index_column.name as \"col\",\n      CASE WHEN pg_index.indoption[index_column.pos-1] & 1 = 1 THEN 'DESC' ELSE 'ASC' END as \"dir\",\n      pg_index.indisunique as \"unique\",\n      pg_index.indisprimary as \"isPrimaryKey\",\n      pg_index.indisreplident as \"isReplicaIdentity\",\n      pg_index.indimmediate as \"isImmediate\"\n    FROM pg_indexes\n    JOIN pg_namespace ON pg_indexes.schemaname = pg_namespace.nspname\n    JOIN pg_class pc ON\n      pc.relname = pg_indexes.indexname\n      AND pc.relnamespace = pg_namespace.oid\n    JOIN pg_publication_tables as pb ON \n      pb.schemaname = pg_indexes.schemaname AND \n      pb.tablename = pg_indexes.tablename\n    JOIN pg_index ON pg_index.indexrelid = pc.oid\n    JOIN LATERAL (\n      SELECT array_agg(attname) as attnames, array_agg(attgenerated != '') as generated FROM pg_attribute\n        WHERE attrelid = pg_index.indrelid\n          AND attnum = ANY( (pg_index.indkey::smallint[] )[:pg_index.indnkeyatts - 1] )\n    ) as indexed ON true\n    JOIN LATERAL (\n      SELECT pg_attribute.attname as name, col.index_pos as pos\n        FROM UNNEST( (pg_index.indkey::smallint[])[:pg_index.indnkeyatts - 1] ) \n          WITH ORDINALITY as col(table_pos, index_pos)\n        JOIN pg_attribute ON attrelid = pg_index.indrelid AND attnum = col.table_pos\n    ) AS index_column ON true\n    LEFT JOIN pg_constraint ON pg_constraint.conindid = pc.oid\n    WHERE pb.pubname IN (${literal(publications)})\n      AND pg_index.indexprs IS NULL\n      AND pg_index.indpred IS NULL\n      AND (pg_constraint.contype IS NULL OR pg_constraint.contype IN ('p', 'u'))\n      AND indexed.attnames <@ pb.attnames\n      AND (current_setting('server_version_num')::int >= 160000 OR false = ALL(indexed.generated))\n    ORDER BY\n      pg_indexes.schemaname,\n      pg_indexes.tablename,\n      pg_indexes.indexname,\n      index_column.pos ASC),\n  \n    indexes AS (SELECT json_build_object(\n      'schema', \"schema\",\n      'tableName', \"tableName\",\n      'name', \"name\",\n      'unique', \"unique\",\n      'isPrimaryKey', \"isPrimaryKey\",\n      'isReplicaIdentity', \"isReplicaIdentity\",\n      'isImmediate', \"isImmediate\",\n      'columns', json_object_agg(\"col\", \"dir\")\n    ) AS index FROM indexed_columns \n      GROUP BY \"schema\", \"tableName\", \"name\", \"unique\", \n         \"isPrimaryKey\", \"isReplicaIdentity\", \"isImmediate\")\n\n    SELECT json_build_object(\n      'tables', COALESCE((SELECT json_agg(\"table\") FROM tables), '[]'::json),\n      'indexes', COALESCE((SELECT json_agg(\"index\") FROM indexes), '[]'::json)\n    ) as \"publishedSchema\"\n  `\n  );\n}\n\nexport const publishedSchema = v\n  .object({\n    tables: v.array(publishedTableSpec),\n    indexes: v.array(publishedIndexSpec),\n  })\n  .map(({tables, indexes}) => {\n    const zqlSpecs = computeZqlSpecsFromLiteSpecs(\n      tables.map(t => mapPostgresToLite(t)),\n      indexes.map(mapPostgresToLiteIndex),\n      {includeBackfillingColumns: true},\n    );\n    return {\n      indexes,\n\n      // Denormalize the schema such that each `table` includes the\n      // `replicaIdentityColumns` corresponding to the table's\n      // replica identity and associated primary key or index.\n      tables: tables.map(table => {\n        const replicaIdentityColumns: string[] = [];\n        switch (table.replicaIdentity) {\n          case 'd':\n            replicaIdentityColumns.push(...(table.primaryKey ?? []));\n            break;\n          case 'i':\n            replicaIdentityColumns.push(\n              ...Object.keys(\n                indexes.find(\n                  ind =>\n                    ind.schema === table.schema &&\n                    ind.tableName === table.name &&\n                    ind.isReplicaIdentity,\n                )?.columns ?? {},\n              ),\n            );\n            break;\n          case 'f': {\n            // For the key columns of replica identity \"full\", use the columns\n            // that the replicator/change-processor will end up using\n            // (in #getKey()) as the row key.\n            const zqlTable = zqlSpecs.get(liteTableName(table));\n            // Note: There zql spec will be absent if the table is not synced,\n            //       e.g. if it has no suitable unique indexes.\n            if (zqlTable) {\n              replicaIdentityColumns.push(...zqlTable.tableSpec.primaryKey);\n            }\n            break;\n          }\n        }\n        return {\n          ...table,\n          replicaIdentityColumns,\n        };\n      }),\n    };\n  });\n\nexport type PublishedSchema = v.Infer<typeof publishedSchema>;\n\nexport type PublishedTableWithReplicaIdentity =\n  PublishedSchema['tables'][number];\n\nconst publicationSchema = v.object({\n  pubname: v.string(),\n  pubinsert: v.boolean(),\n  pubupdate: v.boolean(),\n  pubdelete: v.boolean(),\n  pubtruncate: v.boolean(),\n});\n\nconst publicationsResultSchema = v.array(publicationSchema);\n\nexport type PublicationInfo = PublishedSchema & {\n  publications: v.Infer<typeof publicationsResultSchema>;\n};\n\n/**\n * Retrieves published tables and columns.\n */\nexport async function getPublicationInfo(\n  sql: postgres.Sql,\n  publications: string[],\n): Promise<PublicationInfo> {\n  const result = await sql.unsafe(/*sql*/ `\n  SELECT \n    schemaname AS \"schema\",\n    tablename AS \"table\", \n    json_object_agg(pubname, attnames) AS \"publications\"\n    FROM pg_publication_tables pb\n    WHERE pb.pubname IN (${literal(publications)})\n    GROUP BY schemaname, tablename;\n\n  SELECT ${Object.keys(publicationSchema.shape).join(\n    ',',\n  )} FROM pg_publication pb\n    WHERE pb.pubname IN (${literal(publications)})\n    ORDER BY pubname;\n\n  ${publishedSchemaQuery(publications)};\n`);\n\n  // The first query is used to check that tables in multiple publications\n  // always publish the same set of columns.\n  const publishedColumns = result[0] as {\n    schema: string;\n    table: string;\n    publications: Record<string, string[]>;\n  }[];\n  for (const {table, publications} of publishedColumns) {\n    let expected: Set<string>;\n    Object.entries(publications).forEach(([_, columns], i) => {\n      const cols = new Set(columns);\n      if (i === 0) {\n        expected = cols;\n      } else if (!equals(expected, cols)) {\n        throw new Error(\n          `Table ${table} is exported with different columns: [${[\n            ...expected,\n          ]}] vs [${[...cols]}]`,\n        );\n      }\n    });\n  }\n\n  assert(\n    result[2][0].publishedSchema,\n    () => `Invalid publishedSchema result ${BigIntJSON.stringify(result[2])}`,\n  );\n\n  return {\n    publications: v.parse(result[1], publicationsResultSchema),\n    ...v.parse(result[2][0].publishedSchema, publishedSchema),\n  };\n}\n"],"mappings":";;;;;;;;;;AAcA,SAAgB,qBAAqB,cAAiC;CAOpE,OACU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBA+BW,QAAQ,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BA6FlB,QAAQ,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BjD;AAEA,IAAa,kBAAkB,eAC5B,OAAO;CACN,QAAQ,eAAE,MAAM,kBAAkB;CAClC,SAAS,eAAE,MAAM,kBAAkB;AACrC,CAAC,EACA,KAAK,EAAC,QAAQ,cAAa;CAC1B,MAAM,WAAW,6BACf,OAAO,KAAI,MAAK,kBAAkB,CAAC,CAAC,GACpC,QAAQ,IAAI,sBAAsB,GAClC,EAAC,2BAA2B,KAAI,CAClC;CACA,OAAO;EACL;EAKA,QAAQ,OAAO,KAAI,UAAS;GAC1B,MAAM,yBAAmC,CAAC;GAC1C,QAAQ,MAAM,iBAAd;IACE,KAAK;KACH,uBAAuB,KAAK,GAAI,MAAM,cAAc,CAAC,CAAE;KACvD;IACF,KAAK;KACH,uBAAuB,KACrB,GAAG,OAAO,KACR,QAAQ,MACN,QACE,IAAI,WAAW,MAAM,UACrB,IAAI,cAAc,MAAM,QACxB,IAAI,iBACR,GAAG,WAAW,CAAC,CACjB,CACF;KACA;IACF,KAAK,KAAK;KAIR,MAAM,WAAW,SAAS,IAAI,cAAc,KAAK,CAAC;KAGlD,IAAI,UACF,uBAAuB,KAAK,GAAG,SAAS,UAAU,UAAU;KAE9D;IACF;GACF;GACA,OAAO;IACL,GAAG;IACH;GACF;EACF,CAAC;CACH;AACF,CAAC;AAOH,IAAM,oBAAoB,eAAE,OAAO;CACjC,SAAS,eAAE,OAAO;CAClB,WAAW,eAAE,QAAQ;CACrB,WAAW,eAAE,QAAQ;CACrB,WAAW,eAAE,QAAQ;CACrB,aAAa,eAAE,QAAQ;AACzB,CAAC;AAED,IAAM,2BAA2B,eAAE,MAAM,iBAAiB;;;;AAS1D,eAAsB,mBACpB,KACA,cAC0B;CAC1B,MAAM,SAAS,MAAM,IAAI,OAAe;;;;;;2BAMf,QAAQ,YAAY,EAAE;;;WAGtC,OAAO,KAAK,kBAAkB,KAAK,EAAE,KAC5C,GACF,EAAE;2BACuB,QAAQ,YAAY,EAAE;;;IAG7C,qBAAqB,YAAY,EAAE;CACtC;CAIC,MAAM,mBAAmB,OAAO;CAKhC,KAAK,MAAM,EAAC,OAAO,kBAAiB,kBAAkB;EACpD,IAAI;EACJ,OAAO,QAAQ,YAAY,EAAE,SAAS,CAAC,GAAG,UAAU,MAAM;GACxD,MAAM,OAAO,IAAI,IAAI,OAAO;GAC5B,IAAI,MAAM,GACR,WAAW;QACN,IAAI,CAAC,OAAO,UAAU,IAAI,GAC/B,MAAM,IAAI,MACR,SAAS,MAAM,wCAAwC,CACrD,GAAG,QACL,EAAE,QAAQ,CAAC,GAAG,IAAI,EAAE,EACtB;EAEJ,CAAC;CACH;CAEA,OACE,OAAO,GAAG,GAAG,uBACP,kCAAkC,WAAW,UAAU,OAAO,EAAE,GACxE;CAEA,OAAO;EACL,cAAc,MAAQ,OAAO,IAAI,wBAAwB;EACzD,GAAG,MAAQ,OAAO,GAAG,GAAG,iBAAiB,eAAe;CAC1D;AACF"}
+{"version":3,"file":"published.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/published.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport type postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../../shared/src/bigint-json.ts';\nimport {equals} from '../../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {computeZqlSpecsFromLiteSpecs} from '../../../../db/lite-tables.ts';\nimport {\n  mapPostgresToLite,\n  mapPostgresToLiteIndex,\n} from '../../../../db/pg-to-lite.ts';\nimport {\n  publishedColumnSpec,\n  publishedIndexSpec,\n  publishedTableSpec,\n  type PublishedIndexSpec,\n} from '../../../../db/specs.ts';\nimport {getOrCreateCounter} from '../../../../observability/metrics.ts';\nimport {liteTableName} from '../../../../types/names.ts';\nimport {\n  translatePostgresIndexPredicate,\n  type UnsupportedPredicateReason,\n} from './index-predicate.ts';\n\nexport function publishedSchemaQuery(\n  publications: readonly string[],\n  includePartialIndexes = true,\n) {\n  // Notes:\n  // * There's a bug in PG15 in which generated columns are incorrectly\n  //   included in pg_publication_tables.attnames, (even though the generated\n  //   column values are not be included in the replication stream).\n  //   The WHERE condition `attgenerated = ''` fixes this by explicitly excluding\n  //   generated columns from the list.\n  return (\n    /*sql*/ `\nWITH published_columns AS (SELECT \n  pc.oid::int8 AS \"oid\",\n  nspname AS \"schema\",\n  pc.relnamespace::int8 AS \"schemaOID\" ,\n  pc.relname AS \"name\", \n  pc.relreplident AS \"replicaIdentity\",\n  attnum AS \"pos\", \n  attname AS \"col\", \n  pt.typname AS \"type\", \n  atttypid::int8 AS \"typeOID\", \n  pt.typtype,\n  elem_pt.typtype AS \"elemTyptype\",\n  coll.collisdeterministic AS \"collationIsDeterministic\",\n  NULLIF(atttypmod, -1) AS \"maxLen\", \n  attndims \"arrayDims\", \n  attnotnull AS \"notNull\",\n  pg_get_expr(pd.adbin, pd.adrelid) as \"dflt\",\n  NULLIF(ARRAY_POSITION(conkey, attnum), -1) AS \"keyPos\", \n  pb.rowfilter as \"rowFilter\",\n  pb.pubname as \"publication\"\nFROM pg_attribute\nJOIN pg_class pc ON pc.oid = attrelid\nJOIN pg_namespace pns ON pns.oid = relnamespace\nJOIN pg_type pt ON atttypid = pt.oid\nLEFT JOIN pg_type elem_pt ON elem_pt.oid = pt.typelem\nLEFT JOIN pg_collation coll ON coll.oid = pg_attribute.attcollation\nJOIN pg_publication_tables as pb ON \n  pb.schemaname = nspname AND \n  pb.tablename = pc.relname AND\n  attname = ANY(pb.attnames)\nLEFT JOIN pg_constraint pk ON pk.contype = 'p' AND pk.connamespace = relnamespace AND pk.conrelid = attrelid\nLEFT JOIN pg_attrdef pd ON pd.adrelid = attrelid AND pd.adnum = attnum\nWHERE pb.pubname IN (${literal(publications)}) AND \n      (current_setting('server_version_num')::int >= 160000 OR attgenerated = '')\nORDER BY nspname, pc.relname),\n\ntables AS (SELECT json_build_object(\n  'oid', \"oid\",\n  'schema', \"schema\", \n  'schemaOID', \"schemaOID\",\n  'name', \"name\", \n  'replicaIdentity', \"replicaIdentity\",\n  'columns', json_object_agg(\n    DISTINCT\n    col,\n    jsonb_build_object(\n      'pos', \"pos\",\n      'dataType', CASE WHEN \"arrayDims\" = 0 \n                       THEN \"type\" \n                       ELSE substring(\"type\" from 2) || repeat('[]', \"arrayDims\") END,\n      'pgTypeClass', \"typtype\",\n      'elemPgTypeClass', \"elemTyptype\",\n      'typeOID', \"typeOID\",\n      'collationIsDeterministic', \"collationIsDeterministic\",\n      -- https://stackoverflow.com/a/52376230\n      'characterMaximumLength', CASE WHEN \"typeOID\" = 1043 OR \"typeOID\" = 1042 \n                                     THEN \"maxLen\" - 4 \n                                     ELSE \"maxLen\" END,\n      'notNull', \"notNull\",\n      'dflt', \"dflt\"\n    )\n  ),\n  'primaryKey', ARRAY( SELECT json_object_keys(\n    json_strip_nulls(\n      json_object_agg(\n        DISTINCT \"col\", \"keyPos\" ORDER BY \"keyPos\"\n      )\n    )\n  )),\n  'publications', json_object_agg(\n    DISTINCT \n    \"publication\", \n    jsonb_build_object('rowFilter', \"rowFilter\")\n  )\n) AS \"table\" FROM published_columns \n  GROUP BY \"schema\", \"schemaOID\", \"name\", \"oid\", \"replicaIdentity\"),\n  ` +\n    // Note: pg_attribute contains column names for tables and for indexes.\n    // However, the latter does not get updated when a column in a table is\n    // renamed.\n    //\n    // https://www.postgresql.org/message-id/5860814f-c91d-4ab0-b771-ded90d7b9c55%40www.fastmail.com\n    //\n    // To address this, the pg_attribute rows are looked up for the index's\n    // table rather than the index itself, using the pg_index.indkey array\n    // to determine the set and order of columns to include.\n    //\n    // Notes:\n    // * The first bit of indoption is 1 for DESC and 0 for ASC:\n    //   https://github.com/postgres/postgres/blob/4e1fad37872e49a711adad5d9870516e5c71a375/src/include/catalog/pg_index.h#L89\n    // * pg_index.indkey is an int2vector which is 0-based instead of 1-based.\n    // * The additional check for attgenerated is required for the aforementioned\n    //   (in publishedTableQuery) bug in PG15 in which generated columns are\n    //   incorrectly included in pg_publication_tables.attnames\n    /*sql*/ `\n  indexed_columns AS (SELECT\n      pg_indexes.schemaname as \"schema\",\n      pg_indexes.tablename as \"tableName\",\n      pg_indexes.indexname as \"name\",\n      index_column.name as \"col\",\n      CASE WHEN pg_index.indoption[index_column.pos-1] & 1 = 1 THEN 'DESC' ELSE 'ASC' END as \"dir\",\n      -- Partial indexes are always reported as non-unique. Nothing on the\n      -- replica relies on the uniqueness of a partial index (it cannot serve\n      -- as a row key), and reporting it as non-unique keeps the ddl event\n      -- format compatible with older versions of the zero-cache, which\n      -- ignore \"predicateSQL\" and would otherwise create a full UNIQUE index\n      -- on the replica.\n      (pg_index.indisunique AND pg_index.indpred IS NULL) as \"unique\",\n      pg_index.indisprimary as \"isPrimaryKey\",\n      pg_index.indisreplident as \"isReplicaIdentity\",\n      pg_index.indimmediate as \"isImmediate\",\n      pg_get_expr(pg_index.indpred, pg_index.indrelid, false) as \"predicateSQL\"\n    FROM pg_indexes\n    JOIN pg_namespace ON pg_indexes.schemaname = pg_namespace.nspname\n    JOIN pg_class pc ON\n      pc.relname = pg_indexes.indexname\n      AND pc.relnamespace = pg_namespace.oid\n    JOIN pg_publication_tables as pb ON \n      pb.schemaname = pg_indexes.schemaname AND \n      pb.tablename = pg_indexes.tablename\n    JOIN pg_index ON pg_index.indexrelid = pc.oid\n    JOIN LATERAL (\n      SELECT array_agg(attname) as attnames, array_agg(attgenerated != '') as generated FROM pg_attribute\n        WHERE attrelid = pg_index.indrelid\n          AND attnum = ANY( (pg_index.indkey::smallint[] )[:pg_index.indnkeyatts - 1] )\n    ) as indexed ON true\n    JOIN LATERAL (\n      SELECT pg_attribute.attname as name, col.index_pos as pos\n        FROM UNNEST( (pg_index.indkey::smallint[])[:pg_index.indnkeyatts - 1] ) \n          WITH ORDINALITY as col(table_pos, index_pos)\n        JOIN pg_attribute ON attrelid = pg_index.indrelid AND attnum = col.table_pos\n    ) AS index_column ON true\n    LEFT JOIN pg_constraint ON pg_constraint.conindid = pc.oid\n    WHERE pb.pubname IN (${literal(publications)})\n      AND pg_index.indexprs IS NULL\n      AND (${includePartialIndexes} OR pg_index.indpred IS NULL)\n      AND (pg_constraint.contype IS NULL OR pg_constraint.contype IN ('p', 'u'))\n      AND indexed.attnames <@ pb.attnames\n      AND (current_setting('server_version_num')::int >= 160000 OR false = ALL(indexed.generated))\n    ORDER BY\n      pg_indexes.schemaname,\n      pg_indexes.tablename,\n      pg_indexes.indexname,\n      index_column.pos ASC),\n  \n    indexes AS (SELECT json_build_object(\n      'schema', \"schema\",\n      'tableName', \"tableName\",\n      'name', \"name\",\n      'unique', \"unique\",\n      'isPrimaryKey', \"isPrimaryKey\",\n      'isReplicaIdentity', \"isReplicaIdentity\",\n      'isImmediate', \"isImmediate\",\n      'predicateSQL', \"predicateSQL\",\n      'columns', json_object_agg(\"col\", \"dir\")\n    ) AS index FROM indexed_columns \n      GROUP BY \"schema\", \"tableName\", \"name\", \"unique\", \n         \"isPrimaryKey\", \"isReplicaIdentity\", \"isImmediate\", \"predicateSQL\")\n\n    SELECT json_build_object(\n      'tables', COALESCE((SELECT json_agg(\"table\") FROM tables), '[]'::json),\n      'indexes', COALESCE((SELECT json_agg(\"index\") FROM indexes), '[]'::json)\n    ) as \"publishedSchema\"\n  `\n  );\n}\n\nconst rawPublishedColumnSpec = publishedColumnSpec.extend({\n  collationIsDeterministic: v.boolean().nullable().optional(),\n});\n\nconst rawPublishedTableSpec = publishedTableSpec.extend({\n  columns: v.record(rawPublishedColumnSpec),\n});\n\nconst rawPublishedIndexSpec = publishedIndexSpec.extend({\n  predicateSQL: v.string().nullable().optional(),\n});\n\nexport type SkippedIndexDiagnostic = {\n  readonly schema: string;\n  readonly index: string;\n  readonly predicate: string;\n  readonly reason: UnsupportedPredicateReason;\n};\n\nconst skippedIndexDiagnostics = new WeakMap<\n  object,\n  readonly SkippedIndexDiagnostic[]\n>();\nexport function getSkippedIndexDiagnostics(\n  schema: PublishedSchema,\n): readonly SkippedIndexDiagnostic[] {\n  return skippedIndexDiagnostics.get(schema) ?? [];\n}\n\nexport function warnForSkippedIndexes(\n  lc: LogContext,\n  schema: PublishedSchema,\n  warned: Set<string> = new Set(),\n): void {\n  for (const diagnostic of getSkippedIndexDiagnostics(schema)) {\n    const key = JSON.stringify(diagnostic);\n    if (warned.has(key)) continue;\n    warned.add(key);\n    lc.warn?.('Skipping unsupported PostgreSQL partial index', diagnostic);\n    getOrCreateCounter('replication', 'skipped_index', {\n      description: 'Number of PostgreSQL indexes skipped during replication.',\n      unit: '{index}',\n    }).add(1, {reason: diagnostic.reason, partial: true});\n  }\n}\n\nexport const publishedSchema = v\n  .object({\n    tables: v.array(rawPublishedTableSpec),\n    indexes: v.array(rawPublishedIndexSpec),\n  })\n  .map(({tables: rawTables, indexes: rawIndexes}) => {\n    const diagnostics: SkippedIndexDiagnostic[] = [];\n    const indexes: PublishedIndexSpec[] = [];\n    for (const rawIndex of rawIndexes) {\n      const {predicateSQL, ...index} = rawIndex;\n      if (predicateSQL !== null && predicateSQL !== undefined) {\n        const table = rawTables.find(\n          candidate =>\n            candidate.schema === index.schema &&\n            candidate.name === index.tableName,\n        );\n        const translation = table\n          ? translatePostgresIndexPredicate(predicateSQL, table)\n          : {supported: false as const, reason: 'unpublished-column' as const};\n        if (!translation.supported) {\n          diagnostics.push({\n            schema: index.schema,\n            index: index.name,\n            predicate: predicateSQL,\n            reason: translation.reason,\n          });\n          continue;\n        }\n        indexes.push({...index, predicate: translation.predicate});\n      } else {\n        indexes.push(index);\n      }\n    }\n\n    const tables = rawTables.map(table => ({\n      ...table,\n      columns: Object.fromEntries(\n        Object.entries(table.columns).map(\n          ([name, {collationIsDeterministic: _, ...column}]) => [name, column],\n        ),\n      ),\n    }));\n    const zqlSpecs = computeZqlSpecsFromLiteSpecs(\n      tables.map(t => mapPostgresToLite(t)),\n      indexes.map(mapPostgresToLiteIndex),\n      {includeBackfillingColumns: true},\n    );\n    const result = {\n      indexes,\n\n      // Denormalize the schema such that each `table` includes the\n      // `replicaIdentityColumns` corresponding to the table's\n      // replica identity and associated primary key or index.\n      tables: tables.map(table => {\n        const replicaIdentityColumns: string[] = [];\n        switch (table.replicaIdentity) {\n          case 'd':\n            replicaIdentityColumns.push(...(table.primaryKey ?? []));\n            break;\n          case 'i':\n            replicaIdentityColumns.push(\n              ...Object.keys(\n                indexes.find(\n                  ind =>\n                    ind.schema === table.schema &&\n                    ind.tableName === table.name &&\n                    ind.isReplicaIdentity,\n                )?.columns ?? {},\n              ),\n            );\n            break;\n          case 'f': {\n            // For the key columns of replica identity \"full\", use the columns\n            // that the replicator/change-processor will end up using\n            // (in #getKey()) as the row key.\n            const zqlTable = zqlSpecs.get(liteTableName(table));\n            // Note: There zql spec will be absent if the table is not synced,\n            //       e.g. if it has no suitable unique indexes.\n            if (zqlTable) {\n              replicaIdentityColumns.push(...zqlTable.tableSpec.primaryKey);\n            }\n            break;\n          }\n        }\n        return {\n          ...table,\n          replicaIdentityColumns,\n        };\n      }),\n    };\n    skippedIndexDiagnostics.set(result, diagnostics);\n    return result;\n  });\n\nexport type PublishedSchema = v.Infer<typeof publishedSchema>;\n\nexport type PublishedTableWithReplicaIdentity =\n  PublishedSchema['tables'][number];\n\nconst publicationSchema = v.object({\n  pubname: v.string(),\n  pubinsert: v.boolean(),\n  pubupdate: v.boolean(),\n  pubdelete: v.boolean(),\n  pubtruncate: v.boolean(),\n});\n\nconst publicationsResultSchema = v.array(publicationSchema);\n\nexport type PublicationInfo = PublishedSchema & {\n  publications: v.Infer<typeof publicationsResultSchema>;\n};\n\n/**\n * Retrieves published tables and columns.\n */\nexport async function getPublicationInfo(\n  sql: postgres.Sql,\n  publications: string[],\n): Promise<PublicationInfo> {\n  const result = await sql.unsafe(/*sql*/ `\n  SELECT \n    schemaname AS \"schema\",\n    tablename AS \"table\", \n    json_object_agg(pubname, attnames) AS \"publications\"\n    FROM pg_publication_tables pb\n    WHERE pb.pubname IN (${literal(publications)})\n    GROUP BY schemaname, tablename;\n\n  SELECT ${Object.keys(publicationSchema.shape).join(\n    ',',\n  )} FROM pg_publication pb\n    WHERE pb.pubname IN (${literal(publications)})\n    ORDER BY pubname;\n\n  ${publishedSchemaQuery(publications)};\n`);\n\n  // The first query is used to check that tables in multiple publications\n  // always publish the same set of columns.\n  const publishedColumns = result[0] as {\n    schema: string;\n    table: string;\n    publications: Record<string, string[]>;\n  }[];\n  for (const {table, publications} of publishedColumns) {\n    let expected: Set<string>;\n    Object.entries(publications).forEach(([_, columns], i) => {\n      const cols = new Set(columns);\n      if (i === 0) {\n        expected = cols;\n      } else if (!equals(expected, cols)) {\n        throw new Error(\n          `Table ${table} is exported with different columns: [${[\n            ...expected,\n          ]}] vs [${[...cols]}]`,\n        );\n      }\n    });\n  }\n\n  assert(\n    result[2][0].publishedSchema,\n    () => `Invalid publishedSchema result ${BigIntJSON.stringify(result[2])}`,\n  );\n\n  const schema = v.parse(result[2][0].publishedSchema, publishedSchema);\n  const info = {\n    publications: v.parse(result[1], publicationsResultSchema),\n    ...schema,\n  };\n  skippedIndexDiagnostics.set(info, getSkippedIndexDiagnostics(schema));\n  return info;\n}\n"],"mappings":";;;;;;;;;;;;AAyBA,SAAgB,qBACd,cACA,wBAAwB,MACxB;CAOA,OACU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBAiCW,QAAQ,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BAqGlB,QAAQ,YAAY,EAAE;;aAEpC,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BnC;AAEA,IAAM,yBAAyB,oBAAoB,OAAO,EACxD,0BAA0B,eAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAC5D,CAAC;AAED,IAAM,wBAAwB,mBAAmB,OAAO,EACtD,SAAS,eAAE,OAAO,sBAAsB,EAC1C,CAAC;AAED,IAAM,wBAAwB,mBAAmB,OAAO,EACtD,cAAc,eAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAC/C,CAAC;AASD,IAAM,0CAA0B,IAAI,QAGlC;AACF,SAAgB,2BACd,QACmC;CACnC,OAAO,wBAAwB,IAAI,MAAM,KAAK,CAAC;AACjD;AAEA,SAAgB,sBACd,IACA,QACA,yBAAsB,IAAI,IAAI,GACxB;CACN,KAAK,MAAM,cAAc,2BAA2B,MAAM,GAAG;EAC3D,MAAM,MAAM,KAAK,UAAU,UAAU;EACrC,IAAI,OAAO,IAAI,GAAG,GAAG;EACrB,OAAO,IAAI,GAAG;EACd,GAAG,OAAO,iDAAiD,UAAU;EACrE,mBAAmB,eAAe,iBAAiB;GACjD,aAAa;GACb,MAAM;EACR,CAAC,EAAE,IAAI,GAAG;GAAC,QAAQ,WAAW;GAAQ,SAAS;EAAI,CAAC;CACtD;AACF;AAEA,IAAa,kBAAkB,eAC5B,OAAO;CACN,QAAQ,eAAE,MAAM,qBAAqB;CACrC,SAAS,eAAE,MAAM,qBAAqB;AACxC,CAAC,EACA,KAAK,EAAC,QAAQ,WAAW,SAAS,iBAAgB;CACjD,MAAM,cAAwC,CAAC;CAC/C,MAAM,UAAgC,CAAC;CACvC,KAAK,MAAM,YAAY,YAAY;EACjC,MAAM,EAAC,cAAc,GAAG,UAAS;EACjC,IAAI,iBAAiB,QAAQ,iBAAiB,KAAA,GAAW;GACvD,MAAM,QAAQ,UAAU,MACtB,cACE,UAAU,WAAW,MAAM,UAC3B,UAAU,SAAS,MAAM,SAC7B;GACA,MAAM,cAAc,QAChB,gCAAgC,cAAc,KAAK,IACnD;IAAC,WAAW;IAAgB,QAAQ;GAA6B;GACrE,IAAI,CAAC,YAAY,WAAW;IAC1B,YAAY,KAAK;KACf,QAAQ,MAAM;KACd,OAAO,MAAM;KACb,WAAW;KACX,QAAQ,YAAY;IACtB,CAAC;IACD;GACF;GACA,QAAQ,KAAK;IAAC,GAAG;IAAO,WAAW,YAAY;GAAS,CAAC;EAC3D,OACE,QAAQ,KAAK,KAAK;CAEtB;CAEA,MAAM,SAAS,UAAU,KAAI,WAAU;EACrC,GAAG;EACH,SAAS,OAAO,YACd,OAAO,QAAQ,MAAM,OAAO,EAAE,KAC3B,CAAC,MAAM,EAAC,0BAA0B,GAAG,GAAG,cAAa,CAAC,MAAM,MAAM,CACrE,CACF;CACF,EAAE;CACF,MAAM,WAAW,6BACf,OAAO,KAAI,MAAK,kBAAkB,CAAC,CAAC,GACpC,QAAQ,IAAI,sBAAsB,GAClC,EAAC,2BAA2B,KAAI,CAClC;CACA,MAAM,SAAS;EACb;EAKA,QAAQ,OAAO,KAAI,UAAS;GAC1B,MAAM,yBAAmC,CAAC;GAC1C,QAAQ,MAAM,iBAAd;IACE,KAAK;KACH,uBAAuB,KAAK,GAAI,MAAM,cAAc,CAAC,CAAE;KACvD;IACF,KAAK;KACH,uBAAuB,KACrB,GAAG,OAAO,KACR,QAAQ,MACN,QACE,IAAI,WAAW,MAAM,UACrB,IAAI,cAAc,MAAM,QACxB,IAAI,iBACR,GAAG,WAAW,CAAC,CACjB,CACF;KACA;IACF,KAAK,KAAK;KAIR,MAAM,WAAW,SAAS,IAAI,cAAc,KAAK,CAAC;KAGlD,IAAI,UACF,uBAAuB,KAAK,GAAG,SAAS,UAAU,UAAU;KAE9D;IACF;GACF;GACA,OAAO;IACL,GAAG;IACH;GACF;EACF,CAAC;CACH;CACA,wBAAwB,IAAI,QAAQ,WAAW;CAC/C,OAAO;AACT,CAAC;AAOH,IAAM,oBAAoB,eAAE,OAAO;CACjC,SAAS,eAAE,OAAO;CAClB,WAAW,eAAE,QAAQ;CACrB,WAAW,eAAE,QAAQ;CACrB,WAAW,eAAE,QAAQ;CACrB,aAAa,eAAE,QAAQ;AACzB,CAAC;AAED,IAAM,2BAA2B,eAAE,MAAM,iBAAiB;;;;AAS1D,eAAsB,mBACpB,KACA,cAC0B;CAC1B,MAAM,SAAS,MAAM,IAAI,OAAe;;;;;;2BAMf,QAAQ,YAAY,EAAE;;;WAGtC,OAAO,KAAK,kBAAkB,KAAK,EAAE,KAC5C,GACF,EAAE;2BACuB,QAAQ,YAAY,EAAE;;;IAG7C,qBAAqB,YAAY,EAAE;CACtC;CAIC,MAAM,mBAAmB,OAAO;CAKhC,KAAK,MAAM,EAAC,OAAO,kBAAiB,kBAAkB;EACpD,IAAI;EACJ,OAAO,QAAQ,YAAY,EAAE,SAAS,CAAC,GAAG,UAAU,MAAM;GACxD,MAAM,OAAO,IAAI,IAAI,OAAO;GAC5B,IAAI,MAAM,GACR,WAAW;QACN,IAAI,CAAC,OAAO,UAAU,IAAI,GAC/B,MAAM,IAAI,MACR,SAAS,MAAM,wCAAwC,CACrD,GAAG,QACL,EAAE,QAAQ,CAAC,GAAG,IAAI,EAAE,EACtB;EAEJ,CAAC;CACH;CAEA,OACE,OAAO,GAAG,GAAG,uBACP,kCAAkC,WAAW,UAAU,OAAO,EAAE,GACxE;CAEA,MAAM,SAAS,MAAQ,OAAO,GAAG,GAAG,iBAAiB,eAAe;CACpE,MAAM,OAAO;EACX,cAAc,MAAQ,OAAO,IAAI,wBAAwB;EACzD,GAAG;CACL;CACA,wBAAwB,IAAI,MAAM,2BAA2B,MAAM,CAAC;CACpE,OAAO;AACT"}