@rocicorp/zero 0.2.2024101100 → 0.3.2024102300

package/out/zero-cache/src/services/change-streamer/pg/schema/ddl.js

@@ -1,9 +1,10 @@
-import { jsonObjectSchema } from '../../../../../../shared/src/json-schema.js';
+import { ident as id, literal as lit } from 'pg-format';
 import * as v from '../../../../../../shared/src/valita.js';
+import { filteredTableSpec, indexSpec } from '../../../../db/specs.js';
 import { indexDefinitionsQuery, publishedTableQuery } from './published.js';
-// Sent in the 'version' tag of "ddl" event messages. This is used to ensure
-// that the message constructed in the upstream Trigger function is compatible
-// with the code processing it in the zero-cache.
+// Sent in the 'version' tag of "ddlStart" and "ddlUpdate" event messages.
+// This is used to ensure that the message constructed in the upstream
+// Trigger function is compatible with the code processing it in the zero-cache.
 //
 // Increment this when changing the format of the contents of the "ddl" events.
 // This will allow old / incompatible code to detect the change and abort.
@@ -11,46 +12,124 @@ export const PROTOCOL_VERSION = 1;
 const triggerEvent = v.object({
     context: v.object({ query: v.string() }).rest(v.string()),
 });
-const createOrAlterTableTagSchema = v.union(v.literal('CREATE TABLE'), v.literal('ALTER TABLE'));
-const createIndexTagSchema = v.literal('CREATE INDEX');
-const createOrAlterTableEventSchema = triggerEvent.extend({
-    tag: createOrAlterTableTagSchema,
-    table: jsonObjectSchema.map(t => t), // TODO: Define FilteredTableSpec schema.
-    indexes: v.array(jsonObjectSchema.map(t => t)), // TODO: Define IndexSpec schema.
+// The Schema type encapsulates a snapshot of the tables and indexes that
+// are published / relevant to the shard.
+const publishedSchema = v.object({
+    tables: v.array(filteredTableSpec),
+    indexes: v.array(indexSpec),
 });
-const createIndexEventSchema = triggerEvent.extend({
-    tag: createIndexTagSchema,
-    index: jsonObjectSchema.map(t => t), // TODO: Define IndexSpec schema.
+// All DDL events contain a snapshot of the current tables and indexes that
+// are published / relevant to the shard.
+export const ddlEventSchema = triggerEvent.extend({
+    version: v.literal(PROTOCOL_VERSION),
+    schema: publishedSchema,
 });
-const dropTableTagSchema = v.literal('DROP TABLE');
-const dropIndexTagSchema = v.literal('DROP INDEX');
-const identifier = v.object({
-    schema: v.string(),
-    // `object_identity` field, as defined in
-    // https://www.postgresql.org/docs/current/functions-event-triggers.html#PG-EVENT-TRIGGER-SQL-DROP-FUNCTIONS
-    objectIdentity: v.string(),
+// The `ddlStart` message is computed before every DDL event, regardless of
+// whether the subsequent event affects the shard. Downstream processing should
+// capture the contained schema information in order to determine the schema
+// changes necessary to apply a subsequent `ddlUpdate` message. Note that a
+// `ddlUpdate` message may not follow, as updates determined to be irrelevant
+// to the shard will not result in a message. However, all `ddlUpdate` messages
+// are guaranteed to be preceded by a `ddlStart` message.
+export const ddlStartEventSchema = ddlEventSchema.extend({
+    type: v.literal('ddlStart'),
 });
-const dropTableEventSchema = triggerEvent.extend({
-    tag: dropTableTagSchema,
-    tables: v.array(identifier),
+/**
+ * An tableEvent indicates the table that was created or altered. Note that
+ * this may result in changes to indexes.
+ *
+ * Note that a table alteration can only consistent of a single change, whether
+ * that be the name of the table, or an aspect of a single column. In particular,
+ * if the update contains a new column and removes an old column, that must
+ * necessarily mean that the old column was renamed.
+ *
+ * tableEvents are only emitted for tables published to the shard.
+ */
+const tableEvent = v.object({
+    tag: v.union(v.literal('CREATE TABLE'), v.literal('ALTER TABLE')),
+    table: v.object({ schema: v.string(), name: v.string() }),
 });
-const dropIndexEventSchema = triggerEvent.extend({
-    tag: dropIndexTagSchema,
-    indexes: v.array(identifier),
+/**
+ * An indexEvent indicates an index that was manually created.
+ *
+ * indexEvents are only emitted for (indexes of) tables published to the shard.
+ */
+const indexEvent = v.object({
+    tag: v.literal('CREATE INDEX'),
+    index: v.object({ schema: v.string(), name: v.string() }),
 });
-export const ddlEventSchema = v.object({
-    type: v.literal('ddl'),
-    version: v.literal(PROTOCOL_VERSION),
-    event: v.union(createOrAlterTableEventSchema, createIndexEventSchema, dropTableEventSchema, dropIndexEventSchema),
+/**
+ * A drop event indicates the dropping of table(s) or index(es). Note
+ * that a `DROP TABLE` event can result in the dropping of both tables
+ * and their indexes.
+ *
+ * Drop events are emitted to all shards. It is up to the downstream
+ * processor to determine (from the preceding {@link DdlStartEvent})
+ * whether the dropped objects are relevant to the shard.
+ */
+const dropEvent = v.object({
+    tag: v.union(v.literal('DROP TABLE'), v.literal('DROP INDEX')),
 });
-export const errorEventSchema = v.object({
-    type: v.literal('error'),
-    event: triggerEvent.extend({ message: v.string() }).rest(v.string()),
+/**
+ * A publication event indicates a change in the visibility of tables
+ * and/or columns. This can mean any number of columns added or
+ * removed. However, it will never represent table or column renames
+ * (as those are only possible with `ALTER TABLE` statements).
+ *
+ * Publication events are not emitted if the altered publication is
+ * known and not included by the shard. However, due to the way
+ * Postgres Triggers work, the publication is not always known, and
+ * so it is possible for a shard to receive irrelevant publication
+ * events.
+ */
+const publicationEvent = v.object({
+    tag: v.literal('ALTER PUBLICATION'),
 });
-const COMMON_TRIGGER_FUNCTIONS = `
-CREATE OR REPLACE FUNCTION zero.get_trigger_context()
-RETURNS record
-AS $$
+/**
+ * The {@link DdlUpdateEvent} contains an updated schema resulting from
+ * a particular ddl event. The event type provides information
+ * (i.e. constraints) on the difference from the schema of the preceding
+ * {@link DdlStartEvent}.
+ *
+ * Note that in almost all cases (the exception being `CREATE` events),
+ * it is possible that there is no relevant difference between the
+ * ddl-start schema and the ddl-update schema, as many aspects of the
+ * schema (e.g. column constraints) are not relevant to downstream
+ * replication.
+ */
+export const ddlUpdateEventSchema = ddlEventSchema.extend({
+    type: v.literal('ddlUpdate'),
+    event: v.union(tableEvent, indexEvent, dropEvent, publicationEvent),
+});
+export const replicationEventSchema = v.union(ddlStartEventSchema, ddlUpdateEventSchema);
+// Creates a function that appends `_SHARD_ID` to the input.
+function append(shardID) {
+    return (name) => id(name + '_' + shardID);
+}
+/**
+ * Event trigger functions contain the core logic that are invoked by triggers.
+ *
+ * Note that although many of these functions can theoretically be parameterized and
+ * shared across shards, it is advantageous to keep the functions in each shard
+ * isolated from each other in order to avoid the complexity of shared-function
+ * versioning.
+ *
+ * In a sense, shards (and their triggers and functions) should be thought of as
+ * execution environments that can be updated at different schedules. If per-shard
+ * triggers called into shared functions, we would have to consider versioning the
+ * functions when changing their behavior, backwards compatibility, removal of
+ * unused versions, etc. (not unlike versioning of npm packages).
+ *
+ * Instead, we opt for the simplicity and isolation of having each shard
+ * completely own (and maintain) the entirety of its trigger/function stack.
+ */
+function createEventFunctionStatements(shardID, publications) {
+    const schema = append(shardID)('zero'); // e.g. "zero_SHARD_ID"
+    return `
+CREATE SCHEMA IF NOT EXISTS ${schema};
+
+CREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()
+RETURNS record AS $$
 DECLARE
   result record;
 BEGIN
@@ -59,179 +138,180 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION zero.emit_error(message TEXT)
-RETURNS void
-AS $$
-DECLARE
-  event text;
+
+CREATE OR REPLACE FUNCTION ${schema}.notice_ignore(object_id TEXT)
+RETURNS void AS $$
 BEGIN
-  SELECT json_build_object(
-    'type', 'error',
-    'event', json_build_object(
-      'context', zero.get_trigger_context(),
-      'message', message
-    )
-  ) into event;
-
-  PERFORM pg_logical_emit_message(true, 'zero', event);
+  RAISE NOTICE 'zero(%) ignoring %', ${lit(shardID)}, object_id;
 END
 $$ LANGUAGE plpgsql;
-`;
-export function replicateCreateOrAlterTable() {
-    return `
-CREATE OR REPLACE FUNCTION zero.replicate_create_or_alter_table()
-RETURNS event_trigger
-AS $$
+
+
+CREATE OR REPLACE FUNCTION ${schema}.schema_specs()
+RETURNS TEXT AS $$
 DECLARE
-  tag text;
   tables record;
   indexes record;
-  event text;
-BEGIN 
-  SELECT command_tag FROM pg_event_trigger_ddl_commands() INTO tag;
-  IF LENGTH(tag) = 0  -- should be impossible
-  THEN
-    RAISE EXCEPTION 'missing command_tag from pg_event_trigger_ddl_commands() for %', query;
-  END IF;
-
-  ${publishedTableQuery(undefined, `JOIN pg_event_trigger_ddl_commands() ddl ON ddl.objid = pc.oid`)} INTO tables;
+BEGIN
+  ${publishedTableQuery(publications)} INTO tables;
+  ${indexDefinitionsQuery(publications)} INTO indexes;
+  RETURN json_build_object(
+    'tables', tables.tables,
+    'indexes', indexes.indexes
+  );
+END
+$$ LANGUAGE plpgsql;
 
-  IF json_array_length(tables.tables) = 0
-  THEN RETURN; END IF;  -- not a table published by "pubPrefix"
 
-  IF json_array_length(tables.tables) > 1
-  THEN
-    PERFORM zero.emit_error(
-      FORMAT('unexpected number of tables for "%s": %s',
-        tag, json_array_length(tables.tables))
-    );
-    RETURN;
-  END IF;
+CREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()
+RETURNS event_trigger AS $$
+DECLARE
+  schema_specs TEXT;
+  message TEXT;
+BEGIN
+  SELECT ${schema}.schema_specs() INTO schema_specs;
 
-  ${indexDefinitionsQuery(undefined, `JOIN pg_event_trigger_ddl_commands() ddl on ddl.objid = pg_index.indrelid`)} INTO indexes;
-  
   SELECT json_build_object(
-    'type', 'ddl',
+    'type', 'ddlStart',
     'version', ${PROTOCOL_VERSION},
-    'event', json_build_object(
-      'context', zero.get_trigger_context(),
-      'tag', tag,
-      'table', tables.tables -> 0,
-      'indexes', indexes.indexes
-    )
-  ) INTO event;
-
-  PERFORM pg_logical_emit_message(true, 'zero', event);
-END 
+    'schema', schema_specs::json,
+    'context', ${schema}.get_trigger_context()
+  ) INTO message;
+
+  PERFORM pg_logical_emit_message(true, ${lit('zero/' + shardID)}, message);
+END
 $$ LANGUAGE plpgsql;
-  `;
-}
-export function replicateCreateIndex() {
-    return `
-CREATE OR REPLACE FUNCTION zero.replicate_create_index()
-RETURNS event_trigger
-AS $$
+
+
+CREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end(tag TEXT)
+RETURNS void AS $$
 DECLARE
-  indexes record;
-  event text;
-BEGIN 
-  ${indexDefinitionsQuery(undefined, `JOIN pg_event_trigger_ddl_commands() ddl on ddl.objid = pc.oid`)} INTO indexes;
+  publications TEXT[];
+  cmd RECORD;
+  target RECORD;
+  pub RECORD;
+  in_shard RECORD;
+  schema_specs TEXT;
+  message TEXT;
+  event TEXT;
+BEGIN
+  publications := ARRAY[${lit(publications)}];
+
+  SELECT objid, object_type, object_identity FROM pg_event_trigger_ddl_commands() LIMIT 1 INTO cmd;
+
+  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.
+
+  IF cmd.object_type = 'table' OR cmd.object_type = 'table column' THEN
+    SELECT ns.nspname as "schema", c.relname as "name" FROM pg_class AS c
+      JOIN pg_namespace As ns ON c.relnamespace = ns.oid
+      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname
+      WHERE c.oid = cmd.objid AND pb.pubname = ANY (publications)
+      INTO target;
+    IF target IS NULL THEN
+      PERFORM ${schema}.notice_ignore(cmd.object_identity);
+      RETURN;
+    END IF;
+
+    cmd.object_type := 'table';  -- normalize the 'table column' target to 'table'
+
+  ELSIF cmd.object_type = 'index' THEN
+    SELECT ns.nspname as "schema", c.relname as "name" FROM pg_class AS c
+      JOIN pg_namespace As ns ON c.relnamespace = ns.oid
+      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname
+      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename
+      WHERE c.oid = cmd.objid AND pb.pubname = ANY (publications)
+      INTO target;
+    IF target IS NULL THEN
+      PERFORM ${schema}.notice_ignore(cmd.object_identity);
+      RETURN;
+    END IF;
+
+  ELSIF cmd.object_type = 'publication relation' THEN
+    SELECT pb.pubname FROM pg_publication_rel AS rel
+      JOIN pg_publication AS pb ON pb.oid = rel.prpubid
+      WHERE rel.oid = cmd.objid AND pb.pubname = ANY (publications) 
+      INTO pub;
+    IF pub IS NULL THEN
+      PERFORM ${schema}.notice_ignore(cmd.object_identity);
+      RETURN;
+    END IF;
+
+  ELSIF cmd.object_type = 'publication namespace' THEN
+    SELECT pb.pubname FROM pg_publication_namespace AS ns
+      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid
+      WHERE ns.oid = cmd.objid AND pb.pubname = ANY (publications) 
+      INTO pub;
+    IF pub IS NULL THEN
+      PERFORM ${schema}.notice_ignore(cmd.object_identity);
+      RETURN;
+    END IF;
+  END IF;
 
-  IF json_array_length(indexes.indexes) = 0
-  THEN RETURN; END IF;  -- not an index published by "pubPrefix"
+  -- Construct and emit the DdlUpdateEvent message.
 
-  IF json_array_length(indexes.indexes) > 1
+  IF target IS NOT NULL
   THEN
-    PERFORM zero.emit_error(
-      FORMAT('unexpected number of indexes for "CREATE INDEX": %s', 
-        json_array_length(indexes.indexes))
-    );
-    RETURN;
+    SELECT json_build_object('tag', tag, cmd.object_type, target) INTO event;
+  ELSE
+    SELECT json_build_object('tag', tag) INTO event;
   END IF;
   
+  SELECT ${schema}.schema_specs() INTO schema_specs;
+
   SELECT json_build_object(
-    'type', 'ddl',
+    'type', 'ddlUpdate',
     'version', ${PROTOCOL_VERSION},
-    'event', json_build_object(
-      'context', zero.get_trigger_context(),
-      'tag', 'CREATE INDEX',
-      'index', indexes.indexes -> 0
-    )
-  ) INTO event;
-
-  PERFORM pg_logical_emit_message(true, 'zero', event);
+    'schema', schema_specs::json,
+    'event', event::json,
+    'context', ${schema}.get_trigger_context()
+  ) INTO message;
+
+  PERFORM pg_logical_emit_message(true, ${lit('zero/' + shardID)}, message);
 END
 $$ LANGUAGE plpgsql;
-  `;
+`;
 }
-const DROP_EVENT_TRIGGERS = `
-CREATE OR REPLACE FUNCTION zero.replicate_drop_event(tag TEXT, type TEXT, dropped TEXT)
-RETURNS void
-AS $$
-DECLARE
-  event text;
+const TAGS = [
+    'CREATE TABLE',
+    'ALTER TABLE',
+    'CREATE INDEX',
+    'DROP TABLE',
+    'DROP INDEX',
+    'ALTER PUBLICATION',
+];
+export function createEventTriggerStatements(shardID, publications) {
+    // Unlike functions, which are namespaced in shard-specific schemas,
+    // EVENT TRIGGER names are in the global namespace and instead have the shardID appended.
+    const sharded = append(shardID);
+    const schema = sharded('zero');
+    const triggers = [createEventFunctionStatements(shardID, publications)];
+    // A single ddl_command_start trigger covering all relevant tags.
+    triggers.push(`
+DROP EVENT TRIGGER IF EXISTS ${sharded('zero_ddl_start')};
+CREATE EVENT TRIGGER ${sharded('zero_ddl_start')}
+  ON ddl_command_start
+  WHEN TAG IN (${lit(TAGS)})
+  EXECUTE PROCEDURE ${schema}.emit_ddl_start();
+`);
+    // A per-tag ddl_command_end trigger that dispatches to ${schema}.emit_ddl_end(tag)
+    for (const tag of TAGS) {
+        const tagID = tag.toLowerCase().replace(' ', '_');
+        triggers.push(`
+CREATE OR REPLACE FUNCTION ${schema}.emit_${tagID}() 
+RETURNS event_trigger AS $$
 BEGIN
-  SELECT json_build_object(
-    'type', 'ddl',
-    'version', ${PROTOCOL_VERSION},
-    'event', json_build_object(
-      'context', zero.get_trigger_context(),
-      'tag', tag,
-      dropped, json_agg(json_build_object(
-        'schema', schema_name,
-        'objectIdentity', object_identity
-      ))
-    )
-  ) FROM pg_event_trigger_dropped_objects() 
-    WHERE object_type = type
-    INTO event;
-
-  PERFORM pg_logical_emit_message(true, 'zero', event);
+  PERFORM ${schema}.emit_ddl_end(${lit(tag)});
 END
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION zero.replicate_drop_table()
-RETURNS event_trigger
-AS $$
-BEGIN
-  PERFORM zero.replicate_drop_event('DROP TABLE', 'table', 'tables');
-END
-$$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION zero.replicate_drop_index()
-RETURNS event_trigger
-AS $$
-BEGIN
-  PERFORM zero.replicate_drop_event('DROP INDEX', 'index', 'indexes');
-END
-$$ LANGUAGE plpgsql;
-`;
-export function createEventTriggerStatements() {
-    return [
-        COMMON_TRIGGER_FUNCTIONS,
-        replicateCreateOrAlterTable(),
-        `DROP EVENT TRIGGER IF EXISTS zero_replicate_create_or_alter_table`,
-        `CREATE EVENT TRIGGER zero_replicate_create_or_alter_table
-      ON ddl_command_end
-      WHEN TAG IN ('CREATE TABLE', 'ALTER TABLE')
-      EXECUTE PROCEDURE zero.replicate_create_or_alter_table()`,
-        replicateCreateIndex(),
-        `DROP EVENT TRIGGER IF EXISTS zero_replicate_create_index`,
-        `CREATE EVENT TRIGGER zero_replicate_create_index
-      ON ddl_command_end
-      WHEN TAG IN ('CREATE INDEX')
-      EXECUTE PROCEDURE zero.replicate_create_index()`,
-        DROP_EVENT_TRIGGERS,
-        `DROP EVENT TRIGGER IF EXISTS zero_replicate_drop_table`,
-        `CREATE EVENT TRIGGER zero_replicate_drop_table
-      ON sql_drop
-      WHEN TAG IN ('DROP TABLE')
-      EXECUTE PROCEDURE zero.replicate_drop_table()`,
-        `DROP EVENT TRIGGER IF EXISTS zero_replicate_drop_index`,
-        `CREATE EVENT TRIGGER zero_replicate_drop_index
-        ON sql_drop
-        WHEN TAG IN ('DROP INDEX')
-        EXECUTE PROCEDURE zero.replicate_drop_index()`,
-    ].join(';\n');
+DROP EVENT TRIGGER IF EXISTS ${sharded(`zero_${tagID}`)};
+CREATE EVENT TRIGGER ${sharded(`zero_${tagID}`)}
+  ON ddl_command_end
+  WHEN TAG IN (${lit(tag)})
+  EXECUTE PROCEDURE ${schema}.emit_${tagID}();
+`);
+    }
+    return triggers.join('');
 }
 //# sourceMappingURL=ddl.js.map

package/out/zero-cache/src/services/change-streamer/pg/schema/ddl.js.map

@@ -1 +1 @@
-{"version":3,"file":"ddl.js","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-streamer/pg/schema/ddl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,6CAA6C,CAAC;AAC7E,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,EAAC,qBAAqB,EAAE,mBAAmB,EAAC,MAAM,gBAAgB,CAAC;AAE1E,4EAA4E;AAC5E,8EAA8E;AAC9E,iDAAiD;AACjD,EAAE;AACF,+EAA+E;AAC/E,0EAA0E;AAC1E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,EAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CACxD,CAAC,CAAC;AAEH,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CACzC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,EACzB,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CACzB,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AAEvD,MAAM,6BAA6B,GAAG,YAAY,CAAC,MAAM,CAAC;IACxD,GAAG,EAAE,2BAA2B;IAChC,KAAK,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAsB,CAAC,EAAE,yCAAyC;IACnG,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAc,CAAC,CAAC,EAAE,iCAAiC;CAC/F,CAAC,CAAC;AAMH,MAAM,sBAAsB,GAAG,YAAY,CAAC,MAAM,CAAC;IACjD,GAAG,EAAE,oBAAoB;IACzB,KAAK,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAc,CAAC,EAAE,iCAAiC;CACpF,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AAEnD,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,yCAAyC;IACzC,4GAA4G;IAC5G,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;CAC3B,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,YAAY,CAAC,MAAM,CAAC;IAC/C,GAAG,EAAE,kBAAkB;IACvB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;CAC5B,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,YAAY,CAAC,MAAM,CAAC;IAC/C,GAAG,EAAE,kBAAkB;IACvB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;CAC7B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IACtB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACpC,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,6BAA6B,EAC7B,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,CACrB;CACF,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACxB,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,EAAC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CACnE,CAAC,CAAC;AAIH,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BhC,CAAC;AAEF,MAAM,UAAU,2BAA2B;IACzC,OAAO;;;;;;;;;;;;;;;;IAgBL,mBAAmB,CACnB,SAAS,EACT,gEAAgE,CACjE;;;;;;;;;;;;;;IAcC,qBAAqB,CACrB,SAAS,EACT,2EAA2E,CAC5E;;;;iBAIc,gBAAgB;;;;;;;;;;;;GAY9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,OAAO;;;;;;;;IAQL,qBAAqB,CACrB,SAAS,EACT,gEAAgE,CACjE;;;;;;;;;;;;;;;;iBAgBc,gBAAgB;;;;;;;;;;;GAW9B,CAAC;AACJ,CAAC;AAED,MAAM,mBAAmB,GAAG;;;;;;;;;iBASX,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgChC,CAAC;AAEF,MAAM,UAAU,4BAA4B;IAC1C,OAAO;QACL,wBAAwB;QAExB,2BAA2B,EAAE;QAC7B,mEAAmE;QACnE;;;+DAG2D;QAE3D,oBAAoB,EAAE;QACtB,0DAA0D;QAC1D;;;sDAGkD;QAElD,mBAAmB;QACnB,wDAAwD;QACxD;;;oDAGgD;QAEhD,wDAAwD;QACxD;;;sDAGkD;KACnD,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAChB,CAAC"}
+{"version":3,"file":"ddl.js","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-streamer/pg/schema/ddl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,IAAI,EAAE,EAAE,OAAO,IAAI,GAAG,EAAC,MAAM,WAAW,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAC5D,OAAO,EAAC,iBAAiB,EAAE,SAAS,EAAC,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAC,qBAAqB,EAAE,mBAAmB,EAAC,MAAM,gBAAgB,CAAC;AAE1E,0EAA0E;AAC1E,sEAAsE;AACtE,gFAAgF;AAChF,EAAE;AACF,+EAA+E;AAC/E,0EAA0E;AAC1E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,EAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CACxD,CAAC,CAAC;AAEH,yEAAyE;AACzE,yCAAyC;AACzC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;CAC5B,CAAC,CAAC;AAIH,2EAA2E;AAC3E,yCAAyC;AACzC,MAAM,CAAC,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC;IAChD,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACpC,MAAM,EAAE,eAAe;CACxB,CAAC,CAAC;AAEH,2EAA2E;AAC3E,+EAA+E;AAC/E,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,+EAA+E;AAC/E,yDAAyD;AACzD,MAAM,CAAC,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,CAAC;IACvD,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;CAC5B,CAAC,CAAC;AAIH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACjE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAC,CAAC;CACxD,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IAC9B,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAC,CAAC;CACxD,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IACzB,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;CAC/D,CAAC,CAAC;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC;CACpC,CAAC,CAAC;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,cAAc,CAAC,MAAM,CAAC;IACxD,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;IAC5B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,gBAAgB,CAAC;CACpE,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAC3C,mBAAmB,EACnB,oBAAoB,CACrB,CAAC;AAIF,4DAA4D;AAC5D,SAAS,MAAM,CAAC,OAAe;IAC7B,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,GAAG,GAAG,GAAG,OAAO,CAAC,CAAC;AACpD,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,6BAA6B,CACpC,OAAe,EACf,YAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB;IAC/D,OAAO;8BACqB,MAAM;;6BAEP,MAAM;;;;;;;;;;;6BAWN,MAAM;;;uCAGI,GAAG,CAAC,OAAO,CAAC;;;;;6BAKtB,MAAM;;;;;;IAM/B,mBAAmB,CAAC,YAAY,CAAC;IACjC,qBAAqB,CAAC,YAAY,CAAC;;;;;;;;;6BASV,MAAM;;;;;;WAMxB,MAAM;;;;iBAIA,gBAAgB;;iBAEhB,MAAM;;;0CAGmB,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC;;;;;6BAKnC,MAAM;;;;;;;;;;;;0BAYT,GAAG,CAAC,YAAY,CAAC;;;;;;;;;;;;;gBAa3B,MAAM;;;;;;;;;;;;;;gBAcN,MAAM;;;;;;;;;;gBAUN,MAAM;;;;;;;;;;gBAUN,MAAM;;;;;;;;;;;;;;WAcX,MAAM;;;;iBAIA,gBAAgB;;;iBAGhB,MAAM;;;0CAGmB,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC;;;CAG/D,CAAC;AACF,CAAC;AAED,MAAM,IAAI,GAAG;IACX,cAAc;IACd,aAAa;IACb,cAAc;IACd,YAAY;IACZ,YAAY;IACZ,mBAAmB;CACX,CAAC;AAEX,MAAM,UAAU,4BAA4B,CAC1C,OAAe,EACf,YAAsB;IAEtB,oEAAoE;IACpE,yFAAyF;IACzF,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE/B,MAAM,QAAQ,GAAG,CAAC,6BAA6B,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;IAExE,iEAAiE;IACjE,QAAQ,CAAC,IAAI,CAAC;+BACe,OAAO,CAAC,gBAAgB,CAAC;uBACjC,OAAO,CAAC,gBAAgB,CAAC;;iBAE/B,GAAG,CAAC,IAAI,CAAC;sBACJ,MAAM;CAC3B,CAAC,CAAC;IAED,mFAAmF;IACnF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAClD,QAAQ,CAAC,IAAI,CAAC;6BACW,MAAM,SAAS,KAAK;;;YAGrC,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC;;;;;+BAKZ,OAAO,CAAC,QAAQ,KAAK,EAAE,CAAC;uBAChC,OAAO,CAAC,QAAQ,KAAK,EAAE,CAAC;;iBAE9B,GAAG,CAAC,GAAG,CAAC;sBACH,MAAM,SAAS,KAAK;CACzC,CAAC,CAAC;IACD,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC3B,CAAC"}

package/out/zero-cache/src/services/change-streamer/pg/schema/published.d.ts

@@ -1,8 +1,8 @@
 import type postgres from 'postgres';
 import * as v from '../../../../../../shared/src/valita.js';
-import type { FilteredTableSpec, IndexSpec } from '../../../../types/specs.js';
-export declare function publishedTableQuery(publications: string[] | undefined, join?: string): string;
-export declare function indexDefinitionsQuery(publications: string[] | undefined, join?: string): string;
+import type { FilteredTableSpec, IndexSpec } from '../../../../db/specs.js';
+export declare function publishedTableQuery(publications: string[]): string;
+export declare function indexDefinitionsQuery(publications: string[]): string;
 declare const publicationSchema: v.ObjectType<{
     pubname: v.Type<string>;
     pubinsert: v.Type<boolean>;
@@ -21,6 +21,6 @@ export type PublicationInfo = {
  * publications that start with "zero_" or "_zero_", but this can be
  * overridden by specifying a specific set of `publications`.
  */
-export declare function getPublicationInfo(sql: postgres.Sql, publications?: string[]): Promise<PublicationInfo>;
+export declare function getPublicationInfo(sql: postgres.Sql, publications: string[]): Promise<PublicationInfo>;
 export {};
 //# sourceMappingURL=published.d.ts.map

package/out/zero-cache/src/services/change-streamer/pg/schema/published.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"published.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-streamer/pg/schema/published.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAC5D,OAAO,KAAK,EAAC,iBAAiB,EAAE,SAAS,EAAC,MAAM,4BAA4B,CAAC;AAc7E,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,MAAM,EAAE,GAAG,SAAS,EAClC,IAAI,SAAK,UAkEV;AAMD,wBAAgB,qBAAqB,CACnC,YAAY,EAAE,MAAM,EAAE,GAAG,SAAS,EAClC,IAAI,SAAK,UAsDV;AAED,QAAA,MAAM,iBAAiB;;;;;;aAMrB,CAAC;AAIH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC;CAC/B,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,QAAQ,CAAC,GAAG,EACjB,YAAY,CAAC,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,eAAe,CAAC,CAiD1B"}
+{"version":3,"file":"published.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-streamer/pg/schema/published.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAC5D,OAAO,KAAK,EAAC,iBAAiB,EAAE,SAAS,EAAC,MAAM,yBAAyB,CAAC;AAM1E,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,MAAM,EAAE,UAgEzD;AAMD,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,UAwD3D;AAED,QAAA,MAAM,iBAAiB;;;;;;aAMrB,CAAC;AAIH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC;CAC/B,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,QAAQ,CAAC,GAAG,EACjB,YAAY,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC,CAiD1B"}

package/out/zero-cache/src/services/change-streamer/pg/schema/published.js

@@ -1,14 +1,7 @@
 import { literal } from 'pg-format';
 import { equals } from '../../../../../../shared/src/set-utils.js';
 import * as v from '../../../../../../shared/src/valita.js';
-import { APP_PUBLICATION_PREFIX, INTERNAL_PUBLICATION_PREFIX } from './zero.js';
-function restrictTo(publications) {
-    return publications !== undefined
-        ? `pb.pubname IN (${literal(publications)})`
-        : `( STARTS_WITH(pb.pubname, ${literal(APP_PUBLICATION_PREFIX)})
-      OR STARTS_WITH(pb.pubname, ${literal(INTERNAL_PUBLICATION_PREFIX)}) )`;
-}
-export function publishedTableQuery(publications, join = '') {
+export function publishedTableQuery(publications) {
     return `
 WITH published_columns AS (SELECT 
   nspname AS "schema", 
@@ -32,10 +25,9 @@ JOIN pg_publication_tables as pb ON
   pb.schemaname = nspname AND 
   pb.tablename = pc.relname AND
   attname = ANY(pb.attnames)
-${join}
 LEFT JOIN pg_constraint pk ON pk.contype = 'p' AND pk.connamespace = relnamespace AND pk.conrelid = attrelid
 LEFT JOIN pg_attrdef pd ON pd.adrelid = attrelid AND pd.adnum = attnum
-WHERE ${restrictTo(publications)}
+WHERE pb.pubname IN (${literal(publications)})
 ORDER BY nspname, pc.relname),
 
 tables AS (SELECT json_build_object(
@@ -74,7 +66,7 @@ tables AS (SELECT json_build_object(
 SELECT COALESCE(json_agg("table"), '[]'::json) as "tables" FROM tables
   `;
 }
-export function indexDefinitionsQuery(publications, join = '') {
+export function indexDefinitionsQuery(publications) {
     // Note: pg_attribute contains column names for tables and for indexes.
     // However, the latter does not get updated when a column in a table is
     // renamed.
@@ -84,12 +76,16 @@ export function indexDefinitionsQuery(publications, join = '') {
     // To address this, the pg_attribute rows are looked up for the index's
     // table rather than the index itself, using the pg_index.indkey array
     // to determine the set and order of columns to include.
+    //
+    // Note: The first bit of indoption is 1 for DESC and 0 for ASC:
+    // https://github.com/postgres/postgres/blob/4e1fad37872e49a711adad5d9870516e5c71a375/src/include/catalog/pg_index.h#L89
     return `
   WITH indexed_columns AS (SELECT
-      pg_indexes.schemaname as "schemaName",
+      pg_indexes.schemaname as "schema",
       pg_indexes.tablename as "tableName",
       pg_indexes.indexname as "name",
       index_column.name as "col",
+      CASE WHEN pg_index.indoption[index_column.pos-1] & 1 = 1 THEN 'DESC' ELSE 'ASC' END as "dir",
       pg_index.indisunique as "unique"
     FROM pg_indexes
     JOIN pg_namespace ON pg_indexes.schemaname = pg_namespace.nspname
@@ -105,9 +101,8 @@ export function indexDefinitionsQuery(publications, join = '') {
         FROM UNNEST(pg_index.indkey) WITH ORDINALITY as col(table_pos, index_pos)
         JOIN pg_attribute ON attrelid = pg_index.indrelid AND attnum = col.table_pos
     ) AS index_column ON true
-    ${join}
     LEFT JOIN pg_constraint ON pg_constraint.conindid = pc.oid
-    WHERE ${restrictTo(publications)}
+    WHERE pb.pubname IN (${literal(publications)})
       AND pg_constraint.contype is distinct from 'p'
       AND pg_constraint.contype is distinct from 'f'
     ORDER BY
@@ -117,13 +112,13 @@ export function indexDefinitionsQuery(publications, join = '') {
       index_column.pos ASC),
   
     indexes AS (SELECT json_build_object(
-      'schemaName', "schemaName",
+      'schema', "schema",
       'tableName', "tableName",
       'name', "name",
       'unique', "unique",
-      'columns', json_agg("col")
+      'columns', json_object_agg(DISTINCT "col", "dir")
     ) AS index FROM indexed_columns 
-      GROUP BY "schemaName", "tableName", "name", "unique")
+      GROUP BY "schema", "tableName", "name", "unique")
 
     SELECT COALESCE(json_agg("index"), '[]'::json) as "indexes" FROM indexes
   `;
@@ -148,11 +143,11 @@ export async function getPublicationInfo(sql, publications) {
     tablename AS "table", 
     json_object_agg(pubname, attnames) AS "publications"
     FROM pg_publication_tables pb
-    WHERE ${restrictTo(publications)}
+    WHERE pb.pubname IN (${literal(publications)})
     GROUP BY schemaname, tablename;
 
   SELECT ${Object.keys(publicationSchema.shape).join(',')} FROM pg_publication pb
-    WHERE ${restrictTo(publications)}
+    WHERE pb.pubname IN (${literal(publications)})
     ORDER BY pubname;
 
   ${publishedTableQuery(publications)};