@undefineds.co/xpod 0.3.75 → 0.4.0-rc.140

package/dist/storage/PostgresDerivedIndexJournal.js

@@ -1,408 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PostgresDerivedIndexJournal = exports.LEGACY_DERIVED_INDEX_CONSUMER_ID = void 0;
-const PostgresRdfSqlExecutor_1 = require("./rdf/PostgresRdfSqlExecutor");
-const PostgresPoolManager_1 = require("./database/PostgresPoolManager");
-exports.LEGACY_DERIVED_INDEX_CONSUMER_ID = 'legacy-resource-change-listener-v1';
-/** PostgreSQL outbox for CSS subscribe events consumed by FTS/VEC derivations. */
-class PostgresDerivedIndexJournal {
-    constructor(connectionString, executor, resolvePodScope, retryDelayMs = 1_000, leaseMs = 60_000, consumers = [], pollIntervalMs = 1_000) {
-        this.replaying = new Map();
-        const durableConsumers = [];
-        const consumerIds = new Set();
-        for (const consumer of consumers) {
-            const consumerId = consumer.consumerId;
-            if (typeof consumerId !== 'string' || !consumerId.trim()) {
-                throw new Error('Derived-index consumer requires a non-empty consumerId');
-            }
-            if (consumerIds.has(consumerId)) {
-                throw new Error(`Duplicate derived-index consumerId: ${consumerId}`);
-            }
-            consumerIds.add(consumerId);
-            durableConsumers.push(consumer);
-        }
-        const resolvedOptions = {
-            connectionString,
-            executor,
-            resolvePodScope,
-            retryDelayMs,
-            leaseMs,
-            consumers: durableConsumers,
-            pollIntervalMs,
-        };
-        this.options = resolvedOptions;
-        if (!resolvedOptions.executor && !resolvedOptions.connectionString) {
-            throw new Error('PostgresDerivedIndexJournal requires executor or connectionString');
-        }
-        this.consumers = new Map(durableConsumers.map((consumer) => [consumer.consumerId, consumer]));
-        this.activeConsumerIds = new Set(durableConsumers.length > 0
-            ? durableConsumers.map((consumer) => consumer.consumerId)
-            : [exports.LEGACY_DERIVED_INDEX_CONSUMER_ID]);
-        this.executor = resolvedOptions.executor
-            ?? new PostgresRdfSqlExecutor_1.PgPoolRdfSqlExecutor((0, PostgresPoolManager_1.getSharedPool)({ connectionString: resolvedOptions.connectionString }));
-        this.ownsSharedPool = !resolvedOptions.executor;
-        this.retryDelayMs = resolvedOptions.retryDelayMs ?? 1_000;
-        this.leaseMs = resolvedOptions.leaseMs ?? 60_000;
-        if (resolvedOptions.consumers?.length) {
-            this.pollTimer = setInterval(() => this.scheduleReplay(), pollIntervalMs);
-            this.pollTimer.unref?.();
-            setImmediate(() => this.scheduleReplay());
-        }
-    }
-    async open() {
-        this.opening ??= this.initialize();
-        await this.opening;
-    }
-    async initialize() {
-        await this.executor.exec(`
-      CREATE TABLE IF NOT EXISTS derived_index_change_journal (
-        id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
-        pod_scope_id TEXT NOT NULL,
-        resource_path TEXT NOT NULL,
-        action TEXT NOT NULL CHECK (action IN ('create', 'update', 'delete')),
-        is_container BOOLEAN NOT NULL,
-        occurred_at BIGINT NOT NULL,
-        stage TEXT NOT NULL DEFAULT 'pending' CHECK (stage IN ('pending', 'processing', 'done')),
-        attempts INTEGER NOT NULL DEFAULT 0,
-        available_at BIGINT NOT NULL DEFAULT 0,
-        lease_until BIGINT,
-        last_error TEXT
-      );
-      CREATE INDEX IF NOT EXISTS derived_index_change_journal_pending
-        ON derived_index_change_journal (stage, available_at, pod_scope_id, id);
-      CREATE TABLE IF NOT EXISTS derived_index_journal_metadata (
-        key TEXT PRIMARY KEY,
-        value TEXT NOT NULL
-      );
-      CREATE TABLE IF NOT EXISTS derived_index_consumers (
-        consumer_id TEXT PRIMARY KEY,
-        created_at BIGINT NOT NULL
-      );
-      CREATE TABLE IF NOT EXISTS derived_index_event_deliveries (
-        consumer_id TEXT NOT NULL REFERENCES derived_index_consumers(consumer_id),
-        event_id BIGINT NOT NULL REFERENCES derived_index_change_journal(id) ON DELETE CASCADE,
-        stage TEXT NOT NULL DEFAULT 'pending' CHECK (stage IN ('pending', 'processing', 'done')),
-        attempts INTEGER NOT NULL DEFAULT 0,
-        available_at BIGINT NOT NULL DEFAULT 0,
-        lease_until BIGINT,
-        last_error TEXT,
-        PRIMARY KEY (consumer_id, event_id)
-      );
-      CREATE INDEX IF NOT EXISTS derived_index_event_deliveries_pending
-        ON derived_index_event_deliveries (consumer_id, stage, available_at, event_id);
-      CREATE TABLE IF NOT EXISTS derived_index_resource_checkpoints (
-        consumer_id TEXT NOT NULL REFERENCES derived_index_consumers(consumer_id),
-        pod_scope_id TEXT NOT NULL,
-        resource_path TEXT NOT NULL,
-        last_event_id BIGINT NOT NULL REFERENCES derived_index_change_journal(id),
-        last_action TEXT NOT NULL CHECK (last_action IN ('create', 'update', 'delete')),
-        updated_at BIGINT NOT NULL,
-        deleted_at BIGINT,
-        PRIMARY KEY (consumer_id, pod_scope_id, resource_path)
-      );
-    `);
-        await this.executor.transaction(async (tx) => {
-            const sourceTable = await tx.query(`
-        SELECT to_regclass('rdf_sources')::text AS relation
-      `);
-            if (!sourceTable[0]?.relation)
-                return;
-            const inserted = await tx.query(`
-        INSERT INTO derived_index_journal_metadata (key, value)
-        VALUES ('rdf_sources_bootstrap_v1', 'complete')
-        ON CONFLICT (key) DO NOTHING
-        RETURNING key
-      `);
-            if (!inserted.length)
-                return;
-            await tx.exec(`
-        INSERT INTO derived_index_change_journal
-          (pod_scope_id, resource_path, action, is_container, occurred_at)
-        SELECT workspace, source, 'update', FALSE,
-               (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
-        FROM rdf_sources
-        ORDER BY id
-      `);
-        });
-        const configuredConsumers = this.options.consumers ?? [];
-        if (configuredConsumers.length === 0) {
-            await this.registerConsumer(exports.LEGACY_DERIVED_INDEX_CONSUMER_ID, true);
-        }
-        else {
-            for (const consumer of configuredConsumers) {
-                await this.registerConsumer(consumer.consumerId, false);
-            }
-        }
-    }
-    async registerConsumer(consumerId, legacy) {
-        await this.executor.transaction(async (tx) => {
-            const now = Date.now();
-            await tx.exec(`
-        INSERT INTO derived_index_consumers (consumer_id, created_at)
-        VALUES ($1, $2)
-        ON CONFLICT (consumer_id) DO NOTHING
-      `, [consumerId, now]);
-            await tx.exec(`
-        INSERT INTO derived_index_event_deliveries
-          (consumer_id, event_id, stage, attempts, available_at, lease_until, last_error)
-        SELECT $1, id,
-               CASE WHEN $2::boolean AND stage = 'done' THEN 'done' ELSE 'pending' END,
-               CASE WHEN $2::boolean THEN attempts ELSE 0 END,
-               CASE WHEN $2::boolean THEN available_at ELSE 0 END,
-               NULL,
-               CASE WHEN $2::boolean THEN last_error ELSE NULL END
-        FROM derived_index_change_journal
-        ON CONFLICT (consumer_id, event_id) DO NOTHING
-      `, [consumerId, legacy]);
-            if (legacy) {
-                await tx.exec(`
-          INSERT INTO derived_index_resource_checkpoints
-            (consumer_id, pod_scope_id, resource_path, last_event_id,
-             last_action, updated_at, deleted_at)
-          SELECT $1, latest.pod_scope_id, latest.resource_path, latest.id,
-                 latest.action, latest.occurred_at,
-                 CASE WHEN latest.action = 'delete' THEN latest.occurred_at ELSE NULL END
-          FROM (
-            SELECT DISTINCT ON (pod_scope_id, resource_path)
-                   id, pod_scope_id, resource_path, action, occurred_at
-            FROM derived_index_change_journal
-            WHERE stage = 'done'
-            ORDER BY pod_scope_id, resource_path, id DESC
-          ) latest
-          ON CONFLICT (consumer_id, pod_scope_id, resource_path) DO UPDATE
-          SET last_event_id = EXCLUDED.last_event_id,
-              last_action = EXCLUDED.last_action,
-              updated_at = EXCLUDED.updated_at,
-              deleted_at = EXCLUDED.deleted_at
-          WHERE derived_index_resource_checkpoints.last_event_id < EXCLUDED.last_event_id
-        `, [consumerId]);
-            }
-        });
-    }
-    async recordResourceChange(event) {
-        await this.open();
-        const podScopeId = await (this.options.resolvePodScope?.(event) ?? defaultPodScope(event.path));
-        if (!podScopeId) {
-            throw new Error(`Cannot journal derived-index change without Pod scope: ${event.path}`);
-        }
-        await this.append(podScopeId, event);
-        this.scheduleReplay();
-    }
-    async append(podScopeId, event) {
-        await this.executor.transaction(async (tx) => {
-            const inserted = await tx.query(`
-        INSERT INTO derived_index_change_journal
-          (pod_scope_id, resource_path, action, is_container, occurred_at)
-        VALUES ($1, $2, $3, $4, $5)
-        RETURNING id
-      `, [podScopeId, event.path, event.action, event.isContainer, event.timestamp]);
-            const eventId = inserted[0]?.id;
-            if (eventId === undefined) {
-                throw new Error(`Failed to append derived-index event: ${event.path}`);
-            }
-            for (const consumerId of this.activeConsumerIds) {
-                await tx.exec(`
-          INSERT INTO derived_index_event_deliveries (consumer_id, event_id)
-          VALUES ($1, $2)
-          ON CONFLICT (consumer_id, event_id) DO NOTHING
-        `, [consumerId, eventId]);
-            }
-        });
-    }
-    async replayPending(listener, limit = 100) {
-        await this.open();
-        this.activeConsumerIds.add(exports.LEGACY_DERIVED_INDEX_CONSUMER_ID);
-        await this.registerConsumer(exports.LEGACY_DERIVED_INDEX_CONSUMER_ID, true);
-        return this.replayDelivery(exports.LEGACY_DERIVED_INDEX_CONSUMER_ID, listener, limit);
-    }
-    async replayConsumer(consumerId, limit = 100) {
-        await this.open();
-        const consumer = this.consumers.get(consumerId);
-        if (!consumer) {
-            throw new Error(`Unknown derived-index consumerId: ${consumerId}`);
-        }
-        return this.replayDelivery(consumerId, consumer, limit);
-    }
-    async replayDelivery(consumerId, listener, limit) {
-        let delivered = 0;
-        let failed = 0;
-        for (let index = 0; index < limit; index += 1) {
-            const row = await this.claimNext(consumerId);
-            if (!row)
-                break;
-            const event = {
-                path: row.resource_path,
-                action: row.action,
-                isContainer: row.is_container,
-                timestamp: Number(row.occurred_at),
-            };
-            try {
-                await listener.onResourceChanged(event);
-                await this.completeDelivery(consumerId, row);
-                delivered += 1;
-            }
-            catch (error) {
-                await this.executor.exec(`
-          UPDATE derived_index_event_deliveries
-          SET stage = 'pending', attempts = attempts + 1,
-              available_at = $3, lease_until = NULL, last_error = $4
-          WHERE consumer_id = $1 AND event_id = $2
-        `, [consumerId, row.id, Date.now() + this.retryDelayMs, errorMessage(error)]);
-                failed += 1;
-            }
-        }
-        return { delivered, failed };
-    }
-    async reconcilePod(podScopeId, resourcePaths) {
-        await this.open();
-        const authorityPaths = [...new Set(resourcePaths)];
-        for (const resourcePath of authorityPaths) {
-            await this.append(podScopeId, {
-                path: resourcePath,
-                action: 'update',
-                isContainer: resourcePath.endsWith('/'),
-                timestamp: Date.now(),
-            });
-        }
-        const missing = await this.executor.query(`
-      SELECT resource_path, MAX(last_event_id) AS last_event_id
-      FROM derived_index_resource_checkpoints
-      WHERE pod_scope_id = $1
-        AND deleted_at IS NULL
-        AND NOT (resource_path = ANY($2::text[]))
-      GROUP BY resource_path
-      ORDER BY resource_path
-    `, [podScopeId, authorityPaths]);
-        for (const { resource_path: resourcePath, last_event_id: lastEventId } of missing) {
-            const covered = await this.executor.query(`
-        SELECT EXISTS (
-          SELECT 1
-          FROM derived_index_change_journal event
-          WHERE event.pod_scope_id = $1
-            AND event.resource_path = $2
-            AND event.action = 'delete'
-            AND event.id > $3
-        ) AS covered
-      `, [podScopeId, resourcePath, lastEventId]);
-            if (covered[0]?.covered)
-                continue;
-            await this.append(podScopeId, {
-                path: resourcePath,
-                action: 'delete',
-                isContainer: resourcePath.endsWith('/'),
-                timestamp: Date.now(),
-            });
-        }
-    }
-    async completeDelivery(consumerId, row) {
-        const completedAt = Date.now();
-        await this.executor.transaction(async (tx) => {
-            await tx.exec(`
-        UPDATE derived_index_event_deliveries
-        SET stage = 'done', lease_until = NULL, last_error = NULL
-        WHERE consumer_id = $1 AND event_id = $2
-      `, [consumerId, row.id]);
-            await tx.exec(`
-        INSERT INTO derived_index_resource_checkpoints
-          (consumer_id, pod_scope_id, resource_path, last_event_id,
-           last_action, updated_at, deleted_at)
-        VALUES ($1, $2, $3, $4, $5::text, $6::bigint,
-                CASE WHEN $5::text = 'delete' THEN $6::bigint ELSE NULL::bigint END)
-        ON CONFLICT (consumer_id, pod_scope_id, resource_path) DO UPDATE
-        SET last_event_id = EXCLUDED.last_event_id,
-            last_action = EXCLUDED.last_action,
-            updated_at = EXCLUDED.updated_at,
-            deleted_at = EXCLUDED.deleted_at
-        WHERE derived_index_resource_checkpoints.last_event_id < EXCLUDED.last_event_id
-      `, [consumerId, row.pod_scope_id, row.resource_path, row.id, row.action, completedAt]);
-        });
-    }
-    async pendingCount(podScopeId, consumerId) {
-        await this.open();
-        const rows = await this.executor.query(`
-      SELECT COUNT(DISTINCT delivery.event_id) AS count
-      FROM derived_index_event_deliveries delivery
-      JOIN derived_index_change_journal event ON event.id = delivery.event_id
-      WHERE delivery.stage <> 'done'
-        AND ($1::text IS NULL OR event.pod_scope_id = $1)
-        AND ($2::text IS NULL OR delivery.consumer_id = $2)
-    `, [podScopeId ?? null, consumerId ?? null]);
-        return Number(rows[0]?.count ?? 0);
-    }
-    async claimNext(consumerId) {
-        return this.executor.transaction(async (tx) => {
-            const now = Date.now();
-            await tx.exec(`
-        UPDATE derived_index_event_deliveries
-        SET stage = 'pending', lease_until = NULL
-        WHERE consumer_id = $1 AND stage = 'processing' AND lease_until < $2
-      `, [consumerId, now]);
-            const rows = await tx.query(`
-        SELECT event.id, event.pod_scope_id, event.resource_path, event.action,
-               event.is_container, event.occurred_at
-        FROM derived_index_event_deliveries delivery
-        JOIN derived_index_change_journal event ON event.id = delivery.event_id
-        WHERE delivery.consumer_id = $1
-          AND delivery.stage = 'pending'
-          AND delivery.available_at <= $2
-          AND NOT EXISTS (
-            SELECT 1
-            FROM derived_index_event_deliveries earlier_delivery
-            JOIN derived_index_change_journal earlier_event
-              ON earlier_event.id = earlier_delivery.event_id
-            WHERE earlier_delivery.consumer_id = delivery.consumer_id
-              AND earlier_event.pod_scope_id = event.pod_scope_id
-              AND earlier_event.id < event.id
-              AND earlier_delivery.stage <> 'done'
-          )
-        ORDER BY event.id
-        LIMIT 1
-        FOR UPDATE SKIP LOCKED
-      `, [consumerId, now]);
-            const row = rows[0];
-            if (!row)
-                return undefined;
-            await tx.exec(`
-        UPDATE derived_index_event_deliveries
-        SET stage = 'processing', lease_until = $3
-        WHERE consumer_id = $1 AND event_id = $2
-      `, [consumerId, row.id, now + this.leaseMs]);
-            return row;
-        });
-    }
-    async close() {
-        if (this.pollTimer)
-            clearInterval(this.pollTimer);
-        await Promise.all(this.replaying.values());
-        if (this.ownsSharedPool && this.options.connectionString) {
-            (0, PostgresPoolManager_1.releaseSharedPool)({ connectionString: this.options.connectionString });
-        }
-    }
-    scheduleReplay() {
-        for (const consumerId of this.consumers.keys()) {
-            if (this.replaying.has(consumerId))
-                continue;
-            const replay = this.replayConsumer(consumerId)
-                .then(() => undefined)
-                .catch(() => undefined)
-                .finally(() => { this.replaying.delete(consumerId); });
-            this.replaying.set(consumerId, replay);
-        }
-    }
-}
-exports.PostgresDerivedIndexJournal = PostgresDerivedIndexJournal;
-function errorMessage(error) {
-    return error instanceof Error ? error.message : String(error);
-}
-function defaultPodScope(resourcePath) {
-    try {
-        const url = new URL(resourcePath);
-        const firstSegment = url.pathname.split('/').filter(Boolean)[0];
-        return firstSegment ? `${url.origin}/${firstSegment}/` : `${url.origin}/`;
-    }
-    catch {
-        const firstSegment = resourcePath.split('/').filter(Boolean)[0];
-        return firstSegment ?? resourcePath;
-    }
-}
-//# sourceMappingURL=PostgresDerivedIndexJournal.js.map