@purposeinplay/payload-version-retention 0.1.1

package/dist/lib/startup-checks.js

@@ -0,0 +1,225 @@
+import { JANITOR_TASK_SLUG, JOBS_COLLECTION_SLUG, LOG_PREFIX, STATE_GLOBAL_SLUG } from '../defaults.js';
+/**
+ * Slug of the global Payload injects into the config as soon as any task or
+ * workflow declares a `schedule` (`jobs.scheduling` is flipped on during
+ * config sanitization). Its table is `payload_jobs_stats`.
+ */ export const JOB_STATS_GLOBAL_SLUG = 'payload-jobs-stats';
+/** Queue `handleSchedules` falls back to when an autoRun entry names none. */ const DEFAULT_AUTORUN_QUEUE = 'default';
+const MISSING_TABLE_MESSAGE = [
+    `${LOG_PREFIX} Could not read the \`payload-jobs-stats\` global.`,
+    `This plugin schedules its \`${JANITOR_TASK_SLUG}\` task, which turns on Payload job`,
+    'scheduling config-wide and requires the `payload_jobs_stats` table',
+    '(wild consumers additionally need the `payload_jobs.meta` column).',
+    'Run `payload migrate:create` and `payload migrate` — the plugin adoption and',
+    'that migration must ship in the same release.',
+    'Until the table exists, `handleSchedules` throws on every autoRun tick before',
+    '`jobs.run` is reached, which stalls the entire `default` queue — not just this',
+    'plugin. Pass `janitor: false` to opt out of scheduling in the meantime.'
+].join(' ');
+/**
+ * Startup probe for the jobs-stats table. Logs an error and resolves `false`
+ * when the global cannot be read; never throws and never creates tables.
+ */ export async function checkJobStatsTable(payload) {
+    try {
+        await payload.db.findGlobal({
+            slug: JOB_STATS_GLOBAL_SLUG
+        });
+        return true;
+    } catch (error) {
+        payload.logger.error({
+            err: error
+        }, MISSING_TABLE_MESSAGE);
+        return false;
+    }
+}
+const MISSING_META_MESSAGE = [
+    `${LOG_PREFIX} Could not read the \`meta\` field of the \`payload-jobs\` collection.`,
+    'Enabling job scheduling also sets `jobs.stats`, which adds that field — its',
+    '`payload_jobs.meta` column ships in the same migration as `payload_jobs_stats`.',
+    'Scheduled jobs are queued with `meta.scheduled = true`, so without the column',
+    'every scheduling attempt fails. Run `payload migrate:create` and `payload migrate`.'
+].join(' ');
+/**
+ * Probe for the `payload_jobs.meta` column. Selecting the field is enough — the
+ * adapter names the column in the query, so a missing one throws. No raw SQL.
+ */ export async function checkJobsMetaColumn(payload) {
+    try {
+        await payload.db.find({
+            collection: JOBS_COLLECTION_SLUG,
+            limit: 1,
+            pagination: false,
+            select: {
+                meta: true
+            }
+        });
+        return true;
+    } catch (error) {
+        payload.logger.error({
+            err: error
+        }, MISSING_META_MESSAGE);
+        return false;
+    }
+}
+const MISSING_STATE_MESSAGE = [
+    `${LOG_PREFIX} Could not read the \`${STATE_GLOBAL_SLUG}\` global.`,
+    'This plugin keeps its dedup cursor there, because Payload deletes a completed',
+    'job row by default and a cursor kept in the job output would not survive to the',
+    'next run. Without the `version_retention_state` table the age sweep still works,',
+    'but identical-snapshot dedup restarts its cycle at the first entity every run and',
+    'never reaches the rest of the corpus.',
+    'Run `payload migrate:create` and `payload migrate`.'
+].join(' ');
+/**
+ * Startup probe for the plugin's own state table. Logs an error and resolves
+ * `false` when the global cannot be read; never throws.
+ */ export async function checkStateGlobal(payload) {
+    try {
+        await payload.db.findGlobal({
+            slug: STATE_GLOBAL_SLUG
+        });
+        return true;
+    } catch (error) {
+        payload.logger.error({
+            err: error
+        }, MISSING_STATE_MESSAGE);
+        return false;
+    }
+}
+/** The two enums Payload derives from the registered task slugs on Postgres. */ export const TASK_SLUG_ENUMS = [
+    'enum_payload_jobs_task_slug',
+    'enum_payload_jobs_log_task_slug'
+];
+function isPoolLike(value) {
+    return typeof value === 'object' && value !== null && typeof value.query === 'function';
+}
+/**
+ * The adapter's raw connection pool, when it has one and is a Postgres.
+ *
+ * `pool` is a Postgres-adapter detail the shared `BaseDatabaseAdapter` type
+ * does not declare, so it is probed rather than asserted. Returns `undefined`
+ * for every other adapter, which is also the "nothing to check" answer:
+ * Mongo and SQLite store the task slug as plain text.
+ */ function postgresPool(payload) {
+    const db = payload.db;
+    if (typeof db !== 'object' || db === null) {
+        return undefined;
+    }
+    const name = 'name' in db ? db.name : undefined;
+    if (typeof name !== 'string' || !name.includes('postgres')) {
+        return undefined;
+    }
+    const pool = 'pool' in db ? db.pool : undefined;
+    return isPoolLike(pool) ? pool : undefined;
+}
+/** The adapter's `schemaName`, or Postgres' default. */ function schemaOf(payload) {
+    const db = payload.db;
+    if (typeof db === 'object' && db !== null && 'schemaName' in db) {
+        const { schemaName } = db;
+        if (typeof schemaName === 'string' && schemaName.length > 0) {
+            return schemaName;
+        }
+    }
+    return 'public';
+}
+// Scoped to the adapter's schema: a multi-tenant database can hold the same
+// enum name in several schemas, and grouping across them would let a sibling
+// tenant's migration vouch for this one.
+const ENUM_QUERY = `SELECT t.typname AS enum_name, bool_or(e.enumlabel = $1) AS has_label
+  FROM pg_type t
+  JOIN pg_namespace n ON n.oid = t.typnamespace
+  JOIN pg_enum e ON e.enumtypid = t.oid
+  WHERE t.typname = ANY($2) AND n.nspname = $3
+  GROUP BY t.typname`;
+function missingEnumMessage(missing) {
+    return [
+        `${LOG_PREFIX} The \`${JANITOR_TASK_SLUG}\` task slug is missing from`,
+        `${missing.map((name)=>`\`${name}\``).join(' and ')}.`,
+        'On Postgres, Payload generates `payload_jobs.task_slug` and',
+        '`payload_jobs_log.task_slug` as enums built from the registered tasks, so queuing',
+        'the janitor fails with `invalid input value for enum ... ` until the slug is added.',
+        'Run `payload migrate:create` and `payload migrate` — it emits, for each enum:',
+        missing.map((name)=>`ALTER TYPE "${name}" ADD VALUE '${JANITOR_TASK_SLUG}';`).join(' '),
+        'The plugin adoption and that migration must ship in the same release.',
+        'MongoDB and SQLite store the slug as plain text and need nothing.'
+    ].join(' ');
+}
+/**
+ * Verifies the janitor's slug is a member of both jobs task-slug enums.
+ *
+ * Postgres only, and only when the adapter exposes a pool — every other
+ * adapter stores the slug as text, and an adapter this cannot inspect is left
+ * alone rather than warned about. Logs an error and resolves `false` when the
+ * slug is missing; never throws.
+ */ export async function checkTaskSlugEnum(payload) {
+    const pool = postgresPool(payload);
+    if (!pool) {
+        return true;
+    }
+    try {
+        const { rows } = await pool.query(ENUM_QUERY, [
+            JANITOR_TASK_SLUG,
+            [
+                ...TASK_SLUG_ENUMS
+            ],
+            schemaOf(payload)
+        ]);
+        // An enum type that does not exist at all means the jobs tables are not
+        // there either — a different problem, already reported by the jobs-stats
+        // check. Only complain about an enum that exists without the slug.
+        const missing = rows.filter((row)=>row.has_label !== true).map((row)=>String(row.enum_name));
+        if (missing.length === 0) {
+            return true;
+        }
+        payload.logger.error(missingEnumMessage(missing));
+        return false;
+    } catch (error) {
+        payload.logger.warn({
+            err: error
+        }, `${LOG_PREFIX} Could not inspect the jobs task-slug enums; skipping that check. ` + 'If queuing the janitor fails with `invalid input value for enum`, the slug is ' + 'missing and a migration must add it.');
+        return true;
+    }
+}
+/**
+ * Warns when nothing in `jobs.autoRun` drains a queue the plugin schedules onto.
+ * `handleSchedules` silently skips every queue the running autoRun entry does
+ * not name (unless it sets `allQueues`), so a schedule on an undrained queue
+ * never fires and produces no error of its own.
+ *
+ * A warning, not an error: the consumer may instead run a dedicated worker with
+ * `payload jobs:run --handle-schedules`, which this cannot see. Returns false
+ * when a warning was emitted.
+ */ export function checkScheduleIsDrained(payload, scheduledQueues) {
+    const autoRun = payload.config.jobs?.autoRun;
+    if (typeof autoRun === 'function') {
+        // Resolved by Payload at cron-init time; not knowable here without calling
+        // the consumer's factory a second time.
+        return true;
+    }
+    const entries = autoRun ?? [];
+    const undrained = scheduledQueues.filter((queue)=>!entries.some((entry)=>!entry.disableScheduling && (entry.allQueues === true || (entry.queue ?? DEFAULT_AUTORUN_QUEUE) === queue)));
+    if (undrained.length === 0) {
+        return true;
+    }
+    payload.logger.warn(`${LOG_PREFIX} Nothing in \`jobs.autoRun\` drains ${undrained.map((queue)=>`\`${queue}\``).join(', ')}, which the version-retention janitor is scheduled onto. ` + '`handleSchedules` skips queues the running autoRun entry does not name (unless it sets ' + '`allQueues`), so the janitor will never be queued. Add an autoRun entry for that queue, ' + 'or run a dedicated worker with `payload jobs:run --handle-schedules` — this check cannot ' + 'see a separate worker process.');
+    return false;
+}
+/**
+ * The boot-time checks that only matter once a schedule exists. The state
+ * global is checked separately, because a manually queued janitor needs it
+ * just as much. Logs; never throws.
+ */ export async function runSchedulingStartupChecks(payload, scheduledQueues) {
+    const statsTableOk = await checkJobStatsTable(payload);
+    // The meta column ships in the same migration; probing it after a failed
+    // stats read would only repeat the same news.
+    if (statsTableOk) {
+        await checkJobsMetaColumn(payload);
+    }
+    await checkTaskSlugEnum(payload);
+    checkScheduleIsDrained(payload, scheduledQueues);
+}
+/**
+ * One-liner for `janitor: { schedule: [] }` — the task is registered but
+ * nothing will ever queue it, which is easy to reach by accident.
+ */ export function warnScheduleDisabled(payload) {
+    payload.logger.warn(`${LOG_PREFIX} \`janitor.schedule\` is empty: the \`${JANITOR_TASK_SLUG}\` task is ` + 'registered but never scheduled, so old versions are not deleted. Queue it yourself, or ' + 'drop `schedule` to use the default (daily 03:00 on the `default` queue).');
+}

package/dist/lib/state.d.ts

@@ -0,0 +1,27 @@
+import type { GlobalConfig, Payload } from 'payload';
+import type { VersionRetentionDedupCursor } from '../types.js';
+/**
+ * The plugin's own scrap of persistent state: one hidden global holding one
+ * JSON field.
+ *
+ * It exists because there is nowhere else to put the dedup cursor. Payload's
+ * `deleteJobOnComplete` defaults to `true` and the completed job row is hard
+ * deleted, so a cursor written into a job's output is unreadable by the time
+ * the next run starts — which made the whole rotation inert under the default
+ * configuration, however carefully the cursor was computed.
+ *
+ * Access is denied to everyone; the plugin reads and writes it with
+ * `overrideAccess`, and nothing else has any business in it.
+ */
+export declare function createStateGlobal(): GlobalConfig;
+/**
+ * The dedup cursor the last run left behind, or `undefined` to start the cycle
+ * from the beginning. Never throws: a consumer who has not run the migration
+ * yet gets a warning and a fresh cycle, not a failed job.
+ */
+export declare function readStoredCursor(payload: Payload): Promise<undefined | VersionRetentionDedupCursor>;
+/**
+ * Records where dedup stopped, so the next run resumes there. `null` means the
+ * cycle completed and the next run starts over.
+ */
+export declare function writeStoredCursor(payload: Payload, dedupCursor: null | VersionRetentionDedupCursor): Promise<void>;

package/dist/lib/state.js

@@ -0,0 +1,87 @@
+import { LOG_PREFIX, STATE_GLOBAL_SLUG } from '../defaults.js';
+import { toDedupCursor } from './dedup-cursor.js';
+/**
+ * The plugin's own scrap of persistent state: one hidden global holding one
+ * JSON field.
+ *
+ * It exists because there is nowhere else to put the dedup cursor. Payload's
+ * `deleteJobOnComplete` defaults to `true` and the completed job row is hard
+ * deleted, so a cursor written into a job's output is unreadable by the time
+ * the next run starts — which made the whole rotation inert under the default
+ * configuration, however carefully the cursor was computed.
+ *
+ * Access is denied to everyone; the plugin reads and writes it with
+ * `overrideAccess`, and nothing else has any business in it.
+ */ export function createStateGlobal() {
+    return {
+        slug: STATE_GLOBAL_SLUG,
+        access: {
+            read: ()=>false,
+            update: ()=>false
+        },
+        admin: {
+            hidden: true
+        },
+        fields: [
+            {
+                name: 'state',
+                type: 'json',
+                admin: {
+                    readOnly: true
+                },
+                label: 'Version retention state'
+            }
+        ],
+        graphQL: false,
+        label: 'Version Retention State',
+        // Nothing edits this in the admin, so a row in `payload_locked_documents`
+        // on every cursor write would be pure overhead.
+        lockDocuments: false,
+        // Versioning our own bookkeeping would be a joke in poor taste.
+        versions: false
+    };
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * The dedup cursor the last run left behind, or `undefined` to start the cycle
+ * from the beginning. Never throws: a consumer who has not run the migration
+ * yet gets a warning and a fresh cycle, not a failed job.
+ */ export async function readStoredCursor(payload) {
+    try {
+        const global = await payload.findGlobal({
+            depth: 0,
+            overrideAccess: true,
+            slug: STATE_GLOBAL_SLUG
+        });
+        const state = isRecord(global) ? global.state : undefined;
+        return isRecord(state) ? toDedupCursor(state.dedupCursor) : undefined;
+    } catch (error) {
+        payload.logger.warn({
+            err: error
+        }, `${LOG_PREFIX} Could not read the \`${STATE_GLOBAL_SLUG}\` global; dedup starts its cycle ` + 'from the first entity. Run the migration that creates `version_retention_state`.');
+        return undefined;
+    }
+}
+/**
+ * Records where dedup stopped, so the next run resumes there. `null` means the
+ * cycle completed and the next run starts over.
+ */ export async function writeStoredCursor(payload, dedupCursor) {
+    try {
+        await payload.updateGlobal({
+            data: {
+                state: {
+                    dedupCursor
+                }
+            },
+            depth: 0,
+            overrideAccess: true,
+            slug: STATE_GLOBAL_SLUG
+        });
+    } catch (error) {
+        payload.logger.warn({
+            err: error
+        }, `${LOG_PREFIX} Could not persist the dedup cursor to \`${STATE_GLOBAL_SLUG}\`. This run's ` + 'work still counts; the next one restarts the dedup cycle from the first entity.');
+    }
+}

package/dist/lib/sweep.d.ts

@@ -0,0 +1,72 @@
+import type { Payload, SelectType } from 'payload';
+import type { ResolvedEntity, VersionRetentionDedupCursor, VersionRetentionJanitorOutput } from '../types.js';
+/** Mutable per-run budget, shared across every entity in one invocation. */
+export interface SweepBudget {
+    /** Version bodies the sweep may still read for dedup. */
+    dedupBodiesRemaining: number;
+    deletionsRemaining: number;
+    /** Only documents that actually yield a deletion draw on this. */
+    documentsRemaining: number;
+}
+/** What one entity's sweep contributed to the run. */
+export interface EntitySweepResult {
+    /** Last document dedup actually read, when it stopped short of the end. */
+    dedupResumeAfter?: number | string;
+    /** True when a budget stopped dedup with documents in this entity unread. */
+    dedupStarved: boolean;
+    dedupedCount: number;
+    deletedCount: number;
+    /** Documents whose sweep threw. Logged, skipped, retried next run. */
+    failedDocuments: number;
+    hasMore: boolean;
+    scannedDocuments: number;
+    skippedRaced: number;
+}
+/**
+ * Columns the retention decision reads. Anything else on a version row is a
+ * full document body — the reason these tables are 54% of the database — and
+ * pulling it into the janitor would defeat the point.
+ *
+ * A drafts-less entity has no `_status`, so `version` is left out entirely
+ * rather than asking the adapter for a field that does not exist.
+ */
+export declare function narrowVersionSelect(hasDrafts: boolean): SelectType;
+/**
+ * Sweeps one versioned collection.
+ *
+ * Rather than walking every document, the pass walks the stale-version rows
+ * themselves and visits only the parents that still hold deletable history.
+ * Paging is by an ascending `parent` cursor, not an offset, so deleting rows
+ * mid-pass cannot make it skip a document.
+ */
+export declare function sweepCollection(payload: Payload, entity: ResolvedEntity, budget: SweepBudget, now?: Date, 
+/** Resume dedup after this parent id; see `EntitySweepResult.dedupCursor`. */
+dedupCursorAfter?: number | string): Promise<EntitySweepResult>;
+/** Sweeps one versioned global. A global is a single "document". */
+export declare function sweepGlobal(payload: Payload, entity: ResolvedEntity, budget: SweepBudget, now?: Date): Promise<EntitySweepResult>;
+export interface RunSweepArgs {
+    collections: ResolvedEntity[];
+    /** Where the previous run's dedup pass stopped, from its output. */
+    dedupCursor?: null | VersionRetentionDedupCursor;
+    globals: ResolvedEntity[];
+    maxDedupBodiesPerRun: number;
+    maxDeletionsPerRun: number;
+    maxDocumentsPerRun: number;
+    now?: Date;
+    payload: Payload;
+}
+/**
+ * One full janitor pass. Globals go first: there are a handful of them and
+ * they are cheap, and letting a large collection's backlog eat the budget
+ * would otherwise starve them run after run.
+ *
+ * Dedup rides a single cursor over this whole order rather than one per
+ * entity. That is what guarantees progress: a run resumes at the entity the
+ * last one stopped in, skips everything already covered in this cycle, and
+ * carries on into the entities behind it. With a per-entity cursor the
+ * entities *before* the stalled one were re-read from scratch every night and
+ * consumed the entire body budget, so the pass never advanced — measured on
+ * the production copy, where dedup collapsed pairs only in the four
+ * collections ahead of `categories` and never reached `pages` or `games`.
+ */
+export declare function runRetentionSweep({ collections, dedupCursor, globals, maxDedupBodiesPerRun, maxDeletionsPerRun, maxDocumentsPerRun, now, payload, }: RunSweepArgs): Promise<VersionRetentionJanitorOutput>;