@pgpmjs/core 7.13.0 → 7.15.0

package/esm/apply/reuse.js

@@ -0,0 +1,221 @@
+import { mkdtempSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { hashString } from '@pgpmjs/ast';
+import { bundleFromModule, materializeBundle, splitBundle, transpileBundle, verifyBundle } from '@pgpmjs/bundle';
+import { partitionModule } from '@pgpmjs/slice';
+import { loadModule, makeSchemaTranspiler } from '@pgpmjs/transform';
+/** Whether an apply spec is a reuse (shared/per-tenant split) spec. */
+export function isReuseSpec(spec) {
+    return spec.reuse !== undefined;
+}
+/** pgpm change-path folder → object route kind. */
+const FOLDER_KIND = {
+    tables: 'table',
+    views: 'view',
+    functions: 'function',
+    procedures: 'procedure',
+    types: 'type'
+};
+// Mirrors materialize.ts: bare-schema-name literal helpers the AST pass can't
+// reach are remapped by a narrow string pre-pass.
+const SCHEMA_NAME_LITERAL_FUNCS = ['verify_schema', 'has_schema_privilege'];
+const schemaNameLiteralPass = (content, schemaMapping) => {
+    const pattern = new RegExp(`\\b(${SCHEMA_NAME_LITERAL_FUNCS.join('|')})\\s*\\(\\s*'([^']+)'`, 'gi');
+    return content.replace(pattern, (match, _fn, schema) => {
+        const mapped = schemaMapping.get(schema);
+        return mapped ? match.replace(`'${schema}'`, `'${mapped}'`) : match;
+    });
+};
+/** Parse a `schemas/<schema>/<folder>/<name>…` change path into its object identity. */
+function parseObjectChange(change) {
+    const parts = change.split('/');
+    if (parts[0] !== 'schemas' || parts.length < 2)
+        return null;
+    const schema = parts[1];
+    const folder = parts[2];
+    if (folder === undefined || folder === 'schema')
+        return { schema, folder: 'schema' };
+    return { schema, folder, name: parts[3] };
+}
+/**
+ * Deterministic name of the shared module a reuse proxy expands into. Derived
+ * from the source module, the shared-schema mapping, and the per-tenant seed
+ * set — the exact inputs that determine the shared partition — so two tenant
+ * proxies over the same source/shared/seed configuration resolve to *one*
+ * shared module (deployed once), while any difference isolates them.
+ */
+export function resolveSharedModuleName(spec) {
+    if (spec.reuse.sharedName)
+        return spec.reuse.sharedName;
+    const canonical = JSON.stringify({
+        source: spec.source.module,
+        sharedSchema: Object.entries(spec.reuse.sharedSchema).sort(([a], [b]) => a.localeCompare(b)),
+        seeds: spec.reuse.perTenant
+            .map(s => `${s.fromSchema}.${s.kind}.${s.name}`)
+            .sort((a, b) => a.localeCompare(b))
+    });
+    return `${spec.source.module}-shared-${hashString(canonical).slice(0, 8)}`;
+}
+/**
+ * Expand a reuse proxy into two deployable modules: a shared module (objects the
+ * classifier proves tenant-independent, deployed once) and a per-tenant module
+ * (objects reachable from the per-tenant seeds, materialized per instance) that
+ * `requires` the shared one.
+ *
+ * Pipeline over existing primitives:
+ * `bundleFromModule` → `partitionModule` (classifier) →
+ * `transpileBundle(makeSchemaTranspiler)` routing shared objects to the shared
+ * schema and per-tenant objects to the per-tenant schema in *one* pass (so
+ * cross-boundary references resolve correctly) → `splitBundle` with a per-tenant
+ * schema bootstrap → `verifyBundle` → `materializeBundle`.
+ */
+export async function materializeReuseModule(options) {
+    const { sourceDir, spec } = options;
+    const perTenantName = spec.name;
+    const sharedName = resolveSharedModuleName(spec);
+    const reuse = spec.reuse;
+    await loadModule();
+    const source = bundleFromModule(sourceDir);
+    if (spec.source.bundleDigest && spec.source.bundleDigest !== source.manifest.digest) {
+        throw new Error(`Apply spec for "${perTenantName}" pins source bundle digest ${spec.source.bundleDigest}, ` +
+            `but the installed source "${spec.source.module}" hashes to ${source.manifest.digest}. ` +
+            `Reinstall the pinned version or update the spec.`);
+    }
+    // 1. Classify: which source changes must be per-tenant (a seed or a dependent).
+    const partition = partitionModule({
+        moduleDir: sourceDir,
+        seedObjects: reuse.perTenant.map(s => ({ schema: s.fromSchema, name: s.name }))
+    });
+    const unknownSeeds = partition.warnings.filter(w => w.kind === 'unknown-seed');
+    if (unknownSeeds.length > 0) {
+        throw new Error(`Reuse spec for "${perTenantName}": seed object(s) not found in source "${spec.source.module}": ` +
+            unknownSeeds.map(w => w.change).join(', '));
+    }
+    // 2. Object routes: every per-tenant change's object goes to the per-tenant
+    //    target; everything else defaults (schemaMap) to the shared target. A
+    //    single router keeps cross-boundary references correct (a per-tenant
+    //    function landing in the tenant schema still references the shared helper
+    //    in the shared schema).
+    const routes = [];
+    for (const change of partition.perTenant) {
+        const obj = parseObjectChange(change);
+        if (!obj || obj.folder === 'schema' || !obj.name) {
+            throw new Error(`Reuse spec for "${perTenantName}": per-tenant change "${change}" is not an object change ` +
+                `and cannot be routed to a per-tenant schema; refine the seeds so it stays shared, or ` +
+                `it names an unsupported object kind`);
+        }
+        const kind = FOLDER_KIND[obj.folder];
+        if (!kind) {
+            throw new Error(`Reuse spec for "${perTenantName}": per-tenant change "${change}" has unsupported object ` +
+                `folder "${obj.folder}"`);
+        }
+        const toSchema = spec.schemas[obj.schema];
+        if (!toSchema) {
+            throw new Error(`Reuse spec for "${perTenantName}": per-tenant object in schema "${obj.schema}" has no ` +
+                `per-tenant target in "schemas"`);
+        }
+        routes.push({ fromSchema: obj.schema, kind, name: obj.name, toSchema });
+    }
+    const assumeSchemasExist = [
+        ...new Set([...Object.values(reuse.sharedSchema), ...Object.values(spec.schemas)])
+    ];
+    const provenance = {
+        appliedFrom: spec.source.module,
+        ...(spec.source.package ? { sourcePackage: spec.source.package } : {}),
+        ...(spec.source.version ? { sourceVersion: spec.source.version } : {})
+    };
+    // 3. Transpile once: schema default = shared target, per-object overrides =
+    //    per-tenant target.
+    const main = makeSchemaTranspiler({
+        schemaMap: reuse.sharedSchema,
+        routes,
+        extensions: spec.extensions,
+        roles: spec.roles,
+        transform: { prePasses: [schemaNameLiteralPass], assumeSchemasExist }
+    });
+    const transpiled = transpileBundle(source, {
+        renameChange: main.renameChange,
+        transformScript: main.transformScript,
+        provenance
+    });
+    if (main.result.errors.length > 0) {
+        const detail = main.result.errors.map(e => `${e.file}: ${e.error}`).join('; ');
+        throw new Error(`Reuse transpile of "${spec.source.module}" for "${perTenantName}" failed: ${detail}`);
+    }
+    // The partition is in *source* change names; splitBundle works on the
+    // transpiled bundle, so map the per-tenant set through the same rename.
+    const perTenantTranspiled = [...partition.perTenant].map(main.renameChange);
+    // 4. Per-tenant schema bootstrap: the source's `CREATE SCHEMA` lands in the
+    //    shared module (shared target), but each tenant also needs its own
+    //    schema. Duplicate it — transpiled to the per-tenant target — into the
+    //    per-tenant module, re-pointing per-tenant dependencies off the shared
+    //    schema onto this local one.
+    const bootstrap = buildSchemaBootstraps(source, spec, main.renameChange, assumeSchemasExist);
+    const { shared, perTenant } = splitBundle(transpiled, {
+        perTenantChanges: perTenantTranspiled,
+        sharedName,
+        perTenantName,
+        perTenantBootstrap: bootstrap
+    });
+    for (const [label, bundle] of [
+        [sharedName, shared],
+        [perTenantName, perTenant]
+    ]) {
+        const issues = verifyBundle(bundle);
+        if (issues.length > 0) {
+            throw new Error(`Reuse bundle "${label}" failed integrity verification: ${issues.map(i => i.kind).join(', ')}`);
+        }
+    }
+    const sharedOutDir = options.sharedOutDir ?? mkdtempSync(join(tmpdir(), `pgpm-apply-${sharedName}-`));
+    const perTenantOutDir = options.perTenantOutDir ?? mkdtempSync(join(tmpdir(), `pgpm-apply-${perTenantName}-`));
+    materializeBundle(shared, sharedOutDir);
+    materializeBundle(perTenant, perTenantOutDir);
+    return {
+        sharedName,
+        perTenantName,
+        shared: { bundle: shared, outDir: sharedOutDir },
+        perTenant: { bundle: perTenant, outDir: perTenantOutDir }
+    };
+}
+/**
+ * Build a per-tenant `CREATE SCHEMA` bootstrap for each source schema that both
+ * creates a schema and has per-tenant objects. The scripts are the source
+ * schema-creation change transpiled to the per-tenant target; `replacesShared`
+ * is the shared-schema change the per-tenant objects would otherwise depend on.
+ */
+function buildSchemaBootstraps(source, spec, renameSharedChange, assumeSchemasExist) {
+    const bootstraps = [];
+    for (const [fromSchema, perTenantTarget] of Object.entries(spec.schemas)) {
+        const sharedTarget = spec.reuse.sharedSchema[fromSchema];
+        if (!sharedTarget || sharedTarget === perTenantTarget)
+            continue;
+        const sourceSchemaChange = `schemas/${fromSchema}/schema`;
+        const schemaChange = source.changes.find(c => c.name === sourceSchemaChange);
+        if (!schemaChange)
+            continue; // source doesn't create this schema; nothing to duplicate
+        // Transpile just the schema-creation change to the per-tenant target.
+        const single = {
+            ...source,
+            manifest: { ...source.manifest, changeCount: 1, deployOrder: [sourceSchemaChange] },
+            changes: [schemaChange]
+        };
+        const t = makeSchemaTranspiler({
+            schemaMap: { [fromSchema]: perTenantTarget },
+            transform: { prePasses: [schemaNameLiteralPass], assumeSchemasExist }
+        });
+        const perTenantSchema = transpileBundle(single, {
+            renameChange: t.renameChange,
+            transformScript: t.transformScript
+        }).changes[0];
+        bootstraps.push({
+            name: `schemas/${perTenantTarget}/schema`,
+            dependencies: [],
+            deploy: perTenantSchema.deploy?.sql ?? null,
+            revert: perTenantSchema.revert?.sql ?? null,
+            verify: perTenantSchema.verify?.sql ?? null,
+            replacesShared: renameSharedChange(sourceSchemaChange)
+        });
+    }
+    return bootstraps;
+}

package/esm/apply/types.js

@@ -0,0 +1,14 @@
+/**
+ * The apply-spec ("proxy module") types.
+ *
+ * A proxy module is a directory containing a single `pgpm.apply.json` file:
+ * "apply <source module> transpiled with this schema map, under my name". It
+ * needs no `.control`, no `pgpm.plan`, and no deploy/revert/verify scripts —
+ * workspace discovery synthesizes a module entry from the spec, so the proxy
+ * participates in dependency resolution by name exactly like a regular
+ * module. At deploy/verify/revert time the engine transpiles the source
+ * module in memory and runs the normal migration path against the result;
+ * transpiled code is never committed.
+ */
+/** File name of the apply spec inside a proxy module directory. */
+export const APPLY_SPEC_FILE = 'pgpm.apply.json';

package/esm/bundle/artifact.js

@@ -0,0 +1,89 @@
+import { existsSync, readdirSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { getExtensionName, parseControlContent } from '@pgpmjs/ast/files';
+import { BUNDLE_ARCHIVE_EXTENSION, bundleFromModule, readBundleArchiveFile, withExecutableSql, writeBundleArchiveFile } from '@pgpmjs/bundle';
+import { cleanSql } from '../migrate/clean';
+/**
+ * Artifact file name for a module's stored bundle: it lives beside the packaged
+ * `sql/<name>--<version>.sql` as `sql/<name>--<version>.bundle.tar.gz`.
+ */
+export function bundleArtifactFileName(name, version) {
+    return `${name}--${version}${BUNDLE_ARCHIVE_EXTENSION}`;
+}
+/**
+ * Locate a module's stored bundle artifact, if it has one.
+ *
+ * Prefers the version declared in the module's `.control` file, then falls back
+ * to any single `sql/*.bundle.tar.gz` present — so a module packaged at a
+ * different version still gets the fast path instead of silently degrading.
+ * Returns `null` when the module ships no artifact.
+ */
+export function resolveBundleArtifactPath(moduleDir) {
+    const sqlDir = join(moduleDir, 'sql');
+    if (!existsSync(sqlDir))
+        return null;
+    let name;
+    try {
+        name = getExtensionName(moduleDir);
+    }
+    catch {
+        return null;
+    }
+    const controlPath = join(moduleDir, `${name}.control`);
+    if (existsSync(controlPath)) {
+        const { version } = parseControlContent(readFileSync(controlPath, 'utf-8'));
+        if (version) {
+            const versioned = join(sqlDir, bundleArtifactFileName(name, version));
+            if (existsSync(versioned))
+                return versioned;
+        }
+    }
+    const candidates = readdirSync(sqlDir).filter(file => file.startsWith(`${name}--`) && file.endsWith(BUNDLE_ARCHIVE_EXTENSION));
+    if (candidates.length !== 1)
+        return null;
+    return join(sqlDir, candidates[0]);
+}
+/**
+ * Build the executable form of a module's bundle: the content-addressed AST
+ * snapshot plus, per change, the deploy SQL with transaction / `CREATE
+ * EXTENSION` statements stripped (`cleanSql`).
+ *
+ * That pre-computation is the whole point of the artifact — the parse/deparse
+ * work moves from every deploy to the one-time package step, while the digests
+ * still cover the exact bytes that will be executed.
+ */
+export async function buildExecutableBundle(moduleDir, options) {
+    const bundle = bundleFromModule(moduleDir, options);
+    const execSql = {};
+    for (const change of bundle.changes) {
+        if (!change.deploy)
+            continue;
+        execSql[change.name] = await cleanSql(change.deploy.sql, false, '$EOFCODE$');
+    }
+    return withExecutableSql(bundle, execSql);
+}
+/**
+ * Emit a module's stored bundle artifact into `sql/`, returning its path.
+ */
+export async function writeBundleArtifact(moduleDir, version, options) {
+    const bundle = await buildExecutableBundle(moduleDir, options);
+    const outPath = join(moduleDir, 'sql', bundleArtifactFileName(bundle.manifest.name, version));
+    writeBundleArchiveFile(bundle, outPath);
+    return outPath;
+}
+/**
+ * Read a module's stored bundle artifact, or `null` when it has none or the
+ * archive cannot be read. Never throws — a broken artifact must degrade to the
+ * normal packaging path, not fail the deploy.
+ */
+export function readBundleArtifact(moduleDir) {
+    const artifactPath = resolveBundleArtifactPath(moduleDir);
+    if (!artifactPath)
+        return null;
+    try {
+        return readBundleArchiveFile(artifactPath);
+    }
+    catch {
+        return null;
+    }
+}

package/esm/bundle/deploy-bundled.js

@@ -0,0 +1,170 @@
+import { Logger } from '@pgpmjs/logger';
+import { verifyBundle } from '@pgpmjs/bundle';
+import { getPgPool } from 'pg-cache';
+import { PgpmMigrate } from '../migrate/client';
+import { buildExecutableBundle, readBundleArtifact } from './artifact';
+const log = new Logger('deploy-fast');
+/** Thrown when the ledger already holds a change with different content. */
+export class BundledDeployConflictError extends Error {
+    packageName;
+    changeName;
+    constructor(packageName, changeName) {
+        super(`Change ${changeName} already deployed in package ${packageName} with different content`);
+        this.packageName = packageName;
+        this.changeName = changeName;
+        this.name = 'BundledDeployConflictError';
+    }
+}
+/**
+ * The ledger hash for a change: the sha256 of the deploy script bytes.
+ *
+ * This is deliberately the same value `PgpmMigrate.calculateScriptHash` derives
+ * from `deploy/<change>.sql` under the default `content` hash method, so a
+ * change hashed at package time and the hash recorded/compared at deploy time
+ * agree — the one-shot and per-change paths are interchangeable and idempotent
+ * against each other.
+ */
+export function ledgerHashForChange(change) {
+    return change.deploy?.digest ?? null;
+}
+/**
+ * Load a module's executable bundle: its stored artifact when one verifies,
+ * otherwise built from `deploy/` on the fly.
+ *
+ * The artifact is purely an optimization — it removes the read+parse work, not
+ * a capability. Whatever the source, the resulting bundle is byte-identical in
+ * what it executes and in the hashes it records, so a module without an
+ * artifact deploys exactly the same way, just slower.
+ */
+async function loadExecutableBundle(moduleDir) {
+    const stored = readBundleArtifact(moduleDir);
+    if (stored) {
+        const issues = verifyBundle(stored);
+        if (issues.length === 0) {
+            return { bundle: stored, source: 'artifact' };
+        }
+        log.warn(`⚠️ Bundle artifact for ${stored.manifest.name} failed verification ` +
+            `(${issues.map(i => i.kind).join(', ')}); rebuilding from deploy/.`);
+    }
+    return { bundle: await buildExecutableBundle(moduleDir), source: 'packaged' };
+}
+/**
+ * Deploy a module in one shot **and** record the migration ledger — the `fast`
+ * deploy strategy.
+ *
+ * Pending changes' deploy SQL is concatenated and executed as a single
+ * statement (the speed of the old fast path), then one `pgpm_migrate.changes`
+ * row per change is bulk-inserted with the deploy script's own sha256 as
+ * `script_hash` — one round-trip, instead of an `isDeployed` + `readScript` +
+ * `CALL pgpm_migrate.deploy` per change. Because the ledger is written, the
+ * path is idempotent and resumable: re-deploys skip changes whose stored hash
+ * matches, and a same-name/different-content change raises
+ * {@link BundledDeployConflictError} exactly as the per-change path does.
+ *
+ * Returns a {@link BundledDeploySkipReason} (rather than throwing) when it
+ * cannot honour the requested semantics, so the caller falls back to the
+ * per-change migration path. Genuine SQL failures still throw.
+ */
+export async function deployModuleFast(moduleDir, options) {
+    if ((options.hashMethod ?? 'content') !== 'content') {
+        return 'unsupported-hash-method';
+    }
+    const { bundle, source } = await loadExecutableBundle(moduleDir);
+    const executable = bundle.changes.filter(change => change.deploy);
+    if (executable.length === 0 || executable.some(change => !change.exec)) {
+        return 'no-executable-sql';
+    }
+    const packageName = bundle.manifest.name;
+    const config = options.database
+        ? { ...options.config, database: options.database }
+        : options.config;
+    // Ensures the pgpm_migrate schema/procedures exist, exactly as the
+    // per-change deploy path does.
+    await new PgpmMigrate(config, { hashMethod: 'content' }).initialize();
+    const pool = getPgPool(config);
+    const existing = await pool.query('SELECT change_name, script_hash FROM pgpm_migrate.changes WHERE package = $1', [packageName]);
+    const deployedHashes = new Map(existing.rows.map(row => [row.change_name, row.script_hash]));
+    const pending = [];
+    const skipped = [];
+    for (const change of executable) {
+        const hash = ledgerHashForChange(change);
+        const recorded = deployedHashes.get(change.name);
+        if (recorded === undefined) {
+            pending.push(change);
+            continue;
+        }
+        if (recorded !== hash) {
+            throw new BundledDeployConflictError(packageName, change.name);
+        }
+        skipped.push(change.name);
+    }
+    if (pending.length === 0) {
+        log.info(`⚡ ${packageName}: all ${skipped.length} change(s) already deployed.`);
+        return { name: packageName, source, deployed: [], skipped };
+    }
+    const changeNames = pending.map(change => change.name);
+    const hashes = pending.map(change => ledgerHashForChange(change));
+    const depChanges = [];
+    const depRequires = [];
+    for (const change of pending) {
+        for (const dep of new Set(change.dependencies)) {
+            depChanges.push(change.name);
+            depRequires.push(dep.includes(':') ? dep : `${packageName}:${dep}`);
+        }
+    }
+    const deploySql = pending.map(change => change.exec.sql).join('\n');
+    const client = await pool.connect();
+    const useTransaction = options.useTransaction ?? true;
+    try {
+        if (useTransaction)
+            await client.query('BEGIN');
+        if (!options.logOnly) {
+            await client.query(deploySql);
+        }
+        await client.query('INSERT INTO pgpm_migrate.packages (package) VALUES ($1) ON CONFLICT (package) DO NOTHING', [packageName]);
+        await client.query(`WITH input AS (
+         SELECT change_name, script_hash
+         FROM unnest($2::text[], $3::text[]) AS t(change_name, script_hash)
+       ), inserted AS (
+         INSERT INTO pgpm_migrate.changes (change_id, change_name, package, script_hash)
+         SELECT encode(sha256(($1 || change_name || script_hash)::bytea), 'hex'),
+                change_name, $1, script_hash
+         FROM input
+         ON CONFLICT (package, change_name) DO NOTHING
+         RETURNING change_id, change_name
+       ), deps AS (
+         INSERT INTO pgpm_migrate.dependencies (change_id, requires)
+         SELECT inserted.change_id, d.requires
+         FROM unnest($4::text[], $5::text[]) AS d(change_name, requires)
+         JOIN inserted ON inserted.change_name = d.change_name
+         ON CONFLICT DO NOTHING
+         RETURNING change_id
+       )
+       INSERT INTO pgpm_migrate.events (event_type, change_name, package)
+       SELECT 'deploy', change_name, $1 FROM inserted`, [packageName, changeNames, hashes, depChanges, depRequires]);
+        if (useTransaction)
+            await client.query('COMMIT');
+    }
+    catch (error) {
+        if (useTransaction) {
+            try {
+                await client.query('ROLLBACK');
+            }
+            catch {
+                // the connection is already unusable; surface the original error
+            }
+        }
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+    log.success(`⚡ ${packageName}: ${changeNames.length} change(s) ${options.logOnly ? 'logged' : 'deployed'} ` +
+        `from ${source === 'artifact' ? 'bundle artifact' : 'packaged deploy/'} ` +
+        `(${skipped.length} already deployed).`);
+    return { name: packageName, source, deployed: changeNames, skipped };
+}
+/** True when the value is a real deploy result rather than a skip reason. */
+export function isBundledDeployResult(value) {
+    return typeof value !== 'string';
+}

package/esm/bundle/index.js

@@ -15,3 +15,7 @@ export * from '@pgpmjs/bundle';
 export * from './apply';
 // applyEnvelope: schema apply + data/fixtures part replay for a BundleEnvelope.
 export * from './apply-envelope';
+// The stored `sql/<name>--<version>.bundle.tar.gz` artifact: emit + resolve.
+export * from './artifact';
+// The bundle-backed deploy fast path (one-shot execute + ledger).
+export * from './deploy-bundled';