@pgpmjs/core 3.0.0

package/esm/core/template-scaffold.js

@@ -0,0 +1,161 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { CacheManager, GitCloner, Templatizer } from 'create-gen-app';
+import { readBoilerplateConfig, readBoilerplatesConfig, } from './boilerplate-scanner';
+export const DEFAULT_TEMPLATE_REPO = 'https://github.com/constructive-io/pgpm-boilerplates.git';
+export const DEFAULT_TEMPLATE_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week
+export const DEFAULT_TEMPLATE_TOOL_NAME = 'pgpm';
+const templatizer = new Templatizer();
+const looksLikePath = (value) => {
+    return (value.startsWith('.') || value.startsWith('/') || value.startsWith('~'));
+};
+const normalizeQuestions = (questions) => questions?.map((q) => ({
+    ...q,
+    type: q.type || 'text',
+}));
+const attachQuestionsToTemplatizer = (templ, questions) => {
+    if (!questions?.length || typeof templ?.extract !== 'function')
+        return;
+    const originalExtract = templ.extract.bind(templ);
+    templ.extract = async (templateDir) => {
+        const extracted = await originalExtract(templateDir);
+        extracted.projectQuestions = {
+            questions: normalizeQuestions(questions),
+        };
+        return extracted;
+    };
+};
+/**
+ * Resolve the template path using the new metadata-driven resolution.
+ *
+ * Resolution order:
+ * 1. If explicit `templatePath` is provided, use it directly
+ * 2. If `.boilerplates.json` exists, use its `dir` field to find the base directory
+ * 3. Look for `{baseDir}/{type}` (e.g., "default/module")
+ * 4. Fallback to legacy structure: `{type}` directly in root
+ */
+const resolveFromPath = (templateDir, templatePath, type, dirOverride) => {
+    // If explicit templatePath is provided, use it directly
+    if (templatePath) {
+        const candidateDir = path.isAbsolute(templatePath)
+            ? templatePath
+            : path.join(templateDir, templatePath);
+        if (fs.existsSync(candidateDir) &&
+            fs.statSync(candidateDir).isDirectory()) {
+            return {
+                fromPath: path.relative(templateDir, candidateDir) || '.',
+                resolvedTemplatePath: candidateDir,
+            };
+        }
+        return {
+            fromPath: templatePath,
+            resolvedTemplatePath: path.join(templateDir, templatePath),
+        };
+    }
+    // Try new metadata-driven resolution
+    const rootConfig = readBoilerplatesConfig(templateDir);
+    const baseDir = dirOverride ?? rootConfig?.dir;
+    if (baseDir) {
+        // New structure: {templateDir}/{baseDir}/{type}
+        const newStructurePath = path.join(templateDir, baseDir, type);
+        if (fs.existsSync(newStructurePath) &&
+            fs.statSync(newStructurePath).isDirectory()) {
+            return {
+                fromPath: path.join(baseDir, type),
+                resolvedTemplatePath: newStructurePath,
+            };
+        }
+    }
+    // Fallback to legacy structure: {templateDir}/{type}
+    const legacyPath = path.join(templateDir, type);
+    if (fs.existsSync(legacyPath) && fs.statSync(legacyPath).isDirectory()) {
+        return {
+            fromPath: type,
+            resolvedTemplatePath: legacyPath,
+        };
+    }
+    // Default fallback
+    return {
+        fromPath: type,
+        resolvedTemplatePath: path.join(templateDir, type),
+    };
+};
+export async function scaffoldTemplate(options) {
+    const { type, outputDir, templateRepo = DEFAULT_TEMPLATE_REPO, branch, templatePath, answers, noTty = false, cacheTtlMs = DEFAULT_TEMPLATE_TTL_MS, toolName = DEFAULT_TEMPLATE_TOOL_NAME, cwd, cacheBaseDir, dir, } = options;
+    const resolvedRepo = looksLikePath(templateRepo)
+        ? path.resolve(cwd ?? process.cwd(), templateRepo)
+        : templateRepo;
+    // Handle local template directories without caching
+    if (looksLikePath(templateRepo) &&
+        fs.existsSync(resolvedRepo) &&
+        fs.statSync(resolvedRepo).isDirectory()) {
+        const { fromPath, resolvedTemplatePath } = resolveFromPath(resolvedRepo, templatePath, type, dir);
+        // Read boilerplate config for questions
+        const boilerplateConfig = readBoilerplateConfig(resolvedTemplatePath);
+        // Inject questions into the templatizer pipeline so prompt types and defaults are applied
+        attachQuestionsToTemplatizer(templatizer, boilerplateConfig?.questions);
+        await templatizer.process(resolvedRepo, outputDir, {
+            argv: answers,
+            noTty,
+            fromPath,
+        });
+        return {
+            cacheUsed: false,
+            cacheExpired: false,
+            templateDir: resolvedRepo,
+            questions: boilerplateConfig?.questions,
+        };
+    }
+    // Remote repo with caching
+    const cacheManager = new CacheManager({
+        toolName,
+        ttl: cacheTtlMs,
+        baseDir: cacheBaseDir ??
+            process.env.PGPM_CACHE_BASE_DIR ??
+            (process.env.JEST_WORKER_ID
+                ? path.join(os.tmpdir(), `pgpm-cache-${process.env.JEST_WORKER_ID}`)
+                : undefined),
+    });
+    const gitCloner = new GitCloner();
+    const normalizedUrl = gitCloner.normalizeUrl(resolvedRepo);
+    const cacheKey = cacheManager.createKey(normalizedUrl, branch);
+    const expiredMetadata = cacheManager.checkExpiration(cacheKey);
+    if (expiredMetadata) {
+        cacheManager.clear(cacheKey);
+    }
+    let templateDir;
+    let cacheUsed = false;
+    const cachedPath = cacheManager.get(cacheKey);
+    if (cachedPath && !expiredMetadata) {
+        templateDir = cachedPath;
+        cacheUsed = true;
+    }
+    else {
+        const tempDest = path.join(cacheManager.getReposDir(), cacheKey);
+        gitCloner.clone(normalizedUrl, tempDest, {
+            branch,
+            depth: 1,
+            singleBranch: true,
+        });
+        cacheManager.set(cacheKey, tempDest);
+        templateDir = tempDest;
+    }
+    const { fromPath, resolvedTemplatePath } = resolveFromPath(templateDir, templatePath, type, dir);
+    // Read boilerplate config for questions
+    const boilerplateConfig = readBoilerplateConfig(resolvedTemplatePath);
+    // Inject questions into the templatizer pipeline so prompt types and defaults are applied
+    attachQuestionsToTemplatizer(templatizer, boilerplateConfig?.questions);
+    await templatizer.process(templateDir, outputDir, {
+        argv: answers,
+        noTty,
+        fromPath,
+    });
+    return {
+        cacheUsed,
+        cacheExpired: Boolean(expiredMetadata),
+        cachePath: templateDir,
+        templateDir,
+        questions: boilerplateConfig?.questions,
+    };
+}

package/esm/export/export-meta.js

@@ -0,0 +1,240 @@
+import { Parser } from 'csv-to-pg';
+import { getPgPool } from 'pg-cache';
+const config = {
+    database: {
+        schema: 'collections_public',
+        table: 'database',
+        fields: {
+            id: 'uuid',
+            owner_id: 'uuid',
+            name: 'text',
+            hash: 'uuid'
+        }
+    },
+    database_extension: {
+        schema: 'collections_public',
+        table: 'database_extensions',
+        fields: {
+            name: 'text',
+            database_id: 'uuid'
+        }
+    },
+    schema: {
+        schema: 'collections_public',
+        table: 'schema',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            name: 'text',
+            schema_name: 'text',
+            description: 'text'
+        }
+    },
+    table: {
+        schema: 'collections_public',
+        table: 'table',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            schema_id: 'uuid',
+            name: 'text',
+            description: 'text'
+        }
+    },
+    field: {
+        schema: 'collections_public',
+        table: 'field',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            table_id: 'uuid',
+            name: 'text',
+            type: 'text',
+            description: 'text'
+        }
+    },
+    domains: {
+        schema: 'meta_public',
+        table: 'domains',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            site_id: 'uuid',
+            api_id: 'uuid',
+            domain: 'text',
+            subdomain: 'text'
+        }
+    },
+    sites: {
+        schema: 'meta_public',
+        table: 'sites',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            title: 'text',
+            description: 'text',
+            og_image: 'image',
+            favicon: 'upload',
+            apple_touch_icon: 'image',
+            logo: 'image',
+            dbname: 'text'
+        }
+    },
+    apis: {
+        schema: 'meta_public',
+        table: 'apis',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            name: 'text',
+            dbname: 'text',
+            is_public: 'boolean',
+            role_name: 'text',
+            anon_role: 'text'
+        }
+    },
+    apps: {
+        schema: 'meta_public',
+        table: 'apps',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            site_id: 'uuid',
+            name: 'text',
+            app_image: 'image',
+            app_store_link: 'url',
+            app_store_id: 'text',
+            app_id_prefix: 'text',
+            play_store_link: 'url'
+        }
+    },
+    site_modules: {
+        schema: 'meta_public',
+        table: 'site_modules',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            site_id: 'uuid',
+            name: 'text',
+            data: 'jsonb'
+        }
+    },
+    site_themes: {
+        schema: 'meta_public',
+        table: 'site_themes',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            site_id: 'uuid',
+            theme: 'jsonb'
+        }
+    },
+    api_modules: {
+        schema: 'meta_public',
+        table: 'api_modules',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            api_id: 'uuid',
+            name: 'text',
+            data: 'jsonb'
+        }
+    },
+    api_extensions: {
+        schema: 'meta_public',
+        table: 'api_extensions',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            api_id: 'uuid',
+            schema_name: 'text'
+        }
+    },
+    api_schemata: {
+        schema: 'meta_public',
+        table: 'api_schemata',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            schema_id: 'uuid',
+            api_id: 'uuid'
+        }
+    },
+    rls_module: {
+        schema: 'meta_public',
+        table: 'rls_module',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            api_id: 'uuid',
+            schema_id: 'uuid',
+            private_schema_id: 'uuid',
+            tokens_table_id: 'uuid',
+            users_table_id: 'uuid',
+            authenticate: 'text',
+            authenticate_strict: 'text',
+            current_role: 'text',
+            current_role_id: 'text'
+        }
+    },
+    user_auth_module: {
+        schema: 'meta_public',
+        table: 'user_auth_module',
+        fields: {
+            id: 'uuid',
+            database_id: 'uuid',
+            schema_id: 'uuid',
+            emails_table_id: 'uuid',
+            users_table_id: 'uuid',
+            secrets_table_id: 'uuid',
+            encrypted_table_id: 'uuid',
+            tokens_table_id: 'uuid',
+            sign_in_function: 'text',
+            sign_up_function: 'text',
+            sign_out_function: 'text',
+            sign_in_one_time_token_function: 'text',
+            one_time_token_function: 'text',
+            extend_token_expires: 'text',
+            send_account_deletion_email_function: 'text',
+            delete_account_function: 'text',
+            set_password_function: 'text',
+            reset_password_function: 'text',
+            forgot_password_function: 'text',
+            send_verification_email_function: 'text',
+            verify_email_function: 'text'
+        }
+    }
+};
+export const exportMeta = async ({ opts, dbname, database_id }) => {
+    const pool = getPgPool({
+        ...opts.pg,
+        database: dbname
+    });
+    const sql = {};
+    const parsers = Object.entries(config).reduce((m, [name, config]) => {
+        m[name] = new Parser(config);
+        return m;
+    }, {});
+    const queryAndParse = async (key, query) => {
+        const result = await pool.query(query, [database_id]);
+        if (result.rows.length) {
+            sql[key] = await parsers[key].parse(result.rows);
+        }
+    };
+    await queryAndParse('database', `SELECT * FROM collections_public.database WHERE id = $1`);
+    await queryAndParse('schema', `SELECT * FROM collections_public.schema WHERE database_id = $1`);
+    await queryAndParse('table', `SELECT * FROM collections_public.table WHERE database_id = $1`);
+    await queryAndParse('domains', `SELECT * FROM meta_public.domains WHERE database_id = $1`);
+    await queryAndParse('apis', `SELECT * FROM meta_public.apis WHERE database_id = $1`);
+    await queryAndParse('sites', `SELECT * FROM meta_public.sites WHERE database_id = $1`);
+    await queryAndParse('api_modules', `SELECT * FROM meta_public.api_modules WHERE database_id = $1`);
+    await queryAndParse('site_modules', `SELECT * FROM meta_public.site_modules WHERE database_id = $1`);
+    await queryAndParse('site_themes', `SELECT * FROM meta_public.site_themes WHERE database_id = $1`);
+    await queryAndParse('apps', `SELECT * FROM meta_public.apps WHERE database_id = $1`);
+    await queryAndParse('database_extension', `SELECT * FROM collections_public.database_extension WHERE database_id = $1`);
+    await queryAndParse('api_extensions', `SELECT * FROM meta_public.api_extensions WHERE database_id = $1`);
+    await queryAndParse('api_schemata', `SELECT * FROM meta_public.api_schemata WHERE database_id = $1`);
+    await queryAndParse('rls_module', `SELECT * FROM meta_public.rls_module WHERE database_id = $1`);
+    await queryAndParse('user_auth_module', `SELECT * FROM meta_public.user_auth_module WHERE database_id = $1`);
+    return Object.entries(sql).reduce((m, [_, v]) => m + '\n\n' + v, '');
+};

package/esm/export/export-migrations.js

@@ -0,0 +1,180 @@
+import { mkdirSync, rmSync } from 'fs';
+import { sync as glob } from 'glob';
+import { toSnakeCase } from 'komoji';
+import path from 'path';
+import { getPgPool } from 'pg-cache';
+import { writeSqitchFiles, writeSqitchPlan } from '../files';
+import { exportMeta } from './export-meta';
+const exportMigrationsToDisk = async ({ project, options, database, databaseId, author, outdir, schema_names, extensionName, metaExtensionName }) => {
+    outdir = outdir + '/';
+    const pgPool = getPgPool({
+        ...options.pg,
+        database
+    });
+    const db = await pgPool.query(`select * from collections_public.database where id=$1`, [databaseId]);
+    const schemas = await pgPool.query(`select * from collections_public.schema where database_id=$1`, [databaseId]);
+    if (!db?.rows?.length) {
+        console.log('NO DATABASES.');
+        return;
+    }
+    if (!schemas?.rows?.length) {
+        console.log('NO SCHEMAS.');
+        return;
+    }
+    const name = extensionName || db.rows[0].name;
+    const { replace, replacer } = makeReplacer({
+        schemas: schemas.rows.filter((schema) => schema_names.includes(schema.schema_name)),
+        name
+    });
+    const results = await pgPool.query(`select * from db_migrate.sql_actions order by id`);
+    const opts = {
+        name,
+        replacer,
+        outdir,
+        author
+    };
+    if (results?.rows?.length > 0) {
+        await preparePackage({
+            project,
+            author,
+            outdir,
+            name,
+            extensions: [
+                'plpgsql',
+                'uuid-ossp',
+                'citext',
+                'pgcrypto',
+                'btree_gist',
+                'postgis',
+                'hstore',
+                'db-meta-schema',
+                'launchql-inflection',
+                'launchql-uuid',
+                'launchql-utils',
+                'launchql-database-jobs',
+                'launchql-jwt-claims',
+                'launchql-stamps',
+                'launchql-base32',
+                'launchql-totp',
+                'launchql-types',
+                'launchql-default-roles'
+            ]
+        });
+        writeSqitchPlan(results.rows, opts);
+        writeSqitchFiles(results.rows, opts);
+        let meta = await exportMeta({
+            opts: options,
+            dbname: database,
+            database_id: databaseId
+        });
+        meta = replacer(meta);
+        await preparePackage({
+            project,
+            author,
+            outdir,
+            extensions: ['plpgsql', 'db-meta-schema', 'db-meta-modules'],
+            name: metaExtensionName
+        });
+        const metaReplacer = makeReplacer({
+            schemas: schemas.rows.filter((schema) => schema_names.includes(schema.schema_name)),
+            name: metaExtensionName
+        });
+        const metaPackage = [
+            {
+                deps: [],
+                deploy: 'migrate/meta',
+                content: `SET session_replication_role TO replica;
+-- using replica in case we are deploying triggers to collections_public
+
+-- unaccent, postgis affected and require grants
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public to public;
+
+DO $LQLMIGRATION$
+  DECLARE
+  BEGIN
+  
+    EXECUTE format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), 'app_user');
+    EXECUTE format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), 'app_admin');
+
+  END;
+$LQLMIGRATION$;
+
+${meta}
+
+UPDATE meta_public.apis
+      SET dbname = current_database() WHERE TRUE;
+
+UPDATE meta_public.sites
+      SET dbname = current_database() WHERE TRUE;
+
+SET session_replication_role TO DEFAULT;
+`
+            }
+        ];
+        opts.replacer = metaReplacer.replacer;
+        opts.name = metaExtensionName;
+        writeSqitchPlan(metaPackage, opts);
+        writeSqitchFiles(metaPackage, opts);
+    }
+    pgPool.end();
+};
+export const exportMigrations = async ({ project, options, dbInfo, author, outdir, schema_names, extensionName, metaExtensionName }) => {
+    for (let v = 0; v < dbInfo.database_ids.length; v++) {
+        const databaseId = dbInfo.database_ids[v];
+        await exportMigrationsToDisk({
+            project,
+            options,
+            extensionName,
+            metaExtensionName,
+            database: dbInfo.dbname,
+            databaseId,
+            schema_names,
+            author,
+            outdir
+        });
+    }
+};
+/**
+ * Creates a Sqitch package directory or resets the deploy/revert/verify directories if it exists.
+ */
+const preparePackage = async ({ project, author, outdir, name, extensions }) => {
+    const curDir = process.cwd();
+    const sqitchDir = path.resolve(path.join(outdir, name));
+    mkdirSync(sqitchDir, { recursive: true });
+    process.chdir(sqitchDir);
+    const plan = glob(path.join(sqitchDir, 'pgpm.plan'));
+    if (!plan.length) {
+        await project.initModule({
+            name,
+            description: name,
+            author,
+            extensions,
+        });
+    }
+    else {
+        rmSync(path.resolve(sqitchDir, 'deploy'), { recursive: true, force: true });
+        rmSync(path.resolve(sqitchDir, 'revert'), { recursive: true, force: true });
+        rmSync(path.resolve(sqitchDir, 'verify'), { recursive: true, force: true });
+    }
+    process.chdir(curDir);
+};
+/**
+ * Generates a function for replacing schema names and extension names in strings.
+ */
+const makeReplacer = ({ schemas, name }) => {
+    const replacements = ['launchql-extension-name', name];
+    const schemaReplacers = schemas.map((schema) => [
+        schema.schema_name,
+        toSnakeCase(`${name}_${schema.name}`)
+    ]);
+    const replace = [...schemaReplacers, replacements].map(([from, to]) => [new RegExp(from, 'g'), to]);
+    const replacer = (str, n = 0) => {
+        if (!str)
+            return '';
+        if (replace[n] && replace[n].length === 2) {
+            return replacer(str.replace(replace[n][0], replace[n][1]), n + 1);
+        }
+        return str;
+    };
+    return { replacer, replace };
+};

package/esm/extensions/extensions.js

@@ -0,0 +1,31 @@
+/**
+ * Get the list of available extensions, including predefined core extensions.
+ */
+export const getAvailableExtensions = (modules) => {
+    const coreExtensions = [
+        'address_standardizer',
+        'address_standardizer_data_us',
+        'bloom',
+        'btree_gin',
+        'btree_gist',
+        'citext',
+        'hstore',
+        'intarray',
+        'pg_trgm',
+        'pgcrypto',
+        'plpgsql',
+        'plperl',
+        'plv8',
+        'postgis_tiger_geocoder',
+        'postgis_topology',
+        'postgis',
+        'postgres_fdw',
+        'unaccent',
+        'uuid-ossp',
+    ];
+    return Object.keys(modules).reduce((acc, module) => {
+        if (!acc.includes(module))
+            acc.push(module);
+        return acc;
+    }, [...coreExtensions]);
+};

package/esm/files/extension/index.js

@@ -0,0 +1,3 @@
+// Re-export all extension functionality
+export * from './reader';
+export * from './writer';

package/esm/files/extension/reader.js

@@ -0,0 +1,79 @@
+import { readFileSync } from 'fs';
+import { basename, dirname, relative } from 'path';
+import { parsePlanFileSimple } from '../plan';
+/**
+ * Parse a .control file and extract its metadata.
+ * https://www.postgresql.org/docs/current/extend-extensions.html
+ */
+export function parseControlFile(filePath, basePath) {
+    const contents = readFileSync(filePath, 'utf-8');
+    const key = basename(filePath).split('.control')[0];
+    const requires = contents
+        .split('\n')
+        .find((line) => /^requires/.test(line))
+        ?.split('=')[1]
+        .split(',')
+        .map((req) => req.replace(/[\'\s]*/g, '').trim()) || [];
+    const version = contents
+        .split('\n')
+        .find((line) => /^default_version/.test(line))
+        ?.split('=')[1]
+        .replace(/[\']*/g, '')
+        .trim() || '';
+    return {
+        path: dirname(relative(basePath, filePath)),
+        requires,
+        version,
+    };
+}
+/**
+ * Parse the pgpm.plan file to get the extension name.
+ */
+export const getExtensionName = (packageDir) => {
+    const planPath = `${packageDir}/pgpm.plan`;
+    const plan = parsePlanFileSimple(planPath);
+    if (!plan.package) {
+        throw new Error('No package name found in pgpm.plan!');
+    }
+    return plan.package;
+};
+/**
+ * Get detailed information about an extension in the specified directory.
+ */
+export const getExtensionInfo = (packageDir) => {
+    const pkgPath = `${packageDir}/package.json`;
+    const pkg = require(pkgPath);
+    const extname = getExtensionName(packageDir);
+    const version = pkg.version;
+    const Makefile = `${packageDir}/Makefile`;
+    const controlFile = `${packageDir}/${extname}.control`;
+    const sqlFile = `sql/${extname}--${version}.sql`;
+    return { extname, packageDir, version, Makefile, controlFile, sqlFile };
+};
+/**
+ * Get a list of extensions required by an extension from its control file.
+ * Returns an empty array if the file doesn't exist or has no requires line.
+ */
+export const getInstalledExtensions = (controlFilePath) => {
+    try {
+        const contents = readFileSync(controlFilePath, 'utf-8');
+        const match = contents.match(/^\s*requires\s*=\s*'([^']*)'/m);
+        if (!match) {
+            return [];
+        }
+        const requiresValue = match[1];
+        if (!requiresValue || requiresValue.trim() === '') {
+            return [];
+        }
+        return requiresValue
+            .split(',')
+            .map((ext) => ext.trim())
+            .filter((ext) => ext.length > 0);
+    }
+    catch (e) {
+        if (e.code === 'ENOENT') {
+            return [];
+        }
+        throw new Error(`Error reading control file at ${controlFilePath}: ${e.message}`);
+    }
+};