@hanzo/cms-drizzle 3.85.2

package/dist/utilities/blocksToJsonMigrator.js

@@ -0,0 +1,587 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
+import path from 'path';
+import { APIError, buildVersionCollectionFields, buildVersionGlobalFields, dynamicImport } from '@hanzo/cms';
+import { findConfig } from '@hanzo/cms/node';
+import { fieldShouldBeLocalized } from '@hanzo/cms/shared';
+import { getTransaction } from './getTransaction.js';
+const DEFAULT_BATCH_SIZE = 100;
+const TEMP_FOLDER_NAME = '.cms-blocks-migration';
+const writeEntitiesToTempFile = (entities, tempFolderPath, batchIndex)=>{
+    const filePath = path.join(tempFolderPath, `entities-batch-${batchIndex}.json`);
+    writeFileSync(filePath, JSON.stringify(entities, null, 2), 'utf-8');
+};
+const acceptDrizzlePrompts = async (callPrompt, { silenceLogs = false } = {})=>{
+    // eslint-disable-next-line @typescript-eslint/unbound-method
+    const write = process.stdout.write;
+    if (silenceLogs) {
+        process.stdout.write = ()=>true;
+    }
+    const promise = callPrompt();
+    const interval = setInterval(()=>process.stdin.emit('keypress', '\n', {
+            name: 'return',
+            ctrl: false
+        }), 25);
+    const res = await promise;
+    if (silenceLogs) {
+        process.stdout.write = write;
+    }
+    clearInterval(interval);
+    return res;
+};
+const entityHasBlocksField = (entity)=>{
+    for (const field of entity.flattenedFields){
+        if (field.type === 'blocks') {
+            return true;
+        }
+        if ('flattenedFields' in field && entityHasBlocksField({
+            flattenedFields: field.flattenedFields
+        })) {
+            return true;
+        }
+    }
+    return false;
+};
+const collectBlocksToMigrate = ({ config, data, fields, parentAccessor, parentIsLocalized })=>{
+    const result = [];
+    for (const field of fields){
+        switch(field.type){
+            case 'array':
+                {
+                    const arrayData = data[field.name];
+                    if (!Array.isArray(arrayData)) {
+                        continue;
+                    }
+                    if (config.localization && fieldShouldBeLocalized({
+                        field,
+                        parentIsLocalized
+                    })) {
+                        result.push(...collectBlocksToMigrate({
+                            config,
+                            data: arrayData,
+                            fields: config.localization.localeCodes.map((code)=>({
+                                    ...field,
+                                    name: code,
+                                    localized: false
+                                })),
+                            parentAccessor: [
+                                ...parentAccessor,
+                                field.name
+                            ],
+                            parentIsLocalized: true
+                        }));
+                        continue;
+                    }
+                    for (const [index, row] of arrayData.entries()){
+                        result.push(...collectBlocksToMigrate({
+                            config,
+                            data: row,
+                            fields: field.flattenedFields,
+                            parentAccessor: [
+                                ...parentAccessor,
+                                field.name,
+                                index
+                            ],
+                            parentIsLocalized
+                        }));
+                    }
+                    break;
+                }
+            case 'blocks':
+                {
+                    result.push({
+                        data: data[field.name],
+                        fieldAccessor: [
+                            ...parentAccessor,
+                            field.name
+                        ]
+                    });
+                    break;
+                }
+            case 'group':
+            case 'tab':
+                {
+                    if (config.localization && fieldShouldBeLocalized({
+                        field,
+                        parentIsLocalized
+                    })) {
+                        result.push(...collectBlocksToMigrate({
+                            config,
+                            data: data[field.name],
+                            fields: config.localization.localeCodes.map((code)=>({
+                                    ...field,
+                                    name: code,
+                                    localized: false
+                                })),
+                            parentAccessor: [
+                                ...parentAccessor,
+                                field.name
+                            ],
+                            parentIsLocalized: true
+                        }));
+                        continue;
+                    }
+                    result.push(...collectBlocksToMigrate({
+                        config,
+                        data: data[field.name],
+                        fields: field.flattenedFields,
+                        parentAccessor: [
+                            ...parentAccessor,
+                            field.name
+                        ],
+                        parentIsLocalized
+                    }));
+                    break;
+                }
+            default:
+                {
+                    continue;
+                }
+        }
+    }
+    return result;
+};
+class BlocksToJsonMigratorImpl {
+    adapter;
+    sanitizeStatements;
+    executeMethod;
+    batchSize;
+    tempFolderPath;
+    constructor(adapter, sanitizeStatements, executeMethod, batchSize){
+        this.adapter = adapter;
+        this.sanitizeStatements = sanitizeStatements;
+        this.executeMethod = executeMethod;
+        this.batchSize = batchSize || DEFAULT_BATCH_SIZE;
+        this.tempFolderPath = path.resolve(this.adapter.migrationDir, TEMP_FOLDER_NAME);
+    }
+    ensureTempFolder() {
+        if (!existsSync(this.tempFolderPath)) {
+            mkdirSync(this.tempFolderPath, {
+                recursive: true
+            });
+        }
+    }
+    *fetchEntitiesFromJsonBatches() {
+        if (!existsSync(this.tempFolderPath)) {
+            return;
+        }
+        const files = readdirSync(this.tempFolderPath).filter((file)=>file.startsWith('entities-batch-') && file.endsWith('.json')).sort((a, b)=>{
+            const aNum = parseInt(a.match(/\d+/)?.[0] || '0', 10);
+            const bNum = parseInt(b.match(/\d+/)?.[0] || '0', 10);
+            return aNum - bNum;
+        });
+        for (const file of files){
+            const filePath = path.join(this.tempFolderPath, file);
+            const fileContent = readFileSync(filePath, 'utf-8');
+            const batchEntities = JSON.parse(fileContent);
+            yield batchEntities;
+        }
+    }
+    async migrateEntities(entities, req) {
+        this.adapter.blocksAsJSON = true;
+        this.resetSchema();
+        await this.adapter.init();
+        await this.adapter.connect();
+        await this.syncTransactionDrizzleInstance(req);
+        const totalEntities = entities.length;
+        let processed = 0;
+        for (const entity of entities){
+            switch(entity.type){
+                case 'collection':
+                    {
+                        await this.adapter.updateOne({
+                            collection: entity.slug,
+                            data: entity.originalData,
+                            joins: false,
+                            req,
+                            where: {
+                                id: {
+                                    equals: entity.id
+                                }
+                            }
+                        });
+                        break;
+                    }
+                case 'collectionVersion':
+                    {
+                        await this.adapter.updateVersion({
+                            id: entity.id,
+                            collection: entity.slug,
+                            req,
+                            versionData: entity.originalData
+                        });
+                        break;
+                    }
+                case 'global':
+                    {
+                        await this.adapter.updateGlobal({
+                            slug: entity.slug,
+                            data: entity.originalData,
+                            req
+                        });
+                        break;
+                    }
+                case 'globalVersion':
+                    {
+                        await this.adapter.updateGlobalVersion({
+                            id: entity.id,
+                            global: entity.slug,
+                            req,
+                            versionData: entity.originalData
+                        });
+                        break;
+                    }
+            }
+            processed++;
+            if (processed % this.batchSize === 0 || processed === totalEntities) {
+                this.adapter.cms.logger.info(`Migrated ${processed}/${totalEntities} entities (${Math.round(processed / totalEntities * 100)}%)`);
+            }
+        }
+    }
+    resetSchema() {
+        this.adapter.schema = {};
+        this.adapter.tables = {};
+        this.adapter.indexes = new Set();
+        this.adapter.foreignKeys = new Set();
+        this.adapter.relations = {};
+        this.adapter.rawTables = {};
+        this.adapter.rawRelations = {};
+        this.adapter.tableNameMap = new Map();
+        this.adapter.enums = {};
+    }
+    async syncTransactionDrizzleInstance(req) {
+        const tsx = await getTransaction(this.adapter, req);
+        tsx._ = this.adapter.drizzle._;
+        tsx.schema = this.adapter.drizzle._;
+        tsx.session.schema = this.adapter.drizzle.session.schema;
+    }
+    cleanupTempFolder() {
+        rmSync(this.tempFolderPath, {
+            force: true,
+            recursive: true
+        });
+        this.adapter.cms.logger.info(`Cleaned up temp folder at ${this.tempFolderPath}`);
+    }
+    async collectAndSaveEntitiesToBatches(req, options) {
+        const batchSize = options?.batchSize ?? this.batchSize;
+        this.cleanupTempFolder();
+        this.ensureTempFolder();
+        this.adapter.blocksAsJSON = false;
+        this.resetSchema();
+        await this.adapter.init();
+        await this.adapter.connect();
+        // Count total entities to migrate
+        let totalExpected = 0;
+        for (const collection of this.adapter.cms.config.collections.filter(entityHasBlocksField)){
+            const { totalDocs } = await this.adapter.count({
+                collection: collection.slug
+            });
+            totalExpected += totalDocs;
+            if (collection.versions) {
+                const { totalDocs: totalVersions } = await this.adapter.countVersions({
+                    collection: collection.slug
+                });
+                totalExpected += totalVersions;
+            }
+        }
+        for (const globalConfig of this.adapter.cms.config.globals.filter(entityHasBlocksField)){
+            totalExpected += 1; // Global itself
+            if (globalConfig.versions) {
+                const { totalDocs: totalGlobalVersions } = await this.adapter.countGlobalVersions({
+                    global: globalConfig.slug
+                });
+                totalExpected += totalGlobalVersions;
+            }
+        }
+        this.adapter.cms.logger.info(`Found ${totalExpected} total entities to collect and save to batches`);
+        let batchIndex = 0;
+        let currentBatch = [];
+        let totalCollected = 0;
+        const flushBatch = ()=>{
+            if (currentBatch.length > 0) {
+                writeEntitiesToTempFile(currentBatch, this.tempFolderPath, batchIndex);
+                const percentage = totalExpected > 0 ? Math.round(totalCollected / totalExpected * 100) : 0;
+                this.adapter.cms.logger.info(`Saved batch ${batchIndex} with ${currentBatch.length} entities (${totalCollected}/${totalExpected} - ${percentage}%)`);
+                batchIndex++;
+                currentBatch = [];
+            }
+        };
+        const addEntity = (entity)=>{
+            currentBatch.push(entity);
+            totalCollected++;
+            if (currentBatch.length >= batchSize) {
+                flushBatch();
+            }
+        };
+        for (const collection of this.adapter.cms.config.collections.filter(entityHasBlocksField)){
+            let page = 1;
+            let hasMore = true;
+            while(hasMore){
+                const { docs, hasNextPage } = await this.adapter.find({
+                    collection: collection.slug,
+                    limit: batchSize,
+                    page
+                });
+                for (const doc of docs){
+                    const entity = {
+                        id: doc.id,
+                        slug: collection.slug,
+                        type: 'collection',
+                        blocks: collectBlocksToMigrate({
+                            config: this.adapter.cms.config,
+                            data: doc,
+                            fields: collection.flattenedFields,
+                            parentAccessor: [],
+                            parentIsLocalized: false
+                        }),
+                        originalData: doc
+                    };
+                    addEntity(entity);
+                }
+                this.adapter.cms.logger.info(`Collected ${docs.length} entries from ${collection.slug} (page ${page})`);
+                hasMore = hasNextPage;
+                page++;
+            }
+            if (collection.versions) {
+                let versionPage = 1;
+                let hasMoreVersions = true;
+                while(hasMoreVersions){
+                    const { docs: versionDocs, hasNextPage: hasNextVersionPage } = await this.adapter.findVersions({
+                        collection: collection.slug,
+                        limit: batchSize,
+                        page: versionPage
+                    });
+                    for (const versionDoc of versionDocs){
+                        const entity = {
+                            id: versionDoc.id,
+                            slug: collection.slug,
+                            type: 'collectionVersion',
+                            blocks: collectBlocksToMigrate({
+                                config: this.adapter.cms.config,
+                                data: versionDoc,
+                                fields: buildVersionCollectionFields(this.adapter.cms.config, collection, true),
+                                parentAccessor: [],
+                                parentIsLocalized: false
+                            }),
+                            originalData: versionDoc
+                        };
+                        addEntity(entity);
+                    }
+                    this.adapter.cms.logger.info(`Collected ${versionDocs.length} versions from ${collection.slug} (page ${versionPage})`);
+                    hasMoreVersions = hasNextVersionPage;
+                    versionPage++;
+                }
+            }
+        }
+        for (const globalConfig of this.adapter.cms.config.globals.filter(entityHasBlocksField)){
+            const globalDoc = await this.adapter.findGlobal({
+                slug: globalConfig.slug
+            });
+            const entity = {
+                slug: globalConfig.slug,
+                type: 'global',
+                blocks: collectBlocksToMigrate({
+                    config: this.adapter.cms.config,
+                    data: globalDoc,
+                    fields: globalConfig.flattenedFields,
+                    parentAccessor: [],
+                    parentIsLocalized: false
+                }),
+                originalData: globalDoc
+            };
+            addEntity(entity);
+            this.adapter.cms.logger.info(`Collected global ${globalConfig.slug}`);
+            if (globalConfig.versions) {
+                let globalVersionPage = 1;
+                let hasMoreGlobalVersions = true;
+                while(hasMoreGlobalVersions){
+                    const { docs: globalVersionDocs, hasNextPage: hasNextGlobalVersionPage } = await this.adapter.findGlobalVersions({
+                        global: globalConfig.slug,
+                        limit: batchSize,
+                        page: globalVersionPage
+                    });
+                    for (const globalVersionDoc of globalVersionDocs){
+                        const entity = {
+                            id: globalVersionDoc.id,
+                            slug: globalConfig.slug,
+                            type: 'globalVersion',
+                            blocks: collectBlocksToMigrate({
+                                config: this.adapter.cms.config,
+                                data: globalVersionDoc,
+                                fields: buildVersionGlobalFields(this.adapter.cms.config, globalConfig, true),
+                                parentAccessor: [],
+                                parentIsLocalized: false
+                            }),
+                            originalData: globalVersionDoc
+                        };
+                        addEntity(entity);
+                    }
+                    this.adapter.cms.logger.info(`Collected ${globalVersionDocs.length} versions from global ${globalConfig.slug} (page ${globalVersionPage})`);
+                    hasMoreGlobalVersions = hasNextGlobalVersionPage;
+                    globalVersionPage++;
+                }
+            }
+        }
+        flushBatch();
+        this.adapter.cms.logger.info(`Collected total of ${totalCollected} entities across ${batchIndex} batches`);
+    }
+    async getMigrationStatements() {
+        const { generateDrizzleJson, generateMigration } = this.adapter.requireDrizzleKit();
+        const drizzleJsonBefore = await generateDrizzleJson(this.adapter.schema);
+        this.adapter.blocksAsJSON = true;
+        this.resetSchema();
+        await this.adapter.init();
+        const drizzleJsonAfter = await generateDrizzleJson(this.adapter.schema);
+        this.adapter.blocksAsJSON = false;
+        this.resetSchema();
+        await this.adapter.init();
+        const statements = await acceptDrizzlePrompts(()=>generateMigration(drizzleJsonBefore, drizzleJsonAfter), {
+            silenceLogs: true
+        });
+        const sqlExecute = `await db.${this.executeMethod}(` + 'sql`';
+        return {
+            statements: this.sanitizeStatements({
+                sqlExecute,
+                statements
+            }),
+            writeDrizzleSnapshot (filePath) {
+                writeFileSync(`${filePath}.json`, JSON.stringify(drizzleJsonAfter, null, 2));
+            }
+        };
+    }
+    async migrateEntitiesFromTempFolder(req, options) {
+        const clearBatches = options?.clearBatches ?? true;
+        let totalEntities = 0;
+        let hasEntities = false;
+        for (const entities of this.fetchEntitiesFromJsonBatches()){
+            hasEntities = true;
+            totalEntities += entities.length;
+            this.adapter.cms.logger.info(`Migrating batch with ${entities.length} entities (total: ${totalEntities})`);
+            await this.migrateEntities(entities, req);
+        }
+        if (!hasEntities) {
+            this.adapter.cms.logger.warn('No entities found in temp folder to migrate');
+            return;
+        }
+        this.adapter.cms.logger.info(`Completed migration of ${totalEntities} entities from temp folder`);
+        if (clearBatches) {
+            this.cleanupTempFolder();
+        } else {
+            this.adapter.cms.logger.info(`Temp folder preserved at ${this.tempFolderPath} (clearBatches: false)`);
+        }
+    }
+    setTempFolder(tempFolderPath) {
+        this.tempFolderPath = tempFolderPath;
+    }
+    async updateCMSConfigFile() {
+        let configPath;
+        try {
+            configPath = findConfig();
+        } catch  {
+            this.adapter.cms.logger.info('updateCMSConfigFile failed - could not find the cms config. Please set the blocksAsJSON DB adapter property manually to "true"');
+            return;
+        }
+        const configFile = readFileSync(configPath, 'utf-8');
+        const ts = await dynamicImport('typescript');
+        const source = ts.createSourceFile(configPath, configFile, ts.ScriptTarget.ESNext);
+        let hadChanges = false;
+        const result = ts.transform(source, [
+            (ctx)=>(sourceFile)=>{
+                    const factory = ctx.factory;
+                    const visit = (node)=>{
+                        if (ts.isPropertyAssignment(node) && ts.isIdentifier(node.name) && node.name.text === 'db' && ts.isCallExpression(node.initializer) && node.initializer.arguments.length > 0 && ts.isObjectLiteralExpression(node.initializer.arguments[0])) {
+                            const call = node.initializer;
+                            const obj = node.initializer.arguments[0];
+                            const filteredProps = obj.properties.filter((prop)=>{
+                                if (!ts.isPropertyAssignment(prop)) {
+                                    return true;
+                                }
+                                return !(ts.isIdentifier(prop.name) && prop.name.text === 'blocksAsJSON');
+                            });
+                            const newProperty = factory.createPropertyAssignment('blocksAsJSON', factory.createTrue());
+                            hadChanges = true;
+                            const newObject = factory.updateObjectLiteralExpression(obj, [
+                                ...filteredProps,
+                                newProperty
+                            ]);
+                            const newCall = factory.updateCallExpression(call, call.expression, call.typeArguments, [
+                                newObject
+                            ]);
+                            return factory.updatePropertyAssignment(node, node.name, newCall);
+                        }
+                        return ts.visitEachChild(node, visit, ctx);
+                    };
+                    return ts.visitNode(sourceFile, visit);
+                }
+        ]);
+        if (!hadChanges) {
+            this.adapter.cms.logger.info('No changes made to cms config. Set blocksAsJSON to true manually.');
+            return;
+        }
+        const printer = ts.createPrinter({
+            newLine: ts.NewLineKind.LineFeed
+        });
+        let output = printer.printFile(result.transformed[0]);
+        try {
+            // eslint-disable-next-line @typescript-eslint/consistent-type-imports
+            const prettier = await dynamicImport('prettier');
+            const configPath = await prettier.resolveConfigFile();
+            const config = configPath ? await prettier.resolveConfig(configPath) : {};
+            output = await prettier.format(output, {
+                ...config,
+                parser: 'typescript'
+            });
+        } catch (err) {
+            this.adapter.cms.logger.error({
+                err,
+                msg: 'Could not format cms config with Prettier'
+            });
+        }
+        writeFileSync(configPath, output, 'utf-8');
+        this.adapter.cms.logger.info(`Updated ${configPath} with blocksAsJSON: true`);
+    }
+}
+export const createBlocksToJsonMigrator = ({ adapter, executeMethod, sanitizeStatements })=>{
+    return new BlocksToJsonMigratorImpl(adapter, sanitizeStatements, executeMethod);
+};
+export const getBlocksToJsonMigrator = (cms)=>{
+    const migrator = cms.db.blocksToJsonMigrator;
+    if (!migrator) {
+        throw new APIError(`blocksToJsonMigrator is not defined for ${cms.db.packageName}`);
+    }
+    return migrator;
+};
+export const buildDynamicPredefinedBlocksToJsonMigration = ({ packageName })=>{
+    return async ({ filePath, cms })=>{
+        const migrator = getBlocksToJsonMigrator(cms);
+        const migrationStatements = await migrator.getMigrationStatements();
+        migrationStatements.writeDrizzleSnapshot(filePath);
+        await migrator.updateCMSConfigFile();
+        const upSQL = `
+const migrator = getBlocksToJsonMigrator(cms)
+migrator.setTempFolder(TEMP_FOLDER)
+await migrator.collectAndSaveEntitiesToBatches(req, { batchSize: BATCH_SIZE })
+
+${migrationStatements.statements}
+cms.logger.info("Executed blocks to JSON migration statements.")
+
+await migrator.migrateEntitiesFromTempFolder(req, { clearBatches: true })
+  `;
+        return {
+            imports: `
+import { getBlocksToJsonMigrator } from '${packageName}/migration-utils'
+import { fileURLToPath } from 'url'
+import path from 'path'
+
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+// Configure migration options (optional)
+const BATCH_SIZE = 100 // Number of entities to process per batch
+const TEMP_FOLDER = path.resolve(dirname, '${TEMP_FOLDER_NAME}') // Folder path to store migration batch
+`,
+            upSQL
+        };
+    };
+};
+
+//# sourceMappingURL=blocksToJsonMigrator.js.map