@milaboratories/pl-middle-layer 1.42.1 → 1.42.3

package/dist/mutator/project.js

@@ -0,0 +1,992 @@
+import { field, Pl, PlClient, isResourceId, isNullResourceId, isResource, isResourceRef, isNotNullResourceId, ensureResourceIdNotNull } from '@milaboratories/pl-client';
+import { createBContextFromUpstreams, createRenderHeavyBlock } from './template/render_block.js';
+import { InitialBlockMeta, ProjectResourceType, SchemaVersionKey, SchemaVersionCurrent, ProjectCreatedTimestamp, ProjectLastModifiedTimestamp, ProjectMetaKey, ProjectStructureKey, InitialBlockStructure, BlockRenderingStateKey, InitialProjectRenderingState, getServiceTemplateField, projectFieldName, FieldsToDuplicate, blockArgsAuthorKey, ProjectStructureAuthorKey, parseProjectField } from '../model/project_model.js';
+import { createBlockPack, BlockPackTemplateField } from './block-pack/block_pack.js';
+import { stagingGraph, productionGraph, allBlocks, graphDiff } from '../model/project_model_util.js';
+import { InitialBlockSettings } from '@milaboratories/pl-model-middle-layer';
+import Denque from 'denque';
+import { getPreparedExportTemplateEnvelope, exportContext } from './context_export.js';
+import { loadTemplate } from './template/template_loading.js';
+import { notEmpty, canonicalJsonBytes, canonicalJsonGzBytes, cachedDeserialize } from '@milaboratories/ts-helpers';
+import { extractConfig, UiError } from '@platforma-sdk/model';
+import { getDebugFlags } from '../debug/index.js';
+
+function cached(modIdCb, valueCb) {
+    let initialized = false;
+    let lastModId = undefined;
+    let value = undefined;
+    return () => {
+        if (!initialized) {
+            initialized = true;
+            lastModId = modIdCb();
+            value = valueCb();
+            return value;
+        }
+        const currentModId = modIdCb();
+        if (lastModId !== currentModId) {
+            lastModId = currentModId;
+            value = valueCb();
+        }
+        return valueCb();
+    };
+}
+class BlockInfo {
+    id;
+    fields;
+    config;
+    source;
+    constructor(id, fields, config, source) {
+        this.id = id;
+        this.fields = fields;
+        this.config = config;
+        this.source = source;
+    }
+    check() {
+        // state assertions
+        if ((this.fields.prodOutput === undefined) !== (this.fields.prodCtx === undefined))
+            throw new Error('inconsistent prod fields');
+        if ((this.fields.stagingOutput === undefined) !== (this.fields.stagingCtx === undefined))
+            throw new Error('inconsistent stage fields');
+        if ((this.fields.prodOutputPrevious === undefined)
+            !== (this.fields.prodCtxPrevious === undefined))
+            throw new Error('inconsistent prod cache fields');
+        if ((this.fields.stagingOutputPrevious === undefined)
+            !== (this.fields.stagingCtxPrevious === undefined))
+            throw new Error('inconsistent stage cache fields');
+        if (this.fields.blockPack === undefined)
+            throw new Error('no block pack field');
+        if (this.fields.currentArgs === undefined)
+            throw new Error('no current args field');
+    }
+    currentArgsC = cached(() => this.fields.currentArgs.modCount, () => cachedDeserialize(this.fields.currentArgs.value));
+    prodArgsC = cached(() => this.fields.prodArgs?.modCount, () => {
+        const bin = this.fields.prodArgs?.value;
+        if (bin === undefined)
+            return undefined;
+        return cachedDeserialize(bin);
+    });
+    get currentArgs() {
+        return this.currentArgsC();
+    }
+    get stagingRendered() {
+        return this.fields.stagingCtx !== undefined;
+    }
+    get productionRendered() {
+        return this.fields.prodCtx !== undefined;
+    }
+    get productionHasErrors() {
+        return this.fields.prodUiCtx?.status === 'Error';
+    }
+    productionStaleC = cached(() => `${this.fields.currentArgs.modCount}_${this.fields.prodArgs?.modCount}`, () => this.fields.prodArgs === undefined
+        || Buffer.compare(this.fields.currentArgs.value, this.fields.prodArgs.value) !== 0);
+    // get productionStale(): boolean {
+    //   return this.productionRendered && this.productionStaleC() && ;
+    // }
+    get requireProductionRendering() {
+        return !this.productionRendered || this.productionStaleC() || this.productionHasErrors;
+    }
+    get prodArgs() {
+        return this.prodArgsC();
+    }
+    getTemplate(tx) {
+        return tx.getFutureFieldValue(Pl.unwrapHolder(tx, this.fields.blockPack.ref), BlockPackTemplateField, 'Input');
+    }
+}
+const NoNewBlocks = (blockId) => {
+    throw new Error(`No new block info for ${blockId}`);
+};
+class ProjectMutator {
+    rid;
+    tx;
+    author;
+    schema;
+    lastModified;
+    meta;
+    struct;
+    renderingState;
+    blocksInLimbo;
+    blockInfos;
+    ctxExportTplHolder;
+    projectHelper;
+    globalModCount = 0;
+    fieldsChanged = false;
+    //
+    // Change trackers
+    //
+    lastModifiedChanged = false;
+    structureChanged = false;
+    metaChanged = false;
+    renderingStateChanged = false;
+    /** Set blocks will be assigned current mutator author marker on save */
+    blocksWithChangedInputs = new Set();
+    constructor(rid, tx, author, schema, lastModified, meta, struct, renderingState, blocksInLimbo, blockInfos, ctxExportTplHolder, projectHelper) {
+        this.rid = rid;
+        this.tx = tx;
+        this.author = author;
+        this.schema = schema;
+        this.lastModified = lastModified;
+        this.meta = meta;
+        this.struct = struct;
+        this.renderingState = renderingState;
+        this.blocksInLimbo = blocksInLimbo;
+        this.blockInfos = blockInfos;
+        this.ctxExportTplHolder = ctxExportTplHolder;
+        this.projectHelper = projectHelper;
+    }
+    fixProblemsAndMigrate() {
+        // Fixing problems introduced by old code
+        this.blockInfos.forEach((blockInfo) => {
+            if (blockInfo.fields.prodArgs === undefined
+                || blockInfo.fields.prodOutput === undefined
+                || blockInfo.fields.prodCtx === undefined)
+                this.deleteBlockFields(blockInfo.id, 'prodArgs', 'prodOutput', 'prodCtx');
+        });
+        // Migration for addition of block settings field
+        let initialBlockSettings;
+        this.blockInfos.forEach((blockInfo) => {
+            if (blockInfo.fields.blockSettings === undefined) {
+                if (initialBlockSettings === undefined)
+                    initialBlockSettings = this.createJsonFieldValue(InitialBlockSettings);
+                this.setBlockFieldObj(blockInfo.id, 'blockSettings', initialBlockSettings);
+            }
+        });
+    }
+    get wasModified() {
+        return (this.lastModifiedChanged
+            || this.structureChanged
+            || this.fieldsChanged
+            || this.metaChanged
+            || this.renderingStateChanged);
+    }
+    get structure() {
+        // clone
+        return JSON.parse(JSON.stringify(this.struct));
+    }
+    //
+    // Graph calculation
+    //
+    stagingGraph = undefined;
+    pendingProductionGraph = undefined;
+    actualProductionGraph = undefined;
+    getStagingGraph() {
+        if (this.stagingGraph === undefined)
+            this.stagingGraph = stagingGraph(this.struct);
+        return this.stagingGraph;
+    }
+    getProductionGraphBlockInfo(blockId, prod) {
+        const bInfo = this.getBlockInfo(blockId);
+        let argsField;
+        let args;
+        if (prod) {
+            if (bInfo.fields.prodArgs === undefined)
+                return undefined;
+            argsField = bInfo.fields.prodArgs;
+            args = bInfo.prodArgs;
+        }
+        else {
+            argsField = notEmpty(bInfo.fields.currentArgs);
+            args = bInfo.currentArgs;
+        }
+        const blockPackField = notEmpty(bInfo.fields.blockPack);
+        if (isResourceId(argsField.ref) && isResourceId(blockPackField.ref))
+            return {
+                args,
+                enrichmentTargets: this.projectHelper.getEnrichmentTargets(() => bInfo.config, () => args, { argsRid: argsField.ref, blockPackRid: blockPackField.ref }),
+            };
+        else
+            return {
+                args,
+                enrichmentTargets: this.projectHelper.getEnrichmentTargets(() => bInfo.config, () => args),
+            };
+    }
+    getPendingProductionGraph() {
+        if (this.pendingProductionGraph === undefined)
+            this.pendingProductionGraph = productionGraph(this.struct, (blockId) => this.getProductionGraphBlockInfo(blockId, false));
+        return this.pendingProductionGraph;
+    }
+    getActualProductionGraph() {
+        if (this.actualProductionGraph === undefined)
+            this.actualProductionGraph = productionGraph(this.struct, (blockId) => this.getProductionGraphBlockInfo(blockId, true));
+        return this.actualProductionGraph;
+    }
+    //
+    // Generic helpers to interact with project state
+    //
+    getBlockInfo(blockId) {
+        const info = this.blockInfos.get(blockId);
+        if (info === undefined)
+            throw new Error(`No such block: ${blockId}`);
+        return info;
+    }
+    createJsonFieldValueByContent(content) {
+        const value = Buffer.from(content);
+        const ref = this.tx.createValue(Pl.JsonObject, value);
+        return { ref, value, status: 'Ready' };
+    }
+    createJsonFieldValue(obj) {
+        return this.createJsonFieldValueByContent(JSON.stringify(obj));
+    }
+    getBlock(blockId) {
+        for (const block of allBlocks(this.struct))
+            if (block.id === blockId)
+                return block;
+        throw new Error('block not found');
+    }
+    setBlockFieldObj(blockId, fieldName, state) {
+        const fid = field(this.rid, projectFieldName(blockId, fieldName));
+        if (state.ref === undefined)
+            throw new Error('Can\'t set value with empty ref');
+        if (this.getBlockInfo(blockId).fields[fieldName] === undefined)
+            this.tx.createField(fid, 'Dynamic', state.ref);
+        else
+            this.tx.setField(fid, state.ref);
+        this.getBlockInfo(blockId).fields[fieldName] = {
+            modCount: this.globalModCount++,
+            ...state,
+        };
+        this.fieldsChanged = true;
+    }
+    setBlockField(blockId, fieldName, ref, status, value) {
+        this.setBlockFieldObj(blockId, fieldName, { ref, status, value });
+    }
+    deleteBlockFields(blockId, ...fieldNames) {
+        let deleted = false;
+        const info = this.getBlockInfo(blockId);
+        for (const fieldName of fieldNames) {
+            const fields = info.fields;
+            if (!(fieldName in fields))
+                continue;
+            this.tx.removeField(field(this.rid, projectFieldName(blockId, fieldName)));
+            delete fields[fieldName];
+            this.fieldsChanged = true;
+            deleted = true;
+        }
+        return deleted;
+    }
+    updateLastModified() {
+        this.lastModified = Date.now();
+        this.lastModifiedChanged = true;
+    }
+    //
+    // Main project actions
+    //
+    resetStagingRefreshTimestamp() {
+        this.renderingState.stagingRefreshTimestamp = Date.now();
+        this.renderingStateChanged = true;
+    }
+    resetStaging(blockId) {
+        const fields = this.getBlockInfo(blockId).fields;
+        if (fields.stagingOutput?.status === 'Ready'
+            && fields.stagingCtx?.status === 'Ready'
+            && fields.stagingUiCtx?.status === 'Ready') {
+            this.setBlockFieldObj(blockId, 'stagingOutputPrevious', fields.stagingOutput);
+            this.setBlockFieldObj(blockId, 'stagingCtxPrevious', fields.stagingCtx);
+            this.setBlockFieldObj(blockId, 'stagingUiCtxPrevious', fields.stagingUiCtx);
+        }
+        if (this.deleteBlockFields(blockId, 'stagingOutput', 'stagingCtx', 'stagingUiCtx'))
+            this.resetStagingRefreshTimestamp();
+    }
+    resetProduction(blockId) {
+        const fields = this.getBlockInfo(blockId).fields;
+        if (fields.prodOutput?.status === 'Ready'
+            && fields.prodCtx?.status === 'Ready'
+            && fields.prodUiCtx?.status === 'Ready') {
+            this.setBlockFieldObj(blockId, 'prodOutputPrevious', fields.prodOutput);
+            this.setBlockFieldObj(blockId, 'prodCtxPrevious', fields.prodCtx);
+            this.setBlockFieldObj(blockId, 'prodUiCtxPrevious', fields.prodUiCtx);
+        }
+        this.deleteBlockFields(blockId, 'prodOutput', 'prodCtx', 'prodUiCtx', 'prodArgs');
+    }
+    /** Running blocks are reset, already computed moved to limbo. Returns if
+     * either of the actions were actually performed.
+     * This method ensures the block is left in a consistent state that passes check() constraints. */
+    resetOrLimboProduction(blockId) {
+        const fields = this.getBlockInfo(blockId).fields;
+        // Check if we can safely move to limbo (both core production fields are ready)
+        if (fields.prodOutput?.status === 'Ready' && fields.prodCtx?.status === 'Ready') {
+            if (this.blocksInLimbo.has(blockId))
+                // we are already in limbo
+                return false;
+            // limbo - keep the ready production results but clean up cache
+            this.blocksInLimbo.add(blockId);
+            this.renderingStateChanged = true;
+            // doing some gc - clean up previous cache fields
+            this.deleteBlockFields(blockId, 'prodOutputPrevious', 'prodCtxPrevious', 'prodUiCtxPrevious');
+            return true;
+        }
+        else {
+            // reset - clean up any partial/inconsistent production stat
+            return this.deleteBlockFields(blockId, 'prodOutput', 'prodCtx', 'prodUiCtx', 'prodArgs', 'prodOutputPrevious', 'prodCtxPrevious', 'prodUiCtxPrevious');
+        }
+    }
+    /** Optimally sets inputs for multiple blocks in one go */
+    setStates(requests) {
+        const changedArgs = [];
+        let somethingChanged = false;
+        for (const req of requests) {
+            const info = this.getBlockInfo(req.blockId);
+            let blockChanged = false;
+            for (const stateKey of ['args', 'uiState']) {
+                if (!(stateKey in req))
+                    continue;
+                const statePart = req[stateKey];
+                if (statePart === undefined || statePart === null)
+                    throw new Error(`Can't set ${stateKey} to null or undefined, please omit the key if you don't want to change it`);
+                const fieldName = stateKey === 'args' ? 'currentArgs' : 'uiState';
+                let data;
+                let gzipped = false;
+                if (stateKey === 'args') {
+                    // don't gzip args, workflow code can't uncompress gzip yet
+                    data = canonicalJsonBytes(statePart);
+                }
+                else {
+                    const { data: binary, isGzipped } = canonicalJsonGzBytes(statePart);
+                    data = binary;
+                    gzipped = isGzipped;
+                }
+                if (Buffer.compare(info.fields[fieldName].value, data) === 0)
+                    continue;
+                console.log('setting', fieldName, gzipped, data.length);
+                const statePartRef = this.tx.createValue(gzipped ? Pl.JsonGzObject : Pl.JsonObject, data);
+                this.setBlockField(req.blockId, fieldName, statePartRef, 'Ready', data);
+                blockChanged = true;
+                if (stateKey === 'args')
+                    changedArgs.push(req.blockId);
+            }
+            if (blockChanged) {
+                // will be assigned our author marker
+                this.blocksWithChangedInputs.add(req.blockId);
+                somethingChanged = true;
+            }
+        }
+        // resetting staging outputs for all downstream blocks
+        this.getStagingGraph().traverse('downstream', changedArgs, ({ id }) => this.resetStaging(id));
+        if (somethingChanged)
+            this.updateLastModified();
+    }
+    setBlockSettings(blockId, newValue) {
+        this.setBlockFieldObj(blockId, 'blockSettings', this.createJsonFieldValue(newValue));
+        this.updateLastModified();
+    }
+    createProdCtx(upstream) {
+        const upstreamContexts = [];
+        upstream.forEach((id) => {
+            const info = this.getBlockInfo(id);
+            if (info.fields['prodCtx']?.ref === undefined)
+                throw new Error('One of the upstreams staging is not rendered.');
+            upstreamContexts.push(Pl.unwrapHolder(this.tx, info.fields['prodCtx'].ref));
+        });
+        return createBContextFromUpstreams(this.tx, upstreamContexts);
+    }
+    createStagingCtx(upstream) {
+        const upstreamContexts = [];
+        upstream.forEach((id) => {
+            const info = this.getBlockInfo(id);
+            if (info.fields['stagingCtx']?.ref === undefined)
+                throw new Error('One of the upstreams staging is not rendered.');
+            upstreamContexts.push(Pl.unwrapHolder(this.tx, info.fields['stagingCtx'].ref));
+            if (info.fields['prodCtx']?.ref !== undefined)
+                upstreamContexts.push(Pl.unwrapHolder(this.tx, info.fields['prodCtx'].ref));
+        });
+        return createBContextFromUpstreams(this.tx, upstreamContexts);
+    }
+    exportCtx(ctx) {
+        return exportContext(this.tx, Pl.unwrapHolder(this.tx, this.ctxExportTplHolder), ctx);
+    }
+    renderStagingFor(blockId) {
+        this.resetStaging(blockId);
+        const info = this.getBlockInfo(blockId);
+        const ctx = this.createStagingCtx(this.getStagingGraph().nodes.get(blockId).upstream);
+        if (this.getBlock(blockId).renderingMode !== 'Heavy')
+            throw new Error('not supported yet');
+        const tpl = info.getTemplate(this.tx);
+        const results = createRenderHeavyBlock(this.tx, tpl, {
+            args: info.fields.currentArgs.ref,
+            blockId: this.tx.createValue(Pl.JsonString, JSON.stringify(blockId)),
+            isProduction: this.tx.createValue(Pl.JsonBool, JSON.stringify(false)),
+            context: ctx,
+        });
+        this.setBlockField(blockId, 'stagingCtx', Pl.wrapInEphHolder(this.tx, results.context), 'NotReady');
+        this.setBlockField(blockId, 'stagingUiCtx', this.exportCtx(results.context), 'NotReady');
+        this.setBlockField(blockId, 'stagingOutput', results.result, 'NotReady');
+    }
+    renderProductionFor(blockId) {
+        this.resetProduction(blockId);
+        const info = this.getBlockInfo(blockId);
+        const ctx = this.createProdCtx(this.getPendingProductionGraph().nodes.get(blockId).upstream);
+        if (this.getBlock(blockId).renderingMode === 'Light')
+            throw new Error('Can\'t render production for light block.');
+        const tpl = info.getTemplate(this.tx);
+        const results = createRenderHeavyBlock(this.tx, tpl, {
+            args: info.fields.currentArgs.ref,
+            blockId: this.tx.createValue(Pl.JsonString, JSON.stringify(blockId)),
+            isProduction: this.tx.createValue(Pl.JsonBool, JSON.stringify(true)),
+            context: ctx,
+        });
+        this.setBlockField(blockId, 'prodCtx', Pl.wrapInEphHolder(this.tx, results.context), 'NotReady');
+        this.setBlockField(blockId, 'prodUiCtx', this.exportCtx(results.context), 'NotReady');
+        this.setBlockField(blockId, 'prodOutput', results.result, 'NotReady');
+        // saving inputs for which we rendered the production
+        this.setBlockFieldObj(blockId, 'prodArgs', info.fields.currentArgs);
+        // removing block from limbo as we juts rendered fresh production for it
+        if (this.blocksInLimbo.delete(blockId))
+            this.renderingStateChanged = true;
+    }
+    //
+    // Structure changes
+    //
+    initializeNewBlock(blockId, spec) {
+        const info = new BlockInfo(blockId, {}, extractConfig(spec.blockPack.config), spec.blockPack.source);
+        this.blockInfos.set(blockId, info);
+        // block pack
+        const bp = createBlockPack(this.tx, spec.blockPack);
+        this.setBlockField(blockId, 'blockPack', Pl.wrapInHolder(this.tx, bp), 'NotReady');
+        // settings
+        this.setBlockFieldObj(blockId, 'blockSettings', this.createJsonFieldValue(InitialBlockSettings));
+        // args
+        this.setBlockFieldObj(blockId, 'currentArgs', this.createJsonFieldValueByContent(spec.args));
+        // uiState
+        this.setBlockFieldObj(blockId, 'uiState', this.createJsonFieldValueByContent(spec.uiState ?? '{}'));
+        // checking structure
+        info.check();
+    }
+    getFieldNamesToDuplicate(blockId) {
+        const fields = this.getBlockInfo(blockId).fields;
+        const diff = (setA, setB) => new Set([...setA].filter((x) => !setB.has(x)));
+        // Check if we can safely move to limbo (both core production fields are ready)
+        if (fields.prodOutput?.status === 'Ready' && fields.prodCtx?.status === 'Ready') {
+            if (this.blocksInLimbo.has(blockId))
+                // we are already in limbo
+                return FieldsToDuplicate;
+            return diff(FieldsToDuplicate, new Set([
+                'prodOutputPrevious',
+                'prodCtxPrevious',
+                'prodUiCtxPrevious',
+            ]));
+        }
+        else {
+            return diff(FieldsToDuplicate, new Set([
+                'prodOutput',
+                'prodCtx',
+                'prodUiCtx',
+                'prodArgs',
+                'prodOutputPrevious',
+                'prodCtxPrevious',
+                'prodUiCtxPrevious',
+            ]));
+        }
+    }
+    initializeBlockDuplicate(blockId, originalBlockInfo) {
+        const info = new BlockInfo(blockId, {}, originalBlockInfo.config, originalBlockInfo.source);
+        this.blockInfos.set(blockId, info);
+        const fieldNamesToDuplicate = this.getFieldNamesToDuplicate(blockId);
+        // Copy all fields from original block to new block by sharing references
+        for (const [fieldName, fieldState] of Object.entries(originalBlockInfo.fields)) {
+            if (fieldNamesToDuplicate.has(fieldName) && fieldState && fieldState.ref) {
+                this.setBlockFieldObj(blockId, fieldName, {
+                    ref: fieldState.ref,
+                    status: fieldState.status,
+                    value: fieldState.value,
+                });
+            }
+        }
+        this.resetOrLimboProduction(blockId);
+        info.check();
+    }
+    /** Very generic method, better check for more specialized case-specific methods first. */
+    updateStructure(newStructure, newBlockInitializer = NoNewBlocks) {
+        const currentStagingGraph = this.getStagingGraph();
+        const currentActualProductionGraph = this.getActualProductionGraph();
+        const newStagingGraph = stagingGraph(newStructure);
+        const stagingDiff = graphDiff(currentStagingGraph, newStagingGraph);
+        // removing blocks
+        for (const blockId of stagingDiff.onlyInA) {
+            const { fields } = this.getBlockInfo(blockId);
+            this.deleteBlockFields(blockId, ...Object.keys(fields));
+            this.blockInfos.delete(blockId);
+            if (this.blocksInLimbo.delete(blockId))
+                this.renderingStateChanged = true;
+        }
+        // creating new blocks
+        for (const blockId of stagingDiff.onlyInB) {
+            newBlockInitializer(blockId);
+        }
+        // resetting stagings affected by topology change
+        for (const blockId of stagingDiff.different)
+            this.resetStaging(blockId);
+        // new actual production graph without new blocks
+        const newActualProductionGraph = productionGraph(newStructure, (blockId) => this.getProductionGraphBlockInfo(blockId, true));
+        const prodDiff = graphDiff(currentActualProductionGraph, newActualProductionGraph);
+        // applying changes due to topology change in production to affected nodes and
+        // all their downstreams
+        currentActualProductionGraph.traverse('downstream', [...prodDiff.different], (node) => {
+            this.resetOrLimboProduction(node.id);
+        });
+        if (stagingDiff.onlyInB.size > 0
+            || stagingDiff.onlyInA.size > 0
+            || stagingDiff.different.size > 0)
+            this.resetStagingRefreshTimestamp();
+        this.struct = newStructure;
+        this.structureChanged = true;
+        this.stagingGraph = undefined;
+        this.pendingProductionGraph = undefined;
+        this.actualProductionGraph = undefined;
+        this.updateLastModified();
+    }
+    //
+    // Structure change helpers
+    //
+    addBlock(block, spec, before) {
+        const newStruct = this.structure; // copy current structure
+        if (before === undefined) {
+            // adding as a very last block
+            newStruct.groups[newStruct.groups.length - 1].blocks.push(block);
+        }
+        else {
+            let done = false;
+            for (const group of newStruct.groups) {
+                const idx = group.blocks.findIndex((b) => b.id === before);
+                if (idx < 0)
+                    continue;
+                group.blocks.splice(idx, 0, block);
+                done = true;
+                break;
+            }
+            if (!done)
+                throw new Error(`Can't find element with id: ${before}`);
+        }
+        this.updateStructure(newStruct, (blockId) => {
+            if (blockId !== block.id)
+                throw new Error('Unexpected');
+            this.initializeNewBlock(blockId, spec);
+        });
+    }
+    /**
+     * Duplicates an existing block by copying all its fields and structure.
+     * This method creates a deep copy of the block at the mutator level.
+     *
+     * @param originalBlockId id of the block to duplicate
+     * @param newBlockId id for the new duplicated block
+     * @param after id of the block to insert new block after
+     */
+    duplicateBlock(originalBlockId, newBlockId, after) {
+        // Get the original block from structure
+        const originalBlock = this.getBlock(originalBlockId);
+        const originalBlockInfo = this.getBlockInfo(originalBlockId);
+        // Create new block in structure
+        const newBlock = {
+            id: newBlockId,
+            label: originalBlock.label,
+            renderingMode: originalBlock.renderingMode,
+        };
+        // Add the new block to structure
+        const newStruct = this.structure; // copy current structure
+        if (after === undefined) {
+            // adding as a very last block
+            newStruct.groups[newStruct.groups.length - 1].blocks.push(newBlock);
+        }
+        else {
+            let done = false;
+            for (const group of newStruct.groups) {
+                const idx = group.blocks.findIndex((b) => b.id === after);
+                if (idx < 0)
+                    continue;
+                group.blocks.splice(idx + 1, 0, newBlock);
+                done = true;
+                break;
+            }
+            if (!done)
+                throw new Error(`Can't find element with id: ${after}`);
+        }
+        this.updateStructure(newStruct, (blockId) => {
+            if (blockId !== newBlockId)
+                throw new Error('Unexpected');
+            this.initializeBlockDuplicate(blockId, originalBlockInfo);
+        });
+    }
+    deleteBlock(blockId) {
+        const newStruct = this.structure; // copy current structure
+        let done = false;
+        for (const group of newStruct.groups) {
+            const idx = group.blocks.findIndex((b) => b.id === blockId);
+            if (idx < 0)
+                continue;
+            group.blocks.splice(idx, 1);
+            done = true;
+            break;
+        }
+        if (!done)
+            throw new Error(`Can't find element with id: ${blockId}`);
+        this.updateStructure(newStruct);
+    }
+    //
+    // Block-pack migration
+    //
+    migrateBlockPack(blockId, spec, newArgsAndUiState) {
+        const info = this.getBlockInfo(blockId);
+        this.setBlockField(blockId, 'blockPack', Pl.wrapInHolder(this.tx, createBlockPack(this.tx, spec)), 'NotReady');
+        if (newArgsAndUiState !== undefined) {
+            // this will also reset all downstream stagings
+            this.setStates([{ blockId, args: newArgsAndUiState.args, uiState: newArgsAndUiState.uiState }]);
+        }
+        else {
+            // resetting staging outputs for all downstream blocks
+            this.getStagingGraph().traverse('downstream', [blockId], ({ id }) => this.resetStaging(id));
+        }
+        // also reset or limbo all downstream productions
+        if (info.productionRendered)
+            this.getActualProductionGraph().traverse('downstream', [blockId], ({ id }) => this.resetOrLimboProduction(id));
+        this.updateLastModified();
+    }
+    //
+    // Render
+    //
+    renderProduction(blockIds, addUpstreams = false) {
+        const blockIdsSet = new Set(blockIds);
+        const prodGraph = this.getPendingProductionGraph();
+        if (addUpstreams)
+            // adding all upstreams automatically
+            prodGraph.traverse('upstream', blockIds, (node) => {
+                blockIdsSet.add(node.id);
+            });
+        else // checking that targets contain all upstreams
+            for (const blockId of blockIdsSet) {
+                const node = prodGraph.nodes.get(blockId);
+                if (node === undefined)
+                    throw new Error(`Can't find block with id: ${blockId}`);
+                for (const upstream of node.upstream)
+                    if (!blockIdsSet.has(upstream))
+                        throw new Error('Can\'t render blocks not including all upstreams.');
+            }
+        // traversing in topological order and rendering target blocks
+        const rendered = new Set();
+        for (const block of allBlocks(this.structure)) {
+            if (!blockIdsSet.has(block.id))
+                continue;
+            let render = this.getBlockInfo(block.id).requireProductionRendering || this.blocksInLimbo.has(block.id);
+            if (!render)
+                for (const upstream of prodGraph.nodes.get(block.id).upstream)
+                    if (rendered.has(upstream)) {
+                        render = true;
+                        break;
+                    }
+            if (render) {
+                this.renderProductionFor(block.id);
+                rendered.add(block.id);
+            }
+        }
+        const renderedArray = [...rendered];
+        // sending to limbo all downstream blocks
+        prodGraph.traverse('downstream', renderedArray, (node) => {
+            if (rendered.has(node.id))
+                // don't send to limbo blocks that were just rendered
+                return;
+            this.resetOrLimboProduction(node.id);
+        });
+        // resetting staging outputs for all downstream blocks
+        this.getStagingGraph().traverse('downstream', renderedArray, ({ id }) => {
+            // don't reset staging of the first rendered block
+            if (renderedArray[0] !== id)
+                this.resetStaging(id);
+        });
+        if (rendered.size > 0)
+            this.updateLastModified();
+        return rendered;
+    }
+    /** Stops running blocks from the list and modify states of other blocks
+     * accordingly */
+    stopProduction(...blockIds) {
+        const activeProdGraph = this.getActualProductionGraph();
+        // we will stop all blocks listed in request and all their downstreams
+        const queue = new Denque(blockIds);
+        const queued = new Set(blockIds);
+        const stopped = [];
+        while (!queue.isEmpty()) {
+            const blockId = queue.shift();
+            const fields = this.getBlockInfo(blockId).fields;
+            if (fields.prodOutput?.status === 'Ready' && fields.prodCtx?.status === 'Ready')
+                // skipping finished blocks
+                continue;
+            if (this.deleteBlockFields(blockId, 'prodOutput', 'prodCtx', 'prodUiCtx', 'prodArgs')) {
+                // was actually stopped
+                stopped.push(blockId);
+                // will try to stop all its downstreams
+                for (const downstream of activeProdGraph.traverseIdsExcludingRoots('downstream', blockId)) {
+                    if (queued.has(downstream))
+                        continue;
+                    queue.push(downstream);
+                    queued.add(downstream);
+                }
+            }
+        }
+        // blocks under stopped blocks, but still having results, goes to limbo
+        for (const blockId of activeProdGraph.traverseIdsExcludingRoots('downstream', ...stopped))
+            this.resetOrLimboProduction(blockId);
+    }
+    traverseWithStagingLag(cb) {
+        const lags = new Map();
+        const stagingGraph = this.getStagingGraph();
+        stagingGraph.nodes.forEach((node) => {
+            const info = this.getBlockInfo(node.id);
+            let lag = info.stagingRendered ? 0 : 1;
+            node.upstream.forEach((upstream) => {
+                const upstreamLag = lags.get(upstream);
+                if (upstreamLag === 0)
+                    return;
+                lag = Math.max(upstreamLag + 1, lag);
+            });
+            cb(node.id, lag);
+            lags.set(node.id, lag);
+        });
+    }
+    /** @param stagingRenderingRate rate in blocks per second */
+    refreshStagings(stagingRenderingRate) {
+        const elapsed = Date.now() - this.renderingState.stagingRefreshTimestamp;
+        const lagThreshold = stagingRenderingRate === undefined
+            ? undefined
+            : 1 + Math.max(0, (elapsed * stagingRenderingRate) / 1000);
+        let rendered = 0;
+        this.traverseWithStagingLag((blockId, lag) => {
+            if (lag === 0)
+                // meaning staging already rendered
+                return;
+            if (lagThreshold === undefined || lag <= lagThreshold) {
+                this.renderStagingFor(blockId);
+                rendered++;
+            }
+        });
+        if (rendered > 0)
+            this.resetStagingRefreshTimestamp();
+    }
+    //
+    // Meta
+    //
+    /** Updates project metadata */
+    setMeta(meta) {
+        this.meta = meta;
+        this.metaChanged = true;
+        this.updateLastModified();
+    }
+    //
+    // Maintenance
+    //
+    /** @param stagingRenderingRate rate in blocks per second */
+    doRefresh(stagingRenderingRate) {
+        this.refreshStagings(stagingRenderingRate);
+        this.blockInfos.forEach((blockInfo) => {
+            if (blockInfo.fields.prodCtx?.status === 'Ready'
+                && blockInfo.fields.prodOutput?.status === 'Ready')
+                this.deleteBlockFields(blockInfo.id, 'prodOutputPrevious', 'prodCtxPrevious', 'prodUiCtxPrevious');
+            if (blockInfo.fields.stagingCtx?.status === 'Ready'
+                && blockInfo.fields.stagingOutput?.status === 'Ready')
+                this.deleteBlockFields(blockInfo.id, 'stagingOutputPrevious', 'stagingCtxPrevious', 'stagingUiCtxPrevious');
+        });
+    }
+    assignAuthorMarkers() {
+        const markerStr = this.author ? JSON.stringify(this.author) : undefined;
+        for (const blockId of this.blocksWithChangedInputs)
+            if (markerStr === undefined)
+                this.tx.deleteKValue(this.rid, blockArgsAuthorKey(blockId));
+            else
+                this.tx.setKValue(this.rid, blockArgsAuthorKey(blockId), markerStr);
+        if (this.metaChanged || this.structureChanged) {
+            if (markerStr === undefined)
+                this.tx.deleteKValue(this.rid, ProjectStructureAuthorKey);
+            else
+                this.tx.setKValue(this.rid, ProjectStructureAuthorKey, markerStr);
+        }
+    }
+    save() {
+        if (!this.wasModified)
+            return;
+        if (this.lastModifiedChanged)
+            this.tx.setKValue(this.rid, ProjectLastModifiedTimestamp, JSON.stringify(this.lastModified));
+        if (this.structureChanged)
+            this.tx.setKValue(this.rid, ProjectStructureKey, JSON.stringify(this.struct));
+        if (this.renderingStateChanged)
+            this.tx.setKValue(this.rid, BlockRenderingStateKey, JSON.stringify({
+                ...this.renderingState,
+                blocksInLimbo: [...this.blocksInLimbo],
+            }));
+        if (this.metaChanged)
+            this.tx.setKValue(this.rid, ProjectMetaKey, JSON.stringify(this.meta));
+        this.assignAuthorMarkers();
+    }
+    static async load(projectHelper, tx, rid, author) {
+        //
+        // Sending initial requests to read project state (start of round-trip #1)
+        //
+        const fullResourceStateP = tx.getResourceData(rid, true);
+        const schemaP = tx.getKValueJson(rid, SchemaVersionKey);
+        const lastModifiedP = tx.getKValueJson(rid, ProjectLastModifiedTimestamp);
+        const metaP = tx.getKValueJson(rid, ProjectMetaKey);
+        const structureP = tx.getKValueJson(rid, ProjectStructureKey);
+        const renderingStateP = tx.getKValueJson(rid, BlockRenderingStateKey);
+        const fullResourceState = await fullResourceStateP;
+        // loading field information
+        const blockInfoStates = new Map();
+        for (const f of fullResourceState.fields) {
+            const projectField = parseProjectField(f.name);
+            // processing only fields with known structure
+            if (projectField === undefined)
+                continue;
+            let info = blockInfoStates.get(projectField.blockId);
+            if (info === undefined) {
+                info = {
+                    id: projectField.blockId,
+                    fields: {},
+                };
+                blockInfoStates.set(projectField.blockId, info);
+            }
+            info.fields[projectField.fieldName] = isNullResourceId(f.value)
+                ? { modCount: 0 }
+                : { modCount: 0, ref: f.value };
+        }
+        //
+        // Roundtrip #1 not yet finished, but as soon as field list is received,
+        // we can start sending requests to read states of referenced resources
+        // (start of round-trip #2)
+        //
+        const blockFieldRequests = [];
+        blockInfoStates.forEach((info) => {
+            const fields = info.fields;
+            for (const [fName, state] of Object.entries(fields)) {
+                if (state.ref === undefined)
+                    continue;
+                if (!isResource(state.ref) || isResourceRef(state.ref))
+                    throw new Error('unexpected behaviour');
+                const fieldName = fName;
+                blockFieldRequests.push([
+                    info, fieldName,
+                    state, tx.getResourceData(state.ref, fieldName == 'blockPack')
+                ]);
+            }
+        });
+        // loading jsons
+        const [schema, lastModified, meta, structure, { stagingRefreshTimestamp, blocksInLimbo },] = await Promise.all([
+            schemaP,
+            lastModifiedP,
+            metaP,
+            structureP,
+            renderingStateP,
+        ]);
+        // Checking schema version of the project
+        if (schema !== SchemaVersionCurrent) {
+            if (Number(schema) < Number(SchemaVersionCurrent))
+                throw new UiError(`Can't perform this action on this project because it has older schema. Try (re)loading the project to update it.`);
+            else
+                throw new UiError(`Can't perform this action on this project because it has newer schema. Upgrade your desktop app to the latest version.`);
+        }
+        //
+        // <- at this point we have all the responses from round-trip #1
+        //
+        //
+        // Receiving responses from round-trip #2 and sending requests to read block pack descriptions
+        // (start of round-trip #3)
+        //
+        const blockPackRequests = [];
+        for (const [info, fieldName, state, response] of blockFieldRequests) {
+            const result = await response;
+            state.value = result.data;
+            if (isNotNullResourceId(result.error))
+                state.status = 'Error';
+            else if (result.resourceReady || isNotNullResourceId(result.originalResourceId))
+                state.status = 'Ready';
+            else
+                state.status = 'NotReady';
+            // For block pack we need to traverse the ref field from the resource data
+            if (fieldName === 'blockPack') {
+                const refField = result.fields.find((f) => f.name === Pl.HolderRefField);
+                if (refField === undefined)
+                    throw new Error('Block pack ref field is missing');
+                blockPackRequests.push([info, tx.getResourceData(ensureResourceIdNotNull(refField.value), false)]);
+            }
+        }
+        //
+        // <- at this point we have all the responses from round-trip #2
+        //
+        for (const [info, response] of blockPackRequests) {
+            const result = await response;
+            const bpInfo = cachedDeserialize(notEmpty(result.data));
+            info.blockConfig = extractConfig(bpInfo.config);
+            info.blockPack = bpInfo.source;
+        }
+        //
+        // <- at this point we have all the responses from round-trip #3
+        //
+        // loading ctx export template to check if we already have cached materialized template in our project
+        const ctxExportTplEnvelope = await getPreparedExportTemplateEnvelope();
+        // expected field name
+        const ctxExportTplCacheFieldName = getServiceTemplateField(ctxExportTplEnvelope.hash);
+        const ctxExportTplField = fullResourceState.fields.find((f) => f.name === ctxExportTplCacheFieldName);
+        let ctxExportTplHolder;
+        if (ctxExportTplField !== undefined)
+            ctxExportTplHolder = ensureResourceIdNotNull(ctxExportTplField.value);
+        else {
+            ctxExportTplHolder = Pl.wrapInHolder(tx, loadTemplate(tx, ctxExportTplEnvelope.spec));
+            tx.createField(field(rid, getServiceTemplateField(ctxExportTplEnvelope.hash)), 'Dynamic', ctxExportTplHolder);
+        }
+        const renderingState = { stagingRefreshTimestamp };
+        const blocksInLimboSet = new Set(blocksInLimbo);
+        const blockInfos = new Map();
+        blockInfoStates.forEach(({ id, fields, blockConfig, blockPack }) => blockInfos.set(id, new BlockInfo(id, fields, notEmpty(blockConfig), notEmpty(blockPack))));
+        // check consistency of project state
+        const blockInStruct = new Set();
+        for (const b of allBlocks(structure)) {
+            if (!blockInfos.has(b.id))
+                throw new Error(`Inconsistent project structure: no inputs for ${b.id}`);
+            blockInStruct.add(b.id);
+        }
+        blockInfos.forEach((info) => {
+            if (!blockInStruct.has(info.id))
+                throw new Error(`Inconsistent project structure: no structure entry for ${info.id}`);
+            // checking structure
+            info.check();
+        });
+        const prj = new ProjectMutator(rid, tx, author, schema, lastModified, meta, structure, renderingState, blocksInLimboSet, blockInfos, ctxExportTplHolder, projectHelper);
+        prj.fixProblemsAndMigrate();
+        return prj;
+    }
+}
+async function createProject(tx, meta = InitialBlockMeta) {
+    const prj = tx.createEphemeral(ProjectResourceType);
+    tx.lock(prj);
+    const ts = String(Date.now());
+    tx.setKValue(prj, SchemaVersionKey, JSON.stringify(SchemaVersionCurrent));
+    tx.setKValue(prj, ProjectCreatedTimestamp, ts);
+    tx.setKValue(prj, ProjectLastModifiedTimestamp, ts);
+    tx.setKValue(prj, ProjectMetaKey, JSON.stringify(meta));
+    tx.setKValue(prj, ProjectStructureKey, JSON.stringify(InitialBlockStructure));
+    tx.setKValue(prj, BlockRenderingStateKey, JSON.stringify(InitialProjectRenderingState));
+    const ctxExportTplEnvelope = await getPreparedExportTemplateEnvelope();
+    tx.createField(field(prj, getServiceTemplateField(ctxExportTplEnvelope.hash)), 'Dynamic', Pl.wrapInHolder(tx, loadTemplate(tx, ctxExportTplEnvelope.spec)));
+    return prj;
+}
+async function withProject(projectHelper, txOrPl, rid, cb) {
+    return withProjectAuthored(projectHelper, txOrPl, rid, undefined, cb);
+}
+async function withProjectAuthored(projectHelper, txOrPl, rid, author, cb, ops = {}) {
+    if (txOrPl instanceof PlClient) {
+        return await txOrPl.withWriteTx('ProjectAction', async (tx) => {
+            const mut = await ProjectMutator.load(projectHelper, tx, rid, author);
+            const result = await cb(mut);
+            if (!mut.wasModified)
+                // skipping save and commit altogether if no modifications were actually made
+                return result;
+            mut.save();
+            await tx.commit();
+            if (getDebugFlags().logProjectMutationStat)
+                console.log(JSON.stringify(tx.stat));
+            return result;
+        }, ops);
+    }
+    else {
+        const mut = await ProjectMutator.load(projectHelper, txOrPl, rid, author);
+        const result = await cb(mut);
+        mut.save();
+        return result;
+    }
+}
+
+export { ProjectMutator, createProject, withProject, withProjectAuthored };
+//# sourceMappingURL=project.js.map