@milaboratories/pl-middle-layer 1.45.5 → 1.46.0

package/src/model/project_helper.ts

@@ -1,4 +1,5 @@
-import { extractCodeWithInfo, type BlockConfig, type PlRef } from '@platforma-sdk/model';
+import type { ResultOrError, BlockConfig, PlRef, ConfigRenderLambda } from '@platforma-sdk/model';
+import { extractCodeWithInfo, ensureError } from '@platforma-sdk/model';
 import { LRUCache } from 'lru-cache';
 import type { QuickJSWASMModule } from 'quickjs-emscripten';
 import { executeSingleLambda } from '../js_render';
@@ -13,6 +14,44 @@ type EnrichmentTargetsValue = {
   value: PlRef[] | undefined;
 };
 
+/**
+ * Result of VM-based storage normalization
+ */
+interface NormalizeStorageResult {
+  storage: unknown;
+  data: unknown;
+}
+
+/**
+ * Result of VM-based storage migration.
+ * Returned by migrateStorageInVM().
+ *
+ * - Error result: { error: string } - serious failure (no context, etc.)
+ * - Success result: { newStorageJson: string, info: string, warn?: string } - migration succeeded or reset to initial
+ */
+export type MigrationResult =
+  | { error: string }
+  | { error?: undefined; newStorageJson: string; info: string; warn?: string };
+
+// Internal lambda handles for storage operations (registered by SDK's block_storage_vm.ts)
+// All callbacks are prefixed with `__pl_` to indicate internal SDK use
+const STORAGE_NORMALIZE_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_storage_normalize' };
+const STORAGE_APPLY_UPDATE_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_storage_applyUpdate' };
+const STORAGE_GET_INFO_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_storage_getInfo' };
+const STORAGE_MIGRATE_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_storage_migrate' };
+const ARGS_DERIVE_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_args_derive' };
+const PRERUN_ARGS_DERIVE_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_prerunArgs_derive' };
+// Registered by DataModel.registerCallbacks()
+const INITIAL_STORAGE_HANDLE: ConfigRenderLambda = { __renderLambda: true, handle: '__pl_storage_initial' };
+
+/**
+ * Result of args derivation from storage.
+ * Returned by __pl_args_derive and __pl_prerunArgs_derive VM callbacks.
+ */
+type ArgsDeriveResult =
+  | { error: string }
+  | { error?: undefined; value: unknown };
+
 export class ProjectHelper {
   private readonly enrichmentTargetsCache = new LRUCache<string, EnrichmentTargetsValue, EnrichmentTargetsRequest>({
     max: 256,
@@ -23,6 +62,75 @@ export class ProjectHelper {
 
   constructor(private readonly quickJs: QuickJSWASMModule) {}
 
+  // =============================================================================
+  // Args Derivation from Storage (V3+)
+  // =============================================================================
+
+  /**
+   * Derives args directly from storage JSON using VM callback.
+   * The VM extracts data from storage and calls the block's args() function.
+   *
+   * This allows the middle layer to work only with storage JSON,
+   * without needing to know the underlying data structure.
+   *
+   * @param blockConfig The block configuration (provides the model code)
+   * @param storageJson Storage as JSON string
+   * @returns The derived args object, or error if derivation fails
+   */
+  public deriveArgsFromStorage(blockConfig: BlockConfig, storageJson: string): ResultOrError<unknown> {
+    if (blockConfig.modelAPIVersion !== 2) {
+      return { error: new Error('deriveArgsFromStorage is only supported for model API version 2') };
+    }
+
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        ARGS_DERIVE_HANDLE,
+        extractCodeWithInfo(blockConfig),
+        storageJson,
+      ) as ArgsDeriveResult;
+
+      if (result.error !== undefined) {
+        return { error: new Error(result.error) };
+      }
+      return { value: result.value };
+    } catch (e) {
+      return { error: new Error('Args derivation from storage failed', { cause: ensureError(e) }) };
+    }
+  }
+
+  /**
+   * Derives prerunArgs directly from storage JSON using VM callback.
+   * Falls back to args() if prerunArgs is not defined in the block model.
+   *
+   * @param blockConfig The block configuration (provides the model code)
+   * @param storageJson Storage as JSON string
+   * @returns The derived prerunArgs, or undefined if derivation fails
+   */
+  public derivePrerunArgsFromStorage(blockConfig: BlockConfig, storageJson: string): unknown {
+    if (blockConfig.modelAPIVersion !== 2) {
+      throw new Error('derivePrerunArgsFromStorage is only supported for model API version 2');
+    }
+
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        PRERUN_ARGS_DERIVE_HANDLE,
+        extractCodeWithInfo(blockConfig),
+        storageJson,
+      ) as ArgsDeriveResult;
+
+      if (result.error !== undefined) {
+        // Return undefined if derivation fails (skip block in staging)
+        return undefined;
+      }
+      return result.value;
+    } catch {
+      // Return undefined if derivation fails (skip block in staging)
+      return undefined;
+    }
+  }
+
   private calculateEnrichmentTargets(req: EnrichmentTargetsRequest): PlRef[] | undefined {
     const blockConfig = req.blockConfig();
     if (blockConfig.enrichmentTargets === undefined) return undefined;
@@ -38,4 +146,144 @@ export class ProjectHelper {
     const cacheKey = `${key.argsRid}:${key.blockPackRid}`;
     return this.enrichmentTargetsCache.memo(cacheKey, { context: req }).value;
   }
+
+  // =============================================================================
+  // VM-based Storage Operations
+  // =============================================================================
+
+  /**
+   * Normalizes raw blockStorage data using VM-based transformation.
+   * This calls the model's `__pl_storage_normalize` callback which:
+   * - Handles BlockStorage format (with discriminator)
+   * - Handles legacy V1/V2 format ({ args, uiState })
+   * - Handles raw V3 state
+   *
+   * @param blockConfig The block configuration (provides the model code)
+   * @param rawStorage Raw storage data from resource tree (may be JSON string or object)
+   * @returns Object with { storage, state } or undefined if transformation fails
+   */
+  public normalizeStorageInVM(blockConfig: BlockConfig, rawStorage: unknown): NormalizeStorageResult | undefined {
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        STORAGE_NORMALIZE_HANDLE,
+        extractCodeWithInfo(blockConfig),
+        rawStorage,
+      ) as NormalizeStorageResult;
+      return result;
+    } catch (e) {
+      console.warn('[ProjectHelper.normalizeStorageInVM] Storage normalization failed:', e);
+      return undefined;
+    }
+  }
+
+  /**
+   * Creates initial BlockStorage for a new block using VM-based transformation.
+   * This calls the '__pl_storage_initial' callback registered by DataModel which:
+   * - Gets initial data from DataModel.getDefaultData()
+   * - Creates BlockStorage with correct version
+   *
+   * @param blockConfig The block configuration (provides the model code)
+   * @returns Initial storage as JSON string
+   * @throws Error if storage creation fails
+   */
+  public getInitialStorageInVM(blockConfig: BlockConfig): string {
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        INITIAL_STORAGE_HANDLE,
+        extractCodeWithInfo(blockConfig),
+      ) as string;
+      return result;
+    } catch (e) {
+      console.error('[ProjectHelper.getInitialStorageInVM] Initial storage creation failed:', e);
+      throw new Error(`Block initial storage creation failed: ${e}`);
+    }
+  }
+
+  /**
+   * Applies a state update using VM-based transformation.
+   * This calls the model's `__pl_storage_applyUpdate` callback which:
+   * - Normalizes current storage
+   * - Updates state while preserving other fields (version, plugins)
+   * - Returns the updated storage as JSON string
+   *
+   * @param blockConfig The block configuration (provides the model code)
+   * @param currentStorageJson Current storage as JSON string (must be defined)
+   * @param newState New state from developer
+   * @returns Updated storage as JSON string
+   * @throws Error if storage update fails
+   */
+  public applyStorageUpdateInVM(blockConfig: BlockConfig, currentStorageJson: string, payload: { operation: string; value: unknown }): string {
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        STORAGE_APPLY_UPDATE_HANDLE,
+        extractCodeWithInfo(blockConfig),
+        currentStorageJson,
+        payload,
+      ) as string;
+      return result;
+    } catch (e) {
+      console.error('[ProjectHelper.applyStorageUpdateInVM] Storage update failed:', e);
+      throw new Error(`Block storage update failed: ${e}`);
+    }
+  }
+
+  /**
+   * Gets storage info from raw storage data by calling the VM's __pl_storage_getInfo callback.
+   * Returns structured info about the storage (e.g., dataVersion).
+   *
+   * @param blockConfig Block configuration
+   * @param rawStorageJson Raw storage as JSON string (or undefined)
+   * @returns Storage info as JSON string (e.g., '{"dataVersion": 1}')
+   */
+  public getStorageInfoInVM(blockConfig: BlockConfig, rawStorageJson: string | undefined): string | undefined {
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        STORAGE_GET_INFO_HANDLE,
+        extractCodeWithInfo(blockConfig),
+        rawStorageJson,
+      ) as string;
+      return result;
+    } catch (e) {
+      console.error('[ProjectHelper.getStorageInfoInVM] Get storage info failed:', e);
+      return undefined;
+    }
+  }
+
+  // =============================================================================
+  // Block State Migrations
+  // =============================================================================
+
+  /**
+   * Runs block state migrations via VM-based transformation.
+   * This calls the model's `__pl_storage_migrate` callback which:
+   * - Normalizes current storage to get state and version
+   * - Calculates target version from number of registered migrations
+   * - Runs all necessary migrations sequentially
+   * - Returns new storage with updated state and version
+   *
+   * The middle layer doesn't need to know about dataVersion or storage internals.
+   * All migration logic is encapsulated in the model.
+   *
+   * @param blockConfig The NEW block configuration (provides the model code with migrations)
+   * @param currentStorageJson Current storage as JSON string (or undefined)
+   * @returns MigrationResult with new storage or skip/error info
+   */
+  public migrateStorageInVM(blockConfig: BlockConfig, currentStorageJson: string | undefined): MigrationResult {
+    try {
+      const result = executeSingleLambda(
+        this.quickJs,
+        STORAGE_MIGRATE_HANDLE,
+        extractCodeWithInfo(blockConfig),
+        currentStorageJson,
+      ) as MigrationResult;
+      return result;
+    } catch (e) {
+      console.error('[ProjectHelper.migrateStorageInVM] Migration failed:', e);
+      return { error: `VM execution failed: ${e}` };
+    }
+  }
 }

package/src/model/project_model.ts

@@ -73,7 +73,8 @@ export const InitialBlockMeta: ProjectMeta = {
 export const ProjectResourceType: ResourceType = { name: 'UserProject', version: '2' };
 
 export const SchemaVersionKey = 'SchemaVersion';
-export const SchemaVersionCurrent = '2';
+export const SchemaVersionV2 = '2';
+export const SchemaVersionCurrent = '3';
 
 export const ProjectCreatedTimestamp = 'ProjectCreated';
 export const ProjectLastModifiedTimestamp = 'ProjectLastModified';
@@ -101,9 +102,10 @@ export interface ProjectField {
   fieldName:
     | 'blockPack'
     | 'blockSettings'
-    | 'uiState'
+    | 'blockStorage' // Persistent storage for v3 blocks (state, plugins data, etc.)
     | 'prodArgs'
     | 'currentArgs'
+    | 'currentPrerunArgs' // Derived args for staging/prerun rendering (from prerunArgs() or args())
     | 'prodCtx'
     | 'prodUiCtx'
     | 'prodOutput'
@@ -115,15 +117,17 @@ export interface ProjectField {
     | 'stagingOutput'
     | 'stagingCtxPrevious'
     | 'stagingUiCtxPrevious'
-    | 'stagingOutputPrevious';
+    | 'stagingOutputPrevious'
+  ;
 }
 
 export const FieldsToDuplicate: Set<ProjectField['fieldName']> = new Set([
   'blockPack',
   'blockSettings',
-  'uiState',
+  'blockStorage',
   'prodArgs',
   'currentArgs',
+  'currentPrerunArgs',
   'prodCtx',
   'prodUiCtx',
   'prodOutput',
@@ -134,7 +138,7 @@ export function projectFieldName(blockId: string, fieldName: ProjectField['field
 }
 
 const projectFieldPattern
-  = /^(?<blockId>.*)-(?<fieldName>blockPack|blockSettings|uiState|prodArgs|currentArgs|prodCtx|prodUiCtx|prodOutput|prodCtxPrevious|prodUiCtxPrevious|prodOutputPrevious|stagingCtx|stagingUiCtx|stagingOutput|stagingCtxPrevious|stagingUiCtxPrevious|stagingOutputPrevious)$/;
+  = /^(?<blockId>.*)-(?<fieldName>blockPack|blockSettings|blockStorage|inputsValid|prodArgs|currentArgs|currentPrerunArgs|prodCtx|prodUiCtx|prodOutput|prodCtxPrevious|prodUiCtxPrevious|prodOutputPrevious|stagingCtx|stagingUiCtx|stagingOutput|stagingCtxPrevious|stagingUiCtxPrevious|stagingOutputPrevious)$/;
 
 export function parseProjectField(name: string): ProjectField | undefined {
   const match = name.match(projectFieldPattern);

package/src/mutator/block-pack/block_pack.ts

@@ -38,7 +38,7 @@ function parseStringConfig(configContent: string): BlockConfigContainer {
   }
 
   if (!Code.safeParse(res.data.code).success) {
-    throw new Error('No code bundle');
+    throw new Error('parseStringConfig:No code bundle');
   }
 
   return res.data as BlockConfigContainer;
@@ -75,7 +75,6 @@ export class BlockPackPreparer {
 
       case 'dev-v1': {
         const devPaths = await resolveDevPacket(spec.folder, false);
-        console.log('devPaths', devPaths);
         const configContent = await fs.promises.readFile(devPaths.config, { encoding: 'utf-8' });
         return JSON.parse(configContent);
       }

package/src/mutator/migration.ts

@@ -1,8 +1,8 @@
 import type { PlClient, PlTransaction, ResourceId } from '@milaboratories/pl-client';
-import type { ProjectStructure } from '../model/project_model';
-import { projectFieldName, ProjectStructureKey, SchemaVersionCurrent, SchemaVersionKey } from '../model/project_model';
+import type { ProjectField, ProjectStructure } from '../model/project_model';
+import { projectFieldName, ProjectStructureKey, SchemaVersionCurrent, SchemaVersionKey, SchemaVersionV2 } from '../model/project_model';
 import { BlockFrontendStateKeyPrefixV1, SchemaVersionV1 } from '../model/project_model_v1';
-import { field } from '@milaboratories/pl-client';
+import { field, isNullResourceId } from '@milaboratories/pl-client';
 import { cachedDeserialize } from '@milaboratories/ts-helpers';
 import { allBlocks } from '../model/project_model_util';
 
@@ -14,13 +14,22 @@ import { allBlocks } from '../model/project_model_util';
  */
 export async function applyProjectMigrations(pl: PlClient, rid: ResourceId) {
   await pl.withWriteTx('ProjectMigration', async (tx) => {
-    const schemaVersion = await tx.getKValueJson<string>(rid, SchemaVersionKey);
+    let schemaVersion = await tx.getKValueJson<string>(rid, SchemaVersionKey);
     if (schemaVersion === SchemaVersionCurrent) return;
+
+    // Apply migrations in sequence
     if (schemaVersion === SchemaVersionV1) {
       await migrateV1ToV2(tx, rid);
-    } else {
+      schemaVersion = SchemaVersionV2;
+    }
+
+    if (schemaVersion === SchemaVersionV2) {
+      await migrateV2ToV3(tx, rid);
+    } else if (schemaVersion !== SchemaVersionV1) {
+      // If we got here and it's not v1 (which was handled above), it's unknown
       throw new Error(`Unknown project schema version: ${schemaVersion}`);
     }
+
     tx.setKValue(rid, SchemaVersionKey, JSON.stringify(SchemaVersionCurrent));
     await tx.commit();
   });
@@ -46,8 +55,72 @@ async function migrateV1ToV2(tx: PlTransaction, rid: ResourceId) {
     const uiState = kvMap.get(kvKey);
     const valueJson = uiState ? cachedDeserialize(uiState) : {};
     const uiStateR = tx.createJsonGzValue(valueJson);
-    const uiStateF = field(rid, projectFieldName(block.id, 'uiState'));
+    const uiStateF = field(rid, projectFieldName(block.id, 'blockStorage'));
     tx.createField(uiStateF, 'Dynamic', uiStateR);
     tx.deleteKValue(rid, kvKey);
   }
 }
+
+/**
+ * Migrates the project from schema version 2 to 3.
+ *
+ * Summary of changes:
+ * - Introduces unified 'blockStorage' field containing { args, uiState }
+ * - Adds 'currentPrerunArgs' field for staging/prerun rendering
+ * - For each block:
+ *   1. Read existing 'blockStorage' field (contains uiState in v2)
+ *   2. Read existing 'currentArgs' field (contains args)
+ *   3. Create unified state = { args: currentArgs, uiState: oldState }
+ *   4. Write to new {blockId}-blockStorage field (overwrites)
+ *   5. Initialize {blockId}-currentPrerunArgs (same as prodArgs for v1/v2 blocks)
+ * - Note: currentArgs and prodArgs fields remain for compatibility layer
+ *
+ * @param tx - The transaction to use.
+ * @param rid - The resource id of the project.
+ */
+async function migrateV2ToV3(tx: PlTransaction, rid: ResourceId) {
+  const [structure, fullResourceState] = await Promise.all([
+    tx.getKValueJson<ProjectStructure>(rid, ProjectStructureKey),
+    tx.getResourceData(rid, true),
+  ]);
+
+  // Build a map of field name -> resource id for quick lookup
+  const fieldMap = new Map<string, ResourceId>();
+  for (const f of fullResourceState.fields) {
+    if (!isNullResourceId(f.value)) {
+      fieldMap.set(f.name, f.value);
+    }
+  }
+
+  for (const block of allBlocks(structure)) {
+    // Read existing field values
+    const uiStateFieldName = projectFieldName(block.id, 'uiState' as ProjectField['fieldName']);
+    const currentArgsFieldName = projectFieldName(block.id, 'currentArgs');
+
+    const uiStateRid = fieldMap.get(uiStateFieldName);
+    const currentArgsRid = fieldMap.get(currentArgsFieldName);
+
+    // Read field data in parallel where available
+    const [uiStateData, currentArgsData] = await Promise.all([
+      uiStateRid ? tx.getResourceData(uiStateRid, false) : Promise.resolve(undefined),
+      currentArgsRid ? tx.getResourceData(currentArgsRid, false) : Promise.resolve(undefined),
+    ]);
+
+    // Extract values - in v2, 'blockStorage' contains raw uiState, not wrapped
+    const uiState = uiStateData?.data ? cachedDeserialize(uiStateData.data) : {};
+    const args = currentArgsData?.data ? cachedDeserialize(currentArgsData.data) : {};
+
+    // Create unified state: { args, uiState }
+    const unifiedState = {
+      args,
+      uiState,
+    };
+
+    const blockStorageFieldName = projectFieldName(block.id, 'blockStorage');
+
+    // Write new unified blockStorage field (overwrite existing)
+    const stateR = tx.createJsonGzValue(unifiedState);
+    const stateF = field(rid, blockStorageFieldName);
+    tx.createField(stateF, 'Dynamic', stateR);
+  }
+}