@emulsify/core 4.2.1 → 4.3.0

package/config/vite/plugins/{twig-module.js → twig/twig-module.js}

@@ -10,22 +10,24 @@
  */
 
 import fs from 'fs';
+import { readFile, stat } from 'fs/promises';
 import { basename, dirname, isAbsolute, relative, resolve } from 'path';
 import Twig from 'twig';
 
 import {
   getTwigFunctionMap,
   registerTwigExtensions,
-} from '../../../src/extensions/twig/index.js';
-import { toRootRelativePath } from '../../../src/storybook/twig/reference-paths.js';
-import { resolveProjectStructure } from '../project-structure.js';
+} from '../../../../src/extensions/twig/index.js';
+import { toRootRelativePath } from '../../../../src/storybook/twig/reference-paths.js';
+import { resolveProjectStructure } from '../../project-structure.js';
 import {
-  registerConfiguredTwigExtensions,
   shouldRegisterDrupalTwigFilters,
-} from '../twig-extensions.js';
-import { firstExistingPath, safeExists } from '../utils/fs-safe.js';
-import { toPosixPath } from '../utils/paths.js';
-import { unique } from '../utils/unique.js';
+  installProjectTwigExtensions,
+} from './extensions.js';
+import { firstExistingPath, safeExists } from '../../utils/fs-safe.js';
+import { createLruCache } from '../../utils/lru.js';
+import { toPosixPath } from '../../utils/paths.js';
+import { unique } from '../../../../src/extensions/shared/lists.js';
 
 /** Twig token types that can reference another template file. */
 const templateReferenceTokenTypes = [
@@ -38,6 +40,10 @@ const templateReferenceTokenTypes = [
 
 const includeFunctionName = 'include';
 const sourceFunctionName = 'source';
+const VIRTUAL_TWIG_DEPENDENCY_PREFIX = 'virtual:emulsify-twig-dep:';
+const RESOLVED_VIRTUAL_TWIG_DEPENDENCY_PREFIX = `\0${VIRTUAL_TWIG_DEPENDENCY_PREFIX}`;
+const maxCompiledTemplateCacheEntries = 2000;
+const maxResolutionCacheEntries = 5000;
 const expressionTokenTypes = {
   arrayEnd: 'Twig.expression.type.array.end',
   arrayStart: 'Twig.expression.type.array.start',
@@ -63,16 +69,16 @@ const expressionClosingTokenTypes = new Set([
 /**
  * Cache compiled Twig templates by absolute path for the life of one build.
  *
- * @type {Map<string, { mtimeMs: number, compiled: { code: string, includes: string[], sourceReferences: string[], templateId: string, templateParams: object } }>}
+ * @type {ReturnType<typeof createLruCache>}
  */
-const compileCache = new Map();
+const compileCache = createLruCache(maxCompiledTemplateCacheEntries);
 
 /**
  * Cache resolved Twig include paths by source directory, reference, and roots.
  *
- * @type {Map<string, string|null>}
+ * @type {ReturnType<typeof createLruCache>}
  */
-const resolutionCache = new Map();
+const resolutionCache = createLruCache(maxResolutionCacheEntries);
 
 /**
  * Cache recursively discovered component grouping directories by root.
@@ -121,6 +127,13 @@ const canMemoizeByEnv = (env) => env && typeof env === 'object';
  * @returns {boolean} TRUE when the request is a renderable Twig module.
  */
 const isTwigModuleRequest = (id) => {
+  if (
+    id.startsWith(VIRTUAL_TWIG_DEPENDENCY_PREFIX) ||
+    id.startsWith(RESOLVED_VIRTUAL_TWIG_DEPENDENCY_PREFIX)
+  ) {
+    return false;
+  }
+
   const [filePath, query = ''] = id.split('?');
   if (!filePath.endsWith('.twig')) return false;
   return !query || query === 'twig' || !/(^|&)(raw|url)\b/.test(query);
@@ -134,6 +147,24 @@ const isTwigModuleRequest = (id) => {
  */
 const stripRequestQuery = (id) => id.split('?')[0];
 
+const encodeVirtualTwigDependencyId = (filePath) =>
+  `${VIRTUAL_TWIG_DEPENDENCY_PREFIX}${encodeURIComponent(resolve(filePath))}`;
+
+const resolvedVirtualTwigDependencyId = (filePath) =>
+  `${RESOLVED_VIRTUAL_TWIG_DEPENDENCY_PREFIX}${encodeURIComponent(
+    resolve(filePath),
+  )}`;
+
+const decodeVirtualTwigDependencyId = (id) => {
+  const normalizedId = id.startsWith('\0') ? id.slice(1) : id;
+
+  return normalizedId.startsWith(VIRTUAL_TWIG_DEPENDENCY_PREFIX)
+    ? decodeURIComponent(
+        normalizedId.slice(VIRTUAL_TWIG_DEPENDENCY_PREFIX.length),
+      )
+    : '';
+};
+
 /**
  * Extract the first argument from a Twig function token parameter list.
  *
@@ -445,6 +476,21 @@ const templateIdForPath = (filePath, options) => {
 const makeTemplateInstantiationCode = (templateId, params) =>
   `Twig.twig(${JSON.stringify({ ...params, id: templateId })})`;
 
+const makeTemplateInstantiationFromParamsCode = (
+  templateIdExpression,
+  paramsExpression,
+) => `Twig.twig({ ...${paramsExpression}, id: ${templateIdExpression} })`;
+
+const generateTwigDependencyModule = async (filePath, options) => {
+  const compiled = await compileTwigTemplate(filePath, options, compileCache);
+
+  return [
+    `export const templateId = ${JSON.stringify(compiled.templateId)};`,
+    `export const templateParams = ${JSON.stringify(compiled.templateParams)};`,
+    '',
+  ].join('\n');
+};
+
 /**
  * Rewrite static include/import/embed references to module-local template IDs.
  *
@@ -779,12 +825,12 @@ const invalidateKnownResolutionCacheEntries = (filePath) => {
  * @param {string} filePath - Absolute template file path.
  * @param {ReturnType<typeof makeTwigPluginOptions>} options - Twig plugin options.
  * @param {typeof compileCache} [cache=compileCache] - Shared compile cache.
- * @returns {{ code: string, includes: string[], sourceReferences: string[] }} Compiled template code and references.
+ * @returns {Promise<{ code: string, includes: string[], sourceReferences: string[] }>} Compiled template code and references.
  */
-const compileTwigTemplate = (filePath, options, cache = compileCache) => {
+const compileTwigTemplate = async (filePath, options, cache = compileCache) => {
   const absoluteFilePath = resolve(filePath);
   knownTwigFiles.add(absoluteFilePath);
-  const { mtimeMs } = fs.statSync(absoluteFilePath);
+  const { mtimeMs } = await stat(absoluteFilePath);
   const cached = cache.get(absoluteFilePath);
   if (cached?.mtimeMs === mtimeMs) {
     return cached.compiled;
@@ -792,9 +838,9 @@ const compileTwigTemplate = (filePath, options, cache = compileCache) => {
 
   const compilerTwig = Twig.factory();
   registerTwigExtensions(compilerTwig);
-  registerConfiguredTwigExtensions(compilerTwig, options);
+  installProjectTwigExtensions(compilerTwig, options);
 
-  const source = fs.readFileSync(absoluteFilePath, 'utf8');
+  const source = await readFile(absoluteFilePath, 'utf8');
   const compileOptions = {
     allowInlineIncludes: true,
     namespaces: options.namespaces,
@@ -1042,7 +1088,27 @@ export function emulsifyTwigModulePlugin(options) {
       knownTwigFiles.clear();
       transformedTwigModules.clear();
     },
-    transform(...args) {
+    resolveId(id) {
+      if (id.startsWith(VIRTUAL_TWIG_DEPENDENCY_PREFIX)) {
+        return `\0${id}`;
+      }
+
+      return null;
+    },
+    async load(id) {
+      if (!id.startsWith(RESOLVED_VIRTUAL_TWIG_DEPENDENCY_PREFIX)) {
+        return null;
+      }
+
+      const dependencyPath = decodeVirtualTwigDependencyId(id);
+      if (!dependencyPath) {
+        return null;
+      }
+
+      this.addWatchFile(dependencyPath);
+      return generateTwigDependencyModule(dependencyPath, options);
+    },
+    async transform(...args) {
       const [, id] = args;
       if (!isTwigModuleRequest(id)) {
         return null;
@@ -1051,10 +1117,11 @@ export function emulsifyTwigModulePlugin(options) {
       const filePath = stripRequestQuery(id);
       const sourceFilePath = resolve(filePath);
       transformedTwigModules.add(sourceFilePath);
-      /** @type {Map<string, ReturnType<typeof compileTwigTemplate>>} */
+      /** @type {Map<string, Awaited<ReturnType<typeof compileTwigTemplate>>>} */
       const compiledDependencyTemplates = new Map();
       /** @type {Map<string, Set<string>>} */
       const compiledDependencyReferences = new Map();
+      const queuedDependencyTemplates = new Set();
       clearDependencyImporter(sourceFilePath);
 
       /**
@@ -1089,13 +1156,15 @@ export function emulsifyTwigModulePlugin(options) {
        * @param {string[]} templateReferences - Raw references found in tokens.
        * @param {string} fromFilePath - Absolute importer path.
        * @param {typeof compileCache} cache - Shared compile cache.
-       * @returns {void}
+       * @returns {Promise<void>}
        */
-      const compileDependencyTemplates = (
+      const compileDependencyTemplates = async (
         templateReferences,
         fromFilePath,
         cache,
       ) => {
+        const dependencyRecords = [];
+
         for (const templatePath of templateReferences) {
           const dependencyPath = resolveTwigTemplate(
             templatePath,
@@ -1106,22 +1175,38 @@ export function emulsifyTwigModulePlugin(options) {
           const absoluteDependencyPath = resolve(dependencyPath);
           addCompiledDependencyReference(absoluteDependencyPath, templatePath);
           if (absoluteDependencyPath === sourceFilePath) continue;
+          if (
+            compiledDependencyTemplates.has(absoluteDependencyPath) ||
+            queuedDependencyTemplates.has(absoluteDependencyPath)
+          ) {
+            continue;
+          }
 
-          const dependencyTemplate = compiledDependencyTemplates.get(
-            absoluteDependencyPath,
-          );
-          if (dependencyTemplate) continue;
+          queuedDependencyTemplates.add(absoluteDependencyPath);
 
           addDependencyImporter(absoluteDependencyPath, sourceFilePath);
           this.addWatchFile(absoluteDependencyPath);
 
-          const compiled = compileTwigTemplate(
+          dependencyRecords.push({ absoluteDependencyPath });
+        }
+
+        const compiledDependencyRecords = await Promise.all(
+          dependencyRecords.map(async ({ absoluteDependencyPath }) => ({
             absoluteDependencyPath,
-            options,
-            cache,
-          );
+            compiled: await compileTwigTemplate(
+              absoluteDependencyPath,
+              options,
+              cache,
+            ),
+          })),
+        );
+
+        for (const {
+          absoluteDependencyPath,
+          compiled,
+        } of compiledDependencyRecords) {
           compiledDependencyTemplates.set(absoluteDependencyPath, compiled);
-          compileDependencyTemplates(
+          await compileDependencyTemplates(
             compiled.includes,
             absoluteDependencyPath,
             cache,
@@ -1133,51 +1218,56 @@ export function emulsifyTwigModulePlugin(options) {
        * Emit raw Twig source registrations for static source() references.
        *
        * @param {string} fromFilePath - Absolute file that contains the reference.
-       * @param {ReturnType<typeof compileTwigTemplate>} compiledTemplate - Compiled template metadata.
-       * @returns {string[]} Generated source map registration statements.
+       * @param {Awaited<ReturnType<typeof compileTwigTemplate>>} compiledTemplate - Compiled template metadata.
+       * @returns {Promise<string[]>} Generated source map registration statements.
        */
-      const makeSourceTemplateRegistrations = (
+      const makeSourceTemplateRegistrations = async (
         fromFilePath,
         compiledTemplate,
-      ) =>
-        (compiledTemplate.sourceReferences || []).flatMap((reference) => {
-          if (
-            reference.startsWith('@assets/') ||
-            reference.startsWith('assets/')
-          ) {
-            return [];
-          }
+      ) => {
+        const registrations = await Promise.all(
+          (compiledTemplate.sourceReferences || []).map(async (reference) => {
+            if (
+              reference.startsWith('@assets/') ||
+              reference.startsWith('assets/')
+            ) {
+              return [];
+            }
 
-          const sourcePath = resolveTwigTemplate(
-            reference,
-            dirname(fromFilePath),
-            options,
-          );
-          if (!sourcePath) return [];
-
-          const absoluteSourcePath = resolve(sourcePath);
-          addDependencyImporter(absoluteSourcePath, sourceFilePath);
-          this.addWatchFile(absoluteSourcePath);
-          const sourceText = fs.readFileSync(absoluteSourcePath, 'utf8');
-
-          return unique([
-            reference,
-            templateIdForPath(absoluteSourcePath, options),
-          ]).map(
-            (key) =>
-              `__emulsifySourceTemplates.set(${JSON.stringify(
-                key,
-              )}, ${JSON.stringify(sourceText)});`,
-          );
-        });
+            const sourcePath = resolveTwigTemplate(
+              reference,
+              dirname(fromFilePath),
+              options,
+            );
+            if (!sourcePath) return [];
+
+            const absoluteSourcePath = resolve(sourcePath);
+            addDependencyImporter(absoluteSourcePath, sourceFilePath);
+            this.addWatchFile(absoluteSourcePath);
+            const sourceText = await readFile(absoluteSourcePath, 'utf8');
+
+            return unique([
+              reference,
+              templateIdForPath(absoluteSourcePath, options),
+            ]).map(
+              (key) =>
+                `__emulsifySourceTemplates.set(${JSON.stringify(
+                  key,
+                )}, ${JSON.stringify(sourceText)});`,
+            );
+          }),
+        );
+
+        return registrations.flat();
+      };
 
       try {
-        const compiled = compileTwigTemplate(
+        const compiled = await compileTwigTemplate(
           sourceFilePath,
           options,
           compileCache,
         );
-        compileDependencyTemplates(
+        await compileDependencyTemplates(
           compiled.includes,
           sourceFilePath,
           compileCache,
@@ -1190,29 +1280,40 @@ export function emulsifyTwigModulePlugin(options) {
           .map(([dependencyPath, compiledDependency], index) => ({
             dependencyPath,
             compiledDependency,
-            variableName: `__emulsifyDependency${index}`,
+            idVariableName: `__emulsifyDependencyTemplateId${index}`,
+            paramsVariableName: `__emulsifyDependencyTemplateParams${index}`,
+            templateVariableName: `__emulsifyDependency${index}`,
+            virtualModuleId: encodeVirtualTwigDependencyId(dependencyPath),
           }));
+        const dependencyTemplateImports = dependencyTemplateRecords
+          .map(
+            ({ idVariableName, paramsVariableName, virtualModuleId }) =>
+              `import { templateId as ${idVariableName}, templateParams as ${paramsVariableName} } from ${JSON.stringify(
+                virtualModuleId,
+              )};`,
+          )
+          .join('\n');
         const dependencyTemplateCode = dependencyTemplateRecords
           .map(
-            ({ compiledDependency, variableName }) => `
-              const ${variableName} = ${makeTemplateInstantiationCode(
-                compiledDependency.templateId,
-                compiledDependency.templateParams,
+            ({ idVariableName, paramsVariableName, templateVariableName }) => `
+              const ${templateVariableName} = ${makeTemplateInstantiationFromParamsCode(
+                idVariableName,
+                paramsVariableName,
               )};
-              ${variableName}.method = 'emulsify';
+              ${templateVariableName}.method = 'emulsify';
             `,
           )
           .join('\n');
         const includeTemplateRegistrations = [
           ...dependencyTemplateRecords.flatMap(
-            ({ dependencyPath, variableName }) =>
+            ({ dependencyPath, templateVariableName }) =>
               Array.from(
                 compiledDependencyReferences.get(dependencyPath) || [],
               ).map(
                 (reference) =>
                   `__emulsifyIncludeTemplates.set(${JSON.stringify(
                     reference,
-                  )}, (context = {}) => ${variableName}.render(context));`,
+                  )}, (context = {}) => ${templateVariableName}.render(context));`,
               ),
           ),
           ...Array.from(
@@ -1224,29 +1325,34 @@ export function emulsifyTwigModulePlugin(options) {
               )}, (context = {}) => __emulsifyTemplate.render(context));`,
           ),
         ].join('\n');
-        const sourceTemplateRegistrations = [
-          ...dependencyTemplateRecords.flatMap(
-            ({ dependencyPath, compiledDependency }) =>
-              makeSourceTemplateRegistrations(
-                dependencyPath,
-                compiledDependency,
-              ),
-          ),
-          ...makeSourceTemplateRegistrations(sourceFilePath, compiled),
-        ].join('\n');
+        const sourceTemplateRegistrations = (
+          await Promise.all([
+            ...dependencyTemplateRecords.map(
+              ({ dependencyPath, compiledDependency }) =>
+                makeSourceTemplateRegistrations(
+                  dependencyPath,
+                  compiledDependency,
+                ),
+            ),
+            makeSourceTemplateRegistrations(sourceFilePath, compiled),
+          ])
+        )
+          .flat()
+          .join('\n');
         const renderErrorPrefix = JSON.stringify(
           `An error occurred whilst rendering ${toPosixPath(filePath)}: `,
         );
         const moduleCode = `
+          ${dependencyTemplateImports}
           import { factory } from 'twig';
           import { registerTwigExtensions } from '@emulsify/core/extensions/twig';
-          import { registerConfiguredTwigExtensions } from 'virtual:emulsify-twig-extension-installers';
+	import { installProjectTwigExtensions } from 'virtual:emulsify-twig-extension-installers';
           import { createTwigIncludeFunction } from '@emulsify/core/storybook/twig/include-function';
           import { createTwigSourceFunction } from '@emulsify/core/storybook/twig/source-function';
 
           const Twig = factory();
           registerTwigExtensions(Twig);
-          registerConfiguredTwigExtensions(Twig);
+	installProjectTwigExtensions(Twig);
           Twig.extend((TwigCore) => {
             TwigCore.Templates.registerLoader(
               'emulsify',
@@ -1329,6 +1435,7 @@ export function emulsifyTwigModulePlugin(options) {
 
       const fileExists = safeExists(filePath);
       const knownFile = knownTwigFiles.has(filePath);
+      const dependencyModuleId = resolvedVirtualTwigDependencyId(filePath);
       const importers = dependencyImporters.get(filePath);
       const projectRoot = options.projectDir || options.root;
       const projectPathChanged =
@@ -1388,6 +1495,11 @@ export function emulsifyTwigModulePlugin(options) {
       }
 
       const modules = new Set(moduleGraph.getModulesByFile(filePath) || []);
+      const dependencyModule = moduleGraph.getModuleById?.(dependencyModuleId);
+      if (dependencyModule) {
+        moduleGraph.invalidateModule?.(dependencyModule);
+        modules.add(dependencyModule);
+      }
       for (const importer of affectedImporters) {
         compileCache.delete(importer);