@wdprlib/parser 2.0.9 → 2.1.0

package/dist/index.cjs

@@ -34,6 +34,7 @@ __export(exports_src, {
   text: () => import_ast5.text,
   resolveModules: () => resolveModules,
   resolveListUsers: () => resolveListUsers,
+  resolveIncludesAsync: () => resolveIncludesAsync,
   resolveIncludes: () => resolveIncludes,
   parseTags: () => parseTags,
   parseParent: () => parseParent,
@@ -646,7 +647,139 @@ var BLOCK_START_TOKENS = [
   "CLEAR_FLOAT_RIGHT"
 ];
 
+// packages/parser/src/parser/rules/utils.ts
+var SAFE_ATTRIBUTES = new Set([
+  "accept",
+  "align",
+  "alt",
+  "autocapitalize",
+  "autoplay",
+  "background",
+  "bgcolor",
+  "border",
+  "buffered",
+  "checked",
+  "cite",
+  "class",
+  "cols",
+  "colspan",
+  "contenteditable",
+  "controls",
+  "coords",
+  "datetime",
+  "decoding",
+  "default",
+  "dir",
+  "dirname",
+  "disabled",
+  "download",
+  "draggable",
+  "for",
+  "form",
+  "headers",
+  "height",
+  "hidden",
+  "high",
+  "href",
+  "hreflang",
+  "id",
+  "inputmode",
+  "ismap",
+  "itemprop",
+  "kind",
+  "label",
+  "lang",
+  "list",
+  "loop",
+  "low",
+  "max",
+  "maxlength",
+  "min",
+  "minlength",
+  "multiple",
+  "muted",
+  "name",
+  "optimum",
+  "pattern",
+  "placeholder",
+  "poster",
+  "preload",
+  "readonly",
+  "required",
+  "reversed",
+  "role",
+  "rows",
+  "rowspan",
+  "scope",
+  "selected",
+  "shape",
+  "size",
+  "sizes",
+  "span",
+  "spellcheck",
+  "src",
+  "srclang",
+  "srcset",
+  "start",
+  "step",
+  "style",
+  "tabindex",
+  "target",
+  "title",
+  "translate",
+  "type",
+  "usemap",
+  "value",
+  "width",
+  "wrap"
+]);
+function filterUnsafeAttributes(attrs) {
+  const result = {};
+  for (const [key, value] of Object.entries(attrs)) {
+    const lower = key.toLowerCase();
+    if (lower.startsWith("on"))
+      continue;
+    if (lower.startsWith("aria-") || lower.startsWith("data-")) {
+      result[key] = value;
+      continue;
+    }
+    if (!SAFE_ATTRIBUTES.has(lower))
+      continue;
+    if (lower === "id") {
+      result[key] = value.startsWith("u-") ? value : `u-${value}`;
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+function parseBlockName(ctx, startPos) {
+  let pos = startPos;
+  let consumed = 0;
+  const token = ctx.tokens[pos];
+  if (!token || token.type !== "TEXT" && token.type !== "IDENTIFIER") {
+    return null;
+  }
+  let name = token.value.toLowerCase();
+  consumed++;
+  pos++;
+  if (ctx.tokens[pos]?.type === "UNDERSCORE") {
+    name += "_";
+    consumed++;
+  }
+  return { name, consumed };
+}
+
 // packages/parser/src/parser/rules/inline/utils.ts
+function isExcludedBlockToken(ctx, tokenPos) {
+  if (!ctx.excludedBlockNames?.size)
+    return false;
+  const token = ctx.tokens[tokenPos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN")
+    return false;
+  const nameResult = parseBlockName(ctx, tokenPos + 1);
+  return nameResult !== null && ctx.excludedBlockNames.has(nameResult.name);
+}
 function canApplyInlineRule(rule, token) {
   if (rule.startTokens.length === 0) {
     return true;
@@ -738,7 +871,8 @@ function parseInlineUntil(ctx, endType) {
           isInvalidHeading = true;
         }
       }
-      const isBlockStart = nextMeaningfulToken && BLOCK_START_TOKENS.includes(nextMeaningfulToken.type) && nextMeaningfulToken.lineStart && !isOrphanCloseSpan && !isAnchorName && !isInvalidBlockOpen && !isInvalidHeading;
+      const isExcludedBlock = (nextMeaningfulToken?.type === "BLOCK_OPEN" || nextMeaningfulToken?.type === "BLOCK_END_OPEN") && isExcludedBlockToken(ctx, pos + lookAhead);
+      const isBlockStart = nextMeaningfulToken && BLOCK_START_TOKENS.includes(nextMeaningfulToken.type) && nextMeaningfulToken.lineStart && !isOrphanCloseSpan && !isAnchorName && !isInvalidBlockOpen && !isInvalidHeading && !isExcludedBlock;
       if (!nextMeaningfulToken || nextMeaningfulToken.type === "NEWLINE" || nextMeaningfulToken.type === "EOF" || isBlockStart) {
         if (isBlockStart && nodes.length > 0) {
           const nextPos = pos + lookAhead;
@@ -1076,129 +1210,6 @@ function buildListData(topLtype, list) {
   };
 }
 
-// packages/parser/src/parser/rules/utils.ts
-var SAFE_ATTRIBUTES = new Set([
-  "accept",
-  "align",
-  "alt",
-  "autocapitalize",
-  "autoplay",
-  "background",
-  "bgcolor",
-  "border",
-  "buffered",
-  "checked",
-  "cite",
-  "class",
-  "cols",
-  "colspan",
-  "contenteditable",
-  "controls",
-  "coords",
-  "datetime",
-  "decoding",
-  "default",
-  "dir",
-  "dirname",
-  "disabled",
-  "download",
-  "draggable",
-  "for",
-  "form",
-  "headers",
-  "height",
-  "hidden",
-  "high",
-  "href",
-  "hreflang",
-  "id",
-  "inputmode",
-  "ismap",
-  "itemprop",
-  "kind",
-  "label",
-  "lang",
-  "list",
-  "loop",
-  "low",
-  "max",
-  "maxlength",
-  "min",
-  "minlength",
-  "multiple",
-  "muted",
-  "name",
-  "optimum",
-  "pattern",
-  "placeholder",
-  "poster",
-  "preload",
-  "readonly",
-  "required",
-  "reversed",
-  "role",
-  "rows",
-  "rowspan",
-  "scope",
-  "selected",
-  "shape",
-  "size",
-  "sizes",
-  "span",
-  "spellcheck",
-  "src",
-  "srclang",
-  "srcset",
-  "start",
-  "step",
-  "style",
-  "tabindex",
-  "target",
-  "title",
-  "translate",
-  "type",
-  "usemap",
-  "value",
-  "width",
-  "wrap"
-]);
-function filterUnsafeAttributes(attrs) {
-  const result = {};
-  for (const [key, value] of Object.entries(attrs)) {
-    const lower = key.toLowerCase();
-    if (lower.startsWith("on"))
-      continue;
-    if (lower.startsWith("aria-") || lower.startsWith("data-")) {
-      result[key] = value;
-      continue;
-    }
-    if (!SAFE_ATTRIBUTES.has(lower))
-      continue;
-    if (lower === "id") {
-      result[key] = value.startsWith("u-") ? value : `u-${value}`;
-      continue;
-    }
-    result[key] = value;
-  }
-  return result;
-}
-function parseBlockName(ctx, startPos) {
-  let pos = startPos;
-  let consumed = 0;
-  const token = ctx.tokens[pos];
-  if (!token || token.type !== "TEXT" && token.type !== "IDENTIFIER") {
-    return null;
-  }
-  let name = token.value.toLowerCase();
-  consumed++;
-  pos++;
-  if (ctx.tokens[pos]?.type === "UNDERSCORE") {
-    name += "_";
-    consumed++;
-  }
-  return { name, consumed };
-}
-
 // packages/parser/src/parser/rules/block/utils.ts
 function canApplyBlockRule(rule, token) {
   if (rule.requiresLineStart && !token.lineStart) {
@@ -1209,11 +1220,13 @@ function canApplyBlockRule(rule, token) {
   }
   return rule.startTokens.includes(token.type);
 }
-function parseBlocksUntil(ctx, closeCondition) {
+function parseBlocksUntil(ctx, closeCondition, options) {
   const elements = [];
   let consumed = 0;
   let pos = ctx.pos;
-  const { blockRules, blockFallbackRule } = ctx;
+  const excluded = options?.excludedBlockNames;
+  const blockRules = excluded ? ctx.blockRules.filter((r) => !excluded.has(r.name)) : ctx.blockRules;
+  const { blockFallbackRule } = ctx;
   while (pos < ctx.tokens.length) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
@@ -1234,7 +1247,13 @@ function parseBlocksUntil(ctx, closeCondition) {
       continue;
     }
     let matched = false;
-    const blockCtx = { ...ctx, pos, blockCloseCondition: closeCondition };
+    const blockCtx = {
+      ...ctx,
+      pos,
+      blockRules,
+      blockCloseCondition: closeCondition,
+      excludedBlockNames: excluded
+    };
     for (const rule of blockRules) {
       if (canApplyBlockRule(rule, token)) {
         const result = rule.parse(blockCtx);
@@ -2619,44 +2638,7 @@ function consumeCloseTag2(ctx, pos) {
     closeConsumed++;
   return closeConsumed;
 }
-function mergeParagraphs(elements) {
-  const result = [];
-  let mergedElements = [];
-  for (const elem of elements) {
-    if (elem.element === "container" && elem.data && typeof elem.data === "object" && "type" in elem.data && elem.data.type === "paragraph") {
-      if (mergedElements.length > 0) {
-        mergedElements.push({ element: "line-break" });
-      }
-      if ("elements" in elem.data && Array.isArray(elem.data.elements)) {
-        mergedElements.push(...elem.data.elements);
-      }
-    } else {
-      if (mergedElements.length > 0) {
-        result.push({
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements: mergedElements
-          }
-        });
-        mergedElements = [];
-      }
-      result.push(elem);
-    }
-  }
-  if (mergedElements.length > 0) {
-    result.push({
-      element: "container",
-      data: {
-        type: "paragraph",
-        attributes: {},
-        elements: mergedElements
-      }
-    });
-  }
-  return result;
-}
+var EXCLUDED_BLOCKS = new Set(["collapsible"]);
 var collapsibleRule = {
   name: "collapsible",
   startTokens: ["BLOCK_OPEN"],
@@ -2720,18 +2702,16 @@ var collapsibleRule = {
         bodyElements = [];
       }
     } else {
-      const bodyCtx = {
-        ...ctx,
-        pos,
-        blockRules: ctx.blockRules.filter((r) => r.name !== "collapsible")
-      };
+      const bodyCtx = { ...ctx, pos };
       const closeCondition = (checkCtx) => {
         return isCollapsibleClose(checkCtx, checkCtx.pos);
       };
-      const bodyResult = parseBlocksUntil(bodyCtx, closeCondition);
+      const bodyResult = parseBlocksUntil(bodyCtx, closeCondition, {
+        excludedBlockNames: EXCLUDED_BLOCKS
+      });
       consumed += bodyResult.consumed;
       pos += bodyResult.consumed;
-      bodyElements = mergeParagraphs(bodyResult.elements);
+      bodyElements = bodyResult.elements;
     }
     if (!isCollapsibleClose(ctx, pos)) {
       ctx.diagnostics.push({
@@ -9989,7 +9969,7 @@ function resolveIncludes(source, fetcher, options) {
   if (options?.settings && !options.settings.enablePageSyntax) {
     return source;
   }
-  const maxDepth = options?.maxDepth ?? 5;
+  const maxIterations = options?.maxIterations ?? 10;
   const cache = new Map;
   const cachedFetcher = (pageRef) => {
     const key = normalizePageKey(pageRef);
@@ -10005,7 +9985,29 @@ function resolveIncludes(source, fetcher, options) {
     cache.set(key, result);
     return result;
   };
-  return expandText(source, cachedFetcher, 0, maxDepth, []);
+  return expandIterative(source, cachedFetcher, maxIterations);
+}
+async function resolveIncludesAsync(source, fetcher, options) {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return source;
+  }
+  const maxIterations = options?.maxIterations ?? 10;
+  const cache = new Map;
+  const cachedFetcher = async (pageRef) => {
+    const key = normalizePageKey(pageRef);
+    if (cache.has(key)) {
+      return cache.get(key);
+    }
+    let result;
+    try {
+      result = await fetcher(pageRef);
+    } catch {
+      result = null;
+    }
+    cache.set(key, result);
+    return result;
+  };
+  return expandIterativeAsync(source, cachedFetcher, maxIterations);
 }
 var INCLUDE_PATTERN = /^\[\[include\s([^\]]*(?:\](?!\])[^\]]*)*)\]\]/gim;
 function parseIncludeDirective(inner) {
@@ -10055,26 +10057,55 @@ function parseIncludeDirective(inner) {
   }
   return { location, variables };
 }
-function expandText(source, fetcher, depth, maxDepth, trace) {
-  if (depth >= maxDepth)
-    return source;
-  return source.replace(INCLUDE_PATTERN, (_match, inner) => {
-    const { location, variables } = parseIncludeDirective(inner);
-    const pageKey = normalizePageKey(location);
-    if (trace.includes(pageKey)) {
-      return `[[div class="error-block"]]
-Circular include detected: "${location.page}"
+function replaceOneInclude(_match, inner, fetcher) {
+  const { location, variables } = parseIncludeDirective(inner);
+  const content = fetcher(location);
+  if (content === null) {
+    return `[[div class="error-block"]]
+Page to be included "${location.page}" cannot be found!
 [[/div]]`;
-    }
-    const content = fetcher(location);
-    if (content === null) {
-      return `[[div class="error-block"]]
+  }
+  return substituteVariables(content, variables);
+}
+function expandIterative(source, fetcher, maxIterations) {
+  let current = source;
+  for (let i = 0;i < maxIterations; i++) {
+    const previous = current;
+    current = current.replace(INCLUDE_PATTERN, (_match, inner) => replaceOneInclude(_match, inner, fetcher));
+    if (current === previous)
+      break;
+  }
+  return current;
+}
+async function expandIterativeAsync(source, fetcher, maxIterations) {
+  let current = source;
+  for (let i = 0;i < maxIterations; i++) {
+    const previous = current;
+    const pattern = new RegExp(INCLUDE_PATTERN.source, INCLUDE_PATTERN.flags);
+    let result = "";
+    let lastPos = 0;
+    let match;
+    while ((match = pattern.exec(current)) !== null) {
+      const fullMatch = match[0];
+      const inner = match[1];
+      result += current.slice(lastPos, match.index);
+      const { location, variables } = parseIncludeDirective(inner);
+      const content = await fetcher(location);
+      if (content === null) {
+        result += `[[div class="error-block"]]
 Page to be included "${location.page}" cannot be found!
 [[/div]]`;
+      } else {
+        result += substituteVariables(content, variables);
+      }
+      lastPos = match.index + fullMatch.length;
     }
-    const substituted = substituteVariables(content, variables);
-    return expandText(substituted, fetcher, depth + 1, maxDepth, [...trace, pageKey]);
-  });
+    result += current.slice(lastPos);
+    current = result;
+    if (current === previous)
+      break;
+  }
+  return current;
 }
 function normalizePageKey(location) {
   const site = location.site ?? "";

package/dist/index.d.cts

@@ -859,21 +859,36 @@ import { PageRef, WikitextSettings as WikitextSettings3 } from "@wdprlib/ast";
 */
 type IncludeFetcher = (pageRef: PageRef) => string | null;
 /**
-* Options for resolveIncludes
+* Async callback to fetch page content for include resolution.
+* Returns a promise of the wikitext source, or null if the page does not exist.
+*
+* @security The fetcher is called with user-provided page references.
+* Implementations should validate and sanitize page references before
+* using them in database queries or file system access.
+*/
+type AsyncIncludeFetcher = (pageRef: PageRef) => Promise<string | null>;
+/**
+* Options for resolveIncludes / resolveIncludesAsync
 */
 interface ResolveIncludesOptions {
-	/** Maximum recursion depth for nested includes (default: 5) */
-	maxDepth?: number;
+	/**
+	* Maximum number of expansion iterations (default: 10).
+	*
+	* Each iteration replaces all `[[include]]` directives in the current
+	* source with fetched content. Iteration stops when the source is
+	* unchanged or this limit is reached.
+	*/
+	maxIterations?: number;
 	/** Wikitext settings. If enablePageSyntax is false, includes are not expanded. */
 	settings?: WikitextSettings3;
 }
 /**
 * Expand all [[include]] directives in the source text.
 *
-* Include directives are treated as macro expansions: `[[include page]]`
-* is replaced with the fetched page content (after variable substitution).
-* The result is a single expanded text that can be parsed as a whole,
-* allowing block structures (like div) to span across include boundaries.
+* Uses Wikidot-compatible iterative expansion: each iteration replaces
+* all include directives in the current source with fetched (and
+* variable-substituted) content. Iteration continues until no further
+* changes occur or `maxIterations` is reached.
 *
 * @example
 * ```ts
@@ -883,6 +898,21 @@ interface ResolveIncludesOptions {
 */
 declare function resolveIncludes(source: string, fetcher: IncludeFetcher, options?: ResolveIncludesOptions): string;
 /**
+* Async version of {@link resolveIncludes}.
+*
+* Expand all [[include]] directives using an async fetcher, allowing
+* page content to be loaded from async sources such as databases.
+*
+* @example
+* ```ts
+* const expanded = await resolveIncludesAsync(source, async (ref) => {
+*   return await db.getPageContent(ref.page);
+* });
+* const ast = parse(expanded);
+* ```
+*/
+declare function resolveIncludesAsync(source: string, fetcher: AsyncIncludeFetcher, options?: ResolveIncludesOptions): Promise<string>;
+/**
 * Compile a ListUsers template string into an executable function.
 *
 * The template is split into alternating static strings and dynamic getter
@@ -981,4 +1011,4 @@ interface ResolveOptions {
 */
 declare function resolveModules(ast: SyntaxTree2, dataProvider: DataProvider, options: ResolveOptions): Promise<SyntaxTree2>;
 import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
-export { tokenize, text, resolveModules, resolveListUsers, resolveIncludes, parseTags, parseParent, parseOrder, parseNumericSelector, parseDateSelector, parseCategory, parse, paragraph, normalizeQuery, listItemSubList, listItemElements, list, link, lineBreak, italics, isListUsersModule, horizontalRule, heading, extractListUsersVariables, extractDataRequirements, createToken, createSettings, createPosition, createPoint, container, compileTemplate, compileListUsersTemplate, bold, WikitextSettings4 as WikitextSettings, WikitextMode, Version2 as Version, VariableMap, VariableContext, UserInfo, TokenType, Token, TocEntry2 as TocEntry, TableRow, TableData, TableCell, TabData, SyntaxTree3 as SyntaxTree, SiteContext, STYLE_SLOT_PREFIX, ResolveOptions, ResolveIncludesOptions, Position2 as Position, Point, ParserOptions, Parser, ParseResult2 as ParseResult, ParseFunction, PageRef2 as PageRef, PageData, NormalizedTags, NormalizedParent, NormalizedOrder, NormalizedNumericSelector, NormalizedListPagesQuery, NormalizedDateSelector, NormalizedCategory, Module4 as Module, ListUsersVariableContext, ListUsersVariable, ListUsersUserData, ListUsersExternalData, ListUsersDataRequirement, ListUsersDataFetcher, ListUsersCompiledTemplate, ListType, ListPagesVariable, ListPagesQuery, ListPagesExternalData, ListPagesDataRequirement, ListPagesDataFetcher, ListItem, ListData, LinkType, LinkLocation, LinkLabel, LexerOptions, Lexer, IncludeFetcher, ImageSource, HeadingLevel, Heading, HeaderType, FloatAlignment, ExtractionResult, Embed, Element6 as Element, DiagnosticSeverity, Diagnostic2 as Diagnostic, DefinitionListItem, DateItem, DataRequirements, DataProvider, DEFAULT_SETTINGS, ContainerType, ContainerData, CompiledTemplate, CollapsibleData, CodeBlockData2 as CodeBlockData, ClearFloat, AttributeMap, AnchorTarget, Alignment, AlignType };
+export { tokenize, text, resolveModules, resolveListUsers, resolveIncludesAsync, resolveIncludes, parseTags, parseParent, parseOrder, parseNumericSelector, parseDateSelector, parseCategory, parse, paragraph, normalizeQuery, listItemSubList, listItemElements, list, link, lineBreak, italics, isListUsersModule, horizontalRule, heading, extractListUsersVariables, extractDataRequirements, createToken, createSettings, createPosition, createPoint, container, compileTemplate, compileListUsersTemplate, bold, WikitextSettings4 as WikitextSettings, WikitextMode, Version2 as Version, VariableMap, VariableContext, UserInfo, TokenType, Token, TocEntry2 as TocEntry, TableRow, TableData, TableCell, TabData, SyntaxTree3 as SyntaxTree, SiteContext, STYLE_SLOT_PREFIX, ResolveOptions, ResolveIncludesOptions, Position2 as Position, Point, ParserOptions, Parser, ParseResult2 as ParseResult, ParseFunction, PageRef2 as PageRef, PageData, NormalizedTags, NormalizedParent, NormalizedOrder, NormalizedNumericSelector, NormalizedListPagesQuery, NormalizedDateSelector, NormalizedCategory, Module4 as Module, ListUsersVariableContext, ListUsersVariable, ListUsersUserData, ListUsersExternalData, ListUsersDataRequirement, ListUsersDataFetcher, ListUsersCompiledTemplate, ListType, ListPagesVariable, ListPagesQuery, ListPagesExternalData, ListPagesDataRequirement, ListPagesDataFetcher, ListItem, ListData, LinkType, LinkLocation, LinkLabel, LexerOptions, Lexer, IncludeFetcher, ImageSource, HeadingLevel, Heading, HeaderType, FloatAlignment, ExtractionResult, Embed, Element6 as Element, DiagnosticSeverity, Diagnostic2 as Diagnostic, DefinitionListItem, DateItem, DataRequirements, DataProvider, DEFAULT_SETTINGS, ContainerType, ContainerData, CompiledTemplate, CollapsibleData, CodeBlockData2 as CodeBlockData, ClearFloat, AttributeMap, AsyncIncludeFetcher, AnchorTarget, Alignment, AlignType };

package/dist/index.d.ts

@@ -859,21 +859,36 @@ import { PageRef, WikitextSettings as WikitextSettings3 } from "@wdprlib/ast";
 */
 type IncludeFetcher = (pageRef: PageRef) => string | null;
 /**
-* Options for resolveIncludes
+* Async callback to fetch page content for include resolution.
+* Returns a promise of the wikitext source, or null if the page does not exist.
+*
+* @security The fetcher is called with user-provided page references.
+* Implementations should validate and sanitize page references before
+* using them in database queries or file system access.
+*/
+type AsyncIncludeFetcher = (pageRef: PageRef) => Promise<string | null>;
+/**
+* Options for resolveIncludes / resolveIncludesAsync
 */
 interface ResolveIncludesOptions {
-	/** Maximum recursion depth for nested includes (default: 5) */
-	maxDepth?: number;
+	/**
+	* Maximum number of expansion iterations (default: 10).
+	*
+	* Each iteration replaces all `[[include]]` directives in the current
+	* source with fetched content. Iteration stops when the source is
+	* unchanged or this limit is reached.
+	*/
+	maxIterations?: number;
 	/** Wikitext settings. If enablePageSyntax is false, includes are not expanded. */
 	settings?: WikitextSettings3;
 }
 /**
 * Expand all [[include]] directives in the source text.
 *
-* Include directives are treated as macro expansions: `[[include page]]`
-* is replaced with the fetched page content (after variable substitution).
-* The result is a single expanded text that can be parsed as a whole,
-* allowing block structures (like div) to span across include boundaries.
+* Uses Wikidot-compatible iterative expansion: each iteration replaces
+* all include directives in the current source with fetched (and
+* variable-substituted) content. Iteration continues until no further
+* changes occur or `maxIterations` is reached.
 *
 * @example
 * ```ts
@@ -883,6 +898,21 @@ interface ResolveIncludesOptions {
 */
 declare function resolveIncludes(source: string, fetcher: IncludeFetcher, options?: ResolveIncludesOptions): string;
 /**
+* Async version of {@link resolveIncludes}.
+*
+* Expand all [[include]] directives using an async fetcher, allowing
+* page content to be loaded from async sources such as databases.
+*
+* @example
+* ```ts
+* const expanded = await resolveIncludesAsync(source, async (ref) => {
+*   return await db.getPageContent(ref.page);
+* });
+* const ast = parse(expanded);
+* ```
+*/
+declare function resolveIncludesAsync(source: string, fetcher: AsyncIncludeFetcher, options?: ResolveIncludesOptions): Promise<string>;
+/**
 * Compile a ListUsers template string into an executable function.
 *
 * The template is split into alternating static strings and dynamic getter
@@ -981,4 +1011,4 @@ interface ResolveOptions {
 */
 declare function resolveModules(ast: SyntaxTree2, dataProvider: DataProvider, options: ResolveOptions): Promise<SyntaxTree2>;
 import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
-export { tokenize, text, resolveModules, resolveListUsers, resolveIncludes, parseTags, parseParent, parseOrder, parseNumericSelector, parseDateSelector, parseCategory, parse, paragraph, normalizeQuery, listItemSubList, listItemElements, list, link, lineBreak, italics, isListUsersModule, horizontalRule, heading, extractListUsersVariables, extractDataRequirements, createToken, createSettings, createPosition, createPoint, container, compileTemplate, compileListUsersTemplate, bold, WikitextSettings4 as WikitextSettings, WikitextMode, Version2 as Version, VariableMap, VariableContext, UserInfo, TokenType, Token, TocEntry2 as TocEntry, TableRow, TableData, TableCell, TabData, SyntaxTree3 as SyntaxTree, SiteContext, STYLE_SLOT_PREFIX, ResolveOptions, ResolveIncludesOptions, Position2 as Position, Point, ParserOptions, Parser, ParseResult2 as ParseResult, ParseFunction, PageRef2 as PageRef, PageData, NormalizedTags, NormalizedParent, NormalizedOrder, NormalizedNumericSelector, NormalizedListPagesQuery, NormalizedDateSelector, NormalizedCategory, Module4 as Module, ListUsersVariableContext, ListUsersVariable, ListUsersUserData, ListUsersExternalData, ListUsersDataRequirement, ListUsersDataFetcher, ListUsersCompiledTemplate, ListType, ListPagesVariable, ListPagesQuery, ListPagesExternalData, ListPagesDataRequirement, ListPagesDataFetcher, ListItem, ListData, LinkType, LinkLocation, LinkLabel, LexerOptions, Lexer, IncludeFetcher, ImageSource, HeadingLevel, Heading, HeaderType, FloatAlignment, ExtractionResult, Embed, Element6 as Element, DiagnosticSeverity, Diagnostic2 as Diagnostic, DefinitionListItem, DateItem, DataRequirements, DataProvider, DEFAULT_SETTINGS, ContainerType, ContainerData, CompiledTemplate, CollapsibleData, CodeBlockData2 as CodeBlockData, ClearFloat, AttributeMap, AnchorTarget, Alignment, AlignType };
+export { tokenize, text, resolveModules, resolveListUsers, resolveIncludesAsync, resolveIncludes, parseTags, parseParent, parseOrder, parseNumericSelector, parseDateSelector, parseCategory, parse, paragraph, normalizeQuery, listItemSubList, listItemElements, list, link, lineBreak, italics, isListUsersModule, horizontalRule, heading, extractListUsersVariables, extractDataRequirements, createToken, createSettings, createPosition, createPoint, container, compileTemplate, compileListUsersTemplate, bold, WikitextSettings4 as WikitextSettings, WikitextMode, Version2 as Version, VariableMap, VariableContext, UserInfo, TokenType, Token, TocEntry2 as TocEntry, TableRow, TableData, TableCell, TabData, SyntaxTree3 as SyntaxTree, SiteContext, STYLE_SLOT_PREFIX, ResolveOptions, ResolveIncludesOptions, Position2 as Position, Point, ParserOptions, Parser, ParseResult2 as ParseResult, ParseFunction, PageRef2 as PageRef, PageData, NormalizedTags, NormalizedParent, NormalizedOrder, NormalizedNumericSelector, NormalizedListPagesQuery, NormalizedDateSelector, NormalizedCategory, Module4 as Module, ListUsersVariableContext, ListUsersVariable, ListUsersUserData, ListUsersExternalData, ListUsersDataRequirement, ListUsersDataFetcher, ListUsersCompiledTemplate, ListType, ListPagesVariable, ListPagesQuery, ListPagesExternalData, ListPagesDataRequirement, ListPagesDataFetcher, ListItem, ListData, LinkType, LinkLocation, LinkLabel, LexerOptions, Lexer, IncludeFetcher, ImageSource, HeadingLevel, Heading, HeaderType, FloatAlignment, ExtractionResult, Embed, Element6 as Element, DiagnosticSeverity, Diagnostic2 as Diagnostic, DefinitionListItem, DateItem, DataRequirements, DataProvider, DEFAULT_SETTINGS, ContainerType, ContainerData, CompiledTemplate, CollapsibleData, CodeBlockData2 as CodeBlockData, ClearFloat, AttributeMap, AsyncIncludeFetcher, AnchorTarget, Alignment, AlignType };

package/dist/index.js

@@ -591,7 +591,139 @@ var BLOCK_START_TOKENS = [
   "CLEAR_FLOAT_RIGHT"
 ];
 
+// packages/parser/src/parser/rules/utils.ts
+var SAFE_ATTRIBUTES = new Set([
+  "accept",
+  "align",
+  "alt",
+  "autocapitalize",
+  "autoplay",
+  "background",
+  "bgcolor",
+  "border",
+  "buffered",
+  "checked",
+  "cite",
+  "class",
+  "cols",
+  "colspan",
+  "contenteditable",
+  "controls",
+  "coords",
+  "datetime",
+  "decoding",
+  "default",
+  "dir",
+  "dirname",
+  "disabled",
+  "download",
+  "draggable",
+  "for",
+  "form",
+  "headers",
+  "height",
+  "hidden",
+  "high",
+  "href",
+  "hreflang",
+  "id",
+  "inputmode",
+  "ismap",
+  "itemprop",
+  "kind",
+  "label",
+  "lang",
+  "list",
+  "loop",
+  "low",
+  "max",
+  "maxlength",
+  "min",
+  "minlength",
+  "multiple",
+  "muted",
+  "name",
+  "optimum",
+  "pattern",
+  "placeholder",
+  "poster",
+  "preload",
+  "readonly",
+  "required",
+  "reversed",
+  "role",
+  "rows",
+  "rowspan",
+  "scope",
+  "selected",
+  "shape",
+  "size",
+  "sizes",
+  "span",
+  "spellcheck",
+  "src",
+  "srclang",
+  "srcset",
+  "start",
+  "step",
+  "style",
+  "tabindex",
+  "target",
+  "title",
+  "translate",
+  "type",
+  "usemap",
+  "value",
+  "width",
+  "wrap"
+]);
+function filterUnsafeAttributes(attrs) {
+  const result = {};
+  for (const [key, value] of Object.entries(attrs)) {
+    const lower = key.toLowerCase();
+    if (lower.startsWith("on"))
+      continue;
+    if (lower.startsWith("aria-") || lower.startsWith("data-")) {
+      result[key] = value;
+      continue;
+    }
+    if (!SAFE_ATTRIBUTES.has(lower))
+      continue;
+    if (lower === "id") {
+      result[key] = value.startsWith("u-") ? value : `u-${value}`;
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+function parseBlockName(ctx, startPos) {
+  let pos = startPos;
+  let consumed = 0;
+  const token = ctx.tokens[pos];
+  if (!token || token.type !== "TEXT" && token.type !== "IDENTIFIER") {
+    return null;
+  }
+  let name = token.value.toLowerCase();
+  consumed++;
+  pos++;
+  if (ctx.tokens[pos]?.type === "UNDERSCORE") {
+    name += "_";
+    consumed++;
+  }
+  return { name, consumed };
+}
+
 // packages/parser/src/parser/rules/inline/utils.ts
+function isExcludedBlockToken(ctx, tokenPos) {
+  if (!ctx.excludedBlockNames?.size)
+    return false;
+  const token = ctx.tokens[tokenPos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN")
+    return false;
+  const nameResult = parseBlockName(ctx, tokenPos + 1);
+  return nameResult !== null && ctx.excludedBlockNames.has(nameResult.name);
+}
 function canApplyInlineRule(rule, token) {
   if (rule.startTokens.length === 0) {
     return true;
@@ -683,7 +815,8 @@ function parseInlineUntil(ctx, endType) {
           isInvalidHeading = true;
         }
       }
-      const isBlockStart = nextMeaningfulToken && BLOCK_START_TOKENS.includes(nextMeaningfulToken.type) && nextMeaningfulToken.lineStart && !isOrphanCloseSpan && !isAnchorName && !isInvalidBlockOpen && !isInvalidHeading;
+      const isExcludedBlock = (nextMeaningfulToken?.type === "BLOCK_OPEN" || nextMeaningfulToken?.type === "BLOCK_END_OPEN") && isExcludedBlockToken(ctx, pos + lookAhead);
+      const isBlockStart = nextMeaningfulToken && BLOCK_START_TOKENS.includes(nextMeaningfulToken.type) && nextMeaningfulToken.lineStart && !isOrphanCloseSpan && !isAnchorName && !isInvalidBlockOpen && !isInvalidHeading && !isExcludedBlock;
       if (!nextMeaningfulToken || nextMeaningfulToken.type === "NEWLINE" || nextMeaningfulToken.type === "EOF" || isBlockStart) {
         if (isBlockStart && nodes.length > 0) {
           const nextPos = pos + lookAhead;
@@ -1021,129 +1154,6 @@ function buildListData(topLtype, list) {
   };
 }
 
-// packages/parser/src/parser/rules/utils.ts
-var SAFE_ATTRIBUTES = new Set([
-  "accept",
-  "align",
-  "alt",
-  "autocapitalize",
-  "autoplay",
-  "background",
-  "bgcolor",
-  "border",
-  "buffered",
-  "checked",
-  "cite",
-  "class",
-  "cols",
-  "colspan",
-  "contenteditable",
-  "controls",
-  "coords",
-  "datetime",
-  "decoding",
-  "default",
-  "dir",
-  "dirname",
-  "disabled",
-  "download",
-  "draggable",
-  "for",
-  "form",
-  "headers",
-  "height",
-  "hidden",
-  "high",
-  "href",
-  "hreflang",
-  "id",
-  "inputmode",
-  "ismap",
-  "itemprop",
-  "kind",
-  "label",
-  "lang",
-  "list",
-  "loop",
-  "low",
-  "max",
-  "maxlength",
-  "min",
-  "minlength",
-  "multiple",
-  "muted",
-  "name",
-  "optimum",
-  "pattern",
-  "placeholder",
-  "poster",
-  "preload",
-  "readonly",
-  "required",
-  "reversed",
-  "role",
-  "rows",
-  "rowspan",
-  "scope",
-  "selected",
-  "shape",
-  "size",
-  "sizes",
-  "span",
-  "spellcheck",
-  "src",
-  "srclang",
-  "srcset",
-  "start",
-  "step",
-  "style",
-  "tabindex",
-  "target",
-  "title",
-  "translate",
-  "type",
-  "usemap",
-  "value",
-  "width",
-  "wrap"
-]);
-function filterUnsafeAttributes(attrs) {
-  const result = {};
-  for (const [key, value] of Object.entries(attrs)) {
-    const lower = key.toLowerCase();
-    if (lower.startsWith("on"))
-      continue;
-    if (lower.startsWith("aria-") || lower.startsWith("data-")) {
-      result[key] = value;
-      continue;
-    }
-    if (!SAFE_ATTRIBUTES.has(lower))
-      continue;
-    if (lower === "id") {
-      result[key] = value.startsWith("u-") ? value : `u-${value}`;
-      continue;
-    }
-    result[key] = value;
-  }
-  return result;
-}
-function parseBlockName(ctx, startPos) {
-  let pos = startPos;
-  let consumed = 0;
-  const token = ctx.tokens[pos];
-  if (!token || token.type !== "TEXT" && token.type !== "IDENTIFIER") {
-    return null;
-  }
-  let name = token.value.toLowerCase();
-  consumed++;
-  pos++;
-  if (ctx.tokens[pos]?.type === "UNDERSCORE") {
-    name += "_";
-    consumed++;
-  }
-  return { name, consumed };
-}
-
 // packages/parser/src/parser/rules/block/utils.ts
 function canApplyBlockRule(rule, token) {
   if (rule.requiresLineStart && !token.lineStart) {
@@ -1154,11 +1164,13 @@ function canApplyBlockRule(rule, token) {
   }
   return rule.startTokens.includes(token.type);
 }
-function parseBlocksUntil(ctx, closeCondition) {
+function parseBlocksUntil(ctx, closeCondition, options) {
   const elements = [];
   let consumed = 0;
   let pos = ctx.pos;
-  const { blockRules, blockFallbackRule } = ctx;
+  const excluded = options?.excludedBlockNames;
+  const blockRules = excluded ? ctx.blockRules.filter((r) => !excluded.has(r.name)) : ctx.blockRules;
+  const { blockFallbackRule } = ctx;
   while (pos < ctx.tokens.length) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
@@ -1179,7 +1191,13 @@ function parseBlocksUntil(ctx, closeCondition) {
       continue;
     }
     let matched = false;
-    const blockCtx = { ...ctx, pos, blockCloseCondition: closeCondition };
+    const blockCtx = {
+      ...ctx,
+      pos,
+      blockRules,
+      blockCloseCondition: closeCondition,
+      excludedBlockNames: excluded
+    };
     for (const rule of blockRules) {
       if (canApplyBlockRule(rule, token)) {
         const result = rule.parse(blockCtx);
@@ -2564,44 +2582,7 @@ function consumeCloseTag2(ctx, pos) {
     closeConsumed++;
   return closeConsumed;
 }
-function mergeParagraphs(elements) {
-  const result = [];
-  let mergedElements = [];
-  for (const elem of elements) {
-    if (elem.element === "container" && elem.data && typeof elem.data === "object" && "type" in elem.data && elem.data.type === "paragraph") {
-      if (mergedElements.length > 0) {
-        mergedElements.push({ element: "line-break" });
-      }
-      if ("elements" in elem.data && Array.isArray(elem.data.elements)) {
-        mergedElements.push(...elem.data.elements);
-      }
-    } else {
-      if (mergedElements.length > 0) {
-        result.push({
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements: mergedElements
-          }
-        });
-        mergedElements = [];
-      }
-      result.push(elem);
-    }
-  }
-  if (mergedElements.length > 0) {
-    result.push({
-      element: "container",
-      data: {
-        type: "paragraph",
-        attributes: {},
-        elements: mergedElements
-      }
-    });
-  }
-  return result;
-}
+var EXCLUDED_BLOCKS = new Set(["collapsible"]);
 var collapsibleRule = {
   name: "collapsible",
   startTokens: ["BLOCK_OPEN"],
@@ -2665,18 +2646,16 @@ var collapsibleRule = {
         bodyElements = [];
       }
     } else {
-      const bodyCtx = {
-        ...ctx,
-        pos,
-        blockRules: ctx.blockRules.filter((r) => r.name !== "collapsible")
-      };
+      const bodyCtx = { ...ctx, pos };
       const closeCondition = (checkCtx) => {
         return isCollapsibleClose(checkCtx, checkCtx.pos);
       };
-      const bodyResult = parseBlocksUntil(bodyCtx, closeCondition);
+      const bodyResult = parseBlocksUntil(bodyCtx, closeCondition, {
+        excludedBlockNames: EXCLUDED_BLOCKS
+      });
       consumed += bodyResult.consumed;
       pos += bodyResult.consumed;
-      bodyElements = mergeParagraphs(bodyResult.elements);
+      bodyElements = bodyResult.elements;
     }
     if (!isCollapsibleClose(ctx, pos)) {
       ctx.diagnostics.push({
@@ -9934,7 +9913,7 @@ function resolveIncludes(source, fetcher, options) {
   if (options?.settings && !options.settings.enablePageSyntax) {
     return source;
   }
-  const maxDepth = options?.maxDepth ?? 5;
+  const maxIterations = options?.maxIterations ?? 10;
   const cache = new Map;
   const cachedFetcher = (pageRef) => {
     const key = normalizePageKey(pageRef);
@@ -9950,7 +9929,29 @@ function resolveIncludes(source, fetcher, options) {
     cache.set(key, result);
     return result;
   };
-  return expandText(source, cachedFetcher, 0, maxDepth, []);
+  return expandIterative(source, cachedFetcher, maxIterations);
+}
+async function resolveIncludesAsync(source, fetcher, options) {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return source;
+  }
+  const maxIterations = options?.maxIterations ?? 10;
+  const cache = new Map;
+  const cachedFetcher = async (pageRef) => {
+    const key = normalizePageKey(pageRef);
+    if (cache.has(key)) {
+      return cache.get(key);
+    }
+    let result;
+    try {
+      result = await fetcher(pageRef);
+    } catch {
+      result = null;
+    }
+    cache.set(key, result);
+    return result;
+  };
+  return expandIterativeAsync(source, cachedFetcher, maxIterations);
 }
 var INCLUDE_PATTERN = /^\[\[include\s([^\]]*(?:\](?!\])[^\]]*)*)\]\]/gim;
 function parseIncludeDirective(inner) {
@@ -10000,26 +10001,55 @@ function parseIncludeDirective(inner) {
   }
   return { location, variables };
 }
-function expandText(source, fetcher, depth, maxDepth, trace) {
-  if (depth >= maxDepth)
-    return source;
-  return source.replace(INCLUDE_PATTERN, (_match, inner) => {
-    const { location, variables } = parseIncludeDirective(inner);
-    const pageKey = normalizePageKey(location);
-    if (trace.includes(pageKey)) {
-      return `[[div class="error-block"]]
-Circular include detected: "${location.page}"
+function replaceOneInclude(_match, inner, fetcher) {
+  const { location, variables } = parseIncludeDirective(inner);
+  const content = fetcher(location);
+  if (content === null) {
+    return `[[div class="error-block"]]
+Page to be included "${location.page}" cannot be found!
 [[/div]]`;
-    }
-    const content = fetcher(location);
-    if (content === null) {
-      return `[[div class="error-block"]]
+  }
+  return substituteVariables(content, variables);
+}
+function expandIterative(source, fetcher, maxIterations) {
+  let current = source;
+  for (let i = 0;i < maxIterations; i++) {
+    const previous = current;
+    current = current.replace(INCLUDE_PATTERN, (_match, inner) => replaceOneInclude(_match, inner, fetcher));
+    if (current === previous)
+      break;
+  }
+  return current;
+}
+async function expandIterativeAsync(source, fetcher, maxIterations) {
+  let current = source;
+  for (let i = 0;i < maxIterations; i++) {
+    const previous = current;
+    const pattern = new RegExp(INCLUDE_PATTERN.source, INCLUDE_PATTERN.flags);
+    let result = "";
+    let lastPos = 0;
+    let match;
+    while ((match = pattern.exec(current)) !== null) {
+      const fullMatch = match[0];
+      const inner = match[1];
+      result += current.slice(lastPos, match.index);
+      const { location, variables } = parseIncludeDirective(inner);
+      const content = await fetcher(location);
+      if (content === null) {
+        result += `[[div class="error-block"]]
 Page to be included "${location.page}" cannot be found!
 [[/div]]`;
+      } else {
+        result += substituteVariables(content, variables);
+      }
+      lastPos = match.index + fullMatch.length;
     }
-    const substituted = substituteVariables(content, variables);
-    return expandText(substituted, fetcher, depth + 1, maxDepth, [...trace, pageKey]);
-  });
+    result += current.slice(lastPos);
+    current = result;
+    if (current === previous)
+      break;
+  }
+  return current;
 }
 function normalizePageKey(location) {
   const site = location.site ?? "";
@@ -10248,6 +10278,7 @@ export {
   text,
   resolveModules,
   resolveListUsers,
+  resolveIncludesAsync,
   resolveIncludes,
   parseTags,
   parseParent,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "2.0.9",
+  "version": "2.1.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",