@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/block/paragraph/span-markers.ts

@@ -0,0 +1,52 @@
+import type { Element } from "@wdprlib/ast";
+
+/**
+ * Process closeSpan markers in inline content.
+ *
+ * When a `_closeSpan` marker is found, all preceding content is wrapped
+ * in a span and the marker itself is omitted.
+ */
+export function processCloseSpanMarkers(elements: Element[]): Element[] {
+  let result: Element[] | null = null;
+
+  for (let i = 0; i < elements.length; i++) {
+    const elem = elements[i];
+    if (!elem) continue;
+
+    if (isCloseSpanMarker(elem)) {
+      if (result === null) {
+        result = elements.slice(0, i);
+      }
+      if (result.length > 0) {
+        const spanContent = [...result];
+        result.length = 0;
+        result.push({
+          element: "container",
+          data: {
+            type: "span",
+            attributes: {},
+            elements: spanContent,
+          },
+        });
+      }
+    } else {
+      result?.push(elem);
+    }
+  }
+
+  return result ?? elements;
+}
+
+function isCloseSpanMarker(elem: Element): boolean {
+  return (
+    elem.element === "container" &&
+    elem.data &&
+    typeof elem.data === "object" &&
+    "type" in elem.data &&
+    elem.data.type === "span" &&
+    "attributes" in elem.data &&
+    typeof elem.data.attributes === "object" &&
+    elem.data.attributes !== null &&
+    "_closeSpan" in elem.data.attributes
+  );
+}

package/src/parser/rules/block/parsing/attributes/index.ts

@@ -0,0 +1,32 @@
+import type { ParseContext } from "../../../types";
+import { filterUnsafeAttributes } from "../../../common";
+import { scanAttributes, type AttributeParseResult } from "./scanner";
+
+/**
+ * Parses HTML-style attributes from block opening tags.
+ *
+ * Supports quoted values, single-token unquoted values, boolean attributes,
+ * and hyphenated names like `data-paragraph` or `aria-label`.
+ *
+ * Attribute names are lowercased. The result is filtered through
+ * `filterUnsafeAttributes` to strip potentially dangerous HTML attributes.
+ */
+export function parseAttributes(ctx: ParseContext, startPos: number): AttributeParseResult {
+  const result = scanAttributes(ctx, startPos, { hyphenatedNames: true, strikeHyphens: false });
+  return { attrs: filterUnsafeAttributes(result.attrs), consumed: result.consumed };
+}
+
+/**
+ * Parses attributes from block opening tags without safety filtering.
+ *
+ * Use this for block-specific parameters that are not emitted as HTML
+ * attributes. Hyphenated name handling is configurable for legacy parser
+ * contexts that need the first segment only.
+ */
+export function parseAttributesRaw(
+  ctx: ParseContext,
+  startPos: number,
+  hyphenatedNames = true,
+): AttributeParseResult {
+  return scanAttributes(ctx, startPos, { hyphenatedNames, strikeHyphens: true });
+}

package/src/parser/rules/block/parsing/attributes/names.ts

@@ -0,0 +1,93 @@
+import type { Token } from "../../../../../lexer";
+import type { ParseContext } from "../../../types";
+
+export interface AttributeNameOptions {
+  hyphenatedNames: boolean;
+  strikeHyphens: boolean;
+}
+
+export interface AttributeNameResult {
+  name: string;
+  pos: number;
+  consumed: number;
+}
+
+export function consumeAttributeName(
+  ctx: ParseContext,
+  startPos: number,
+  startConsumed: number,
+  startName: string,
+  options: AttributeNameOptions,
+): AttributeNameResult {
+  return options.strikeHyphens
+    ? consumeRawNameSuffix(ctx, startPos, startConsumed, startName, options.hyphenatedNames)
+    : consumeSafeNameSuffix(ctx, startPos, startConsumed, startName);
+}
+
+export function isAttributeNameToken(token: Token | undefined): token is Token {
+  return token?.type === "TEXT" || token?.type === "IDENTIFIER";
+}
+
+function consumeSafeNameSuffix(
+  ctx: ParseContext,
+  startPos: number,
+  startConsumed: number,
+  startName: string,
+): AttributeNameResult {
+  let name = startName;
+  let pos = startPos;
+  let consumed = startConsumed;
+
+  while (
+    ctx.tokens[pos]?.type === "TEXT" &&
+    ctx.tokens[pos]?.value === "-" &&
+    isAttributeNameToken(ctx.tokens[pos + 1])
+  ) {
+    name += "-";
+    pos++;
+    consumed++;
+    name += ctx.tokens[pos]?.value ?? "";
+    pos++;
+    consumed++;
+  }
+
+  return { name, pos, consumed };
+}
+
+function consumeRawNameSuffix(
+  ctx: ParseContext,
+  startPos: number,
+  startConsumed: number,
+  startName: string,
+  hyphenatedNames: boolean,
+): AttributeNameResult {
+  let name = startName;
+  let pos = startPos;
+  let consumed = startConsumed;
+
+  while (isHyphenToken(ctx.tokens[pos])) {
+    while (isHyphenToken(ctx.tokens[pos])) {
+      if (hyphenatedNames) {
+        name += ctx.tokens[pos]?.value ?? "-";
+      }
+      pos++;
+      consumed++;
+    }
+
+    if (!isAttributeNameToken(ctx.tokens[pos])) {
+      break;
+    }
+
+    if (hyphenatedNames) {
+      name += ctx.tokens[pos]?.value ?? "";
+    }
+    pos++;
+    consumed++;
+  }
+
+  return { name, pos, consumed };
+}
+
+function isHyphenToken(token: Token | undefined): boolean {
+  return (token?.type === "TEXT" && token.value === "-") || token?.type === "STRIKE_MARKER";
+}

package/src/parser/rules/block/parsing/attributes/scanner.ts

@@ -0,0 +1,75 @@
+import type { Token } from "../../../../../lexer";
+import type { ParseContext } from "../../../types";
+import { consumeAttributeName, isAttributeNameToken } from "./names";
+import { consumeAttributeValue } from "./values";
+
+export interface AttributeParseResult {
+  attrs: Record<string, string>;
+  consumed: number;
+}
+
+interface AttributeScanOptions {
+  hyphenatedNames: boolean;
+  strikeHyphens: boolean;
+}
+
+export function scanAttributes(
+  ctx: ParseContext,
+  startPos: number,
+  options: AttributeScanOptions,
+): AttributeParseResult {
+  const attrs: Record<string, string> = {};
+  let pos = startPos;
+  let consumed = 0;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || isAttributeTerminator(token)) {
+      break;
+    }
+
+    if (token.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    if (!isAttributeNameToken(token)) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    let name = token.value;
+    pos++;
+    consumed++;
+
+    const nameResult = consumeAttributeName(ctx, pos, consumed, name, options);
+    pos = nameResult.pos;
+    consumed = nameResult.consumed;
+    name = nameResult.name.toLowerCase();
+
+    if (ctx.tokens[pos]?.type !== "EQUALS") {
+      attrs[name] = "true";
+      continue;
+    }
+
+    pos++;
+    consumed++;
+
+    const valueResult = consumeAttributeValue(ctx.tokens[pos]);
+    if (!valueResult) {
+      continue;
+    }
+
+    attrs[name] = valueResult.value;
+    pos++;
+    consumed++;
+  }
+
+  return { attrs, consumed };
+}
+
+function isAttributeTerminator(token: Token): boolean {
+  return token.type === "BLOCK_CLOSE" || token.type === "NEWLINE" || token.type === "EOF";
+}

package/src/parser/rules/block/parsing/attributes/values.ts

@@ -0,0 +1,26 @@
+import type { Token } from "../../../../../lexer";
+import { isAttributeNameToken } from "./names";
+
+export interface AttributeValueResult {
+  value: string;
+}
+
+export function consumeAttributeValue(token: Token | undefined): AttributeValueResult | null {
+  if (!token) {
+    return null;
+  }
+
+  if (token.type === "QUOTED_STRING") {
+    return { value: stripQuotes(token.value) };
+  }
+
+  if (isAttributeNameToken(token)) {
+    return { value: token.value };
+  }
+
+  return null;
+}
+
+function stripQuotes(value: string): string {
+  return value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value;
+}

package/src/parser/rules/block/parsing/block-item.ts

@@ -0,0 +1,29 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { getCandidateBlockRules } from "./rule-dispatch";
+
+export interface BlockItemResult {
+  elements: Element[];
+  consumed: number;
+}
+
+export function parseBlockItem(ctx: ParseContext): BlockItemResult {
+  const token = ctx.tokens[ctx.pos];
+  if (!token) {
+    return { elements: [], consumed: 0 };
+  }
+
+  for (const rule of getCandidateBlockRules(ctx.blockRules, token)) {
+    const result = rule.parse(ctx);
+    if (result.success) {
+      return { elements: result.elements, consumed: result.consumed };
+    }
+  }
+
+  const fallback = ctx.blockFallbackRule.parse(ctx);
+  if (fallback.success && fallback.elements.length > 0) {
+    return { elements: fallback.elements, consumed: fallback.consumed };
+  }
+
+  return { elements: [], consumed: 1 };
+}

package/src/parser/rules/block/parsing/content.ts

@@ -0,0 +1,127 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseBlockItem } from "./block-item";
+
+export interface BlockParseResult {
+  /** The parsed AST elements. */
+  elements: Element[];
+  /** Total number of tokens consumed from the stream. */
+  consumed: number;
+}
+
+const excludedBlockRulesCache = new WeakMap<
+  ParseContext["blockRules"],
+  WeakMap<ReadonlySet<string>, ParseContext["blockRules"]>
+>();
+
+/**
+ * Parses block-level elements from the token stream until a close
+ * condition is satisfied.
+ *
+ * This is the workhorse parser used by container blocks (div, collapsible,
+ * tabview, iftags, align, etc.) to parse their body content. It loops
+ * through tokens, trying each block rule in priority order, and falls back
+ * to the paragraph rule when nothing else matches.
+ *
+ * Whitespace and newline tokens between blocks are silently consumed.
+ * The close condition receives a ParseContext snapshot at the current
+ * position and should return `true` to stop parsing (the close tag
+ * itself is NOT consumed here -- the caller handles that).
+ *
+ * The close condition is also injected into `blockCloseCondition` on
+ * the context so that the paragraph parser can respect the enclosing
+ * block's boundary.
+ *
+ * @param ctx            - Parse context positioned at the start of the body.
+ * @param closeCondition - Predicate that signals the end of the block body.
+ * @param options        - Optional settings.
+ * @param options.excludedBlockNames - Block names that should be excluded
+ *   from both rule dispatch and paragraph-boundary detection. The named
+ *   rules are filtered out of `blockRules`, and the set is propagated to
+ *   the inline parser via `ParseContext.excludedBlockNames` so that
+ *   `BLOCK_OPEN` / `BLOCK_END_OPEN` tokens for these names do not trigger
+ *   paragraph breaks.
+ * @returns Parsed elements and total tokens consumed.
+ */
+export function parseBlocksUntil(
+  ctx: ParseContext,
+  closeCondition: (ctx: ParseContext) => boolean,
+  options?: { excludedBlockNames?: ReadonlySet<string> },
+): BlockParseResult {
+  const elements: Element[] = [];
+  let consumed = 0;
+  let pos = ctx.pos;
+
+  const excluded = options?.excludedBlockNames;
+  const blockRules = excluded ? getExcludedBlockRules(ctx.blockRules, excluded) : ctx.blockRules;
+  const blockScope = {
+    ...ctx.scope,
+    blockCloseCondition: closeCondition,
+    excludedBlockNames: excluded,
+  };
+  const checkCtx: ParseContext = { ...ctx, pos };
+  const blockCtx: ParseContext = {
+    ...ctx,
+    pos,
+    blockRules,
+    scope: blockScope,
+  };
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    // Check close condition
+    checkCtx.pos = pos;
+    if (closeCondition(checkCtx)) {
+      break;
+    }
+
+    // Skip whitespace
+    if (token.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    // Skip newlines
+    if (token.type === "NEWLINE") {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    // Pass close condition and excluded names to context
+    blockCtx.pos = pos;
+    blockCtx.scope = blockScope;
+
+    const result = parseBlockItem(blockCtx);
+    elements.push(...result.elements);
+    consumed += result.consumed;
+    pos += result.consumed;
+  }
+
+  return { elements, consumed };
+}
+
+function getExcludedBlockRules(
+  blockRules: ParseContext["blockRules"],
+  excluded: ReadonlySet<string>,
+): ParseContext["blockRules"] {
+  let byExcluded = excludedBlockRulesCache.get(blockRules);
+  if (!byExcluded) {
+    byExcluded = new WeakMap();
+    excludedBlockRulesCache.set(blockRules, byExcluded);
+  }
+
+  const cached = byExcluded.get(excluded);
+  if (cached) {
+    return cached;
+  }
+
+  const filtered = blockRules.filter((rule) => !excluded.has(rule.name));
+  byExcluded.set(excluded, filtered);
+  return filtered;
+}

package/src/parser/rules/block/parsing/end-condition.ts

@@ -0,0 +1,51 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../utils";
+
+/**
+ * Creates a reusable close-condition function that matches block end tags
+ * (`[[/name]]`) for one or more block names.
+ *
+ * The returned function inspects the tokens at `ctx.pos` and returns both
+ * whether a match was found and how many tokens the closing tag occupies
+ * (including the optional trailing NEWLINE).
+ *
+ * @param blockNames - Array of block names to match (e.g. `["div"]`).
+ * @returns A function suitable for use as a `closeCondition` argument,
+ *          returning `{ matched, consumed }`.
+ */
+export function createBlockEndCondition(
+  blockNames: string[],
+): (ctx: ParseContext) => { matched: boolean; consumed: number } {
+  return (ctx: ParseContext) => {
+    const token = ctx.tokens[ctx.pos];
+    if (token?.type !== "BLOCK_END_OPEN") {
+      return { matched: false, consumed: 0 };
+    }
+
+    const nameResult = parseBlockName(ctx, ctx.pos + 1);
+    if (!nameResult) {
+      return { matched: false, consumed: 0 };
+    }
+
+    if (!blockNames.includes(nameResult.name)) {
+      return { matched: false, consumed: 0 };
+    }
+
+    // Calculate consumed: [[/ + name + ]]
+    let consumed = 1 + nameResult.consumed;
+
+    // Check for closing ]]
+    const closePos = ctx.pos + 1 + nameResult.consumed;
+    if (ctx.tokens[closePos]?.type === "BLOCK_CLOSE") {
+      consumed++;
+    }
+
+    // Check for trailing newline
+    const newlinePos = closePos + 1;
+    if (ctx.tokens[newlinePos]?.type === "NEWLINE") {
+      consumed++;
+    }
+
+    return { matched: true, consumed };
+  };
+}

package/src/parser/rules/block/parsing/inline-content.ts

@@ -0,0 +1,105 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { getCandidateInlineRules } from "../../inline/utils";
+import type { BlockParseResult } from "./content";
+import { consumeInlineContentNewlines, removeTrailingLineBreaks } from "./inline-newline";
+import { getCandidateBlockRules } from "./rule-dispatch";
+
+/**
+ * Parses mixed inline/block content until a close condition is met,
+ * WITHOUT paragraph wrapping.
+ *
+ * This is used for `div_` (paragraph strip mode) where newlines become
+ * `<br />` elements rather than paragraph separators. Blank lines
+ * (multiple consecutive newlines) are collapsed into a single `<br />`.
+ *
+ * Block-level elements (nested div, collapsible, etc.) are mixed directly
+ * into the inline element stream. Newlines immediately before a BLOCK_OPEN
+ * or BLOCK_END_OPEN are silently consumed (no `<br />` generated).
+ *
+ * Trailing line-break elements are stripped from the result.
+ *
+ * @param ctx            - Parse context positioned at the start of the body.
+ * @param closeCondition - Predicate that signals the end of the content.
+ * @returns Parsed elements and total tokens consumed.
+ */
+export function parseInlineContentUntil(
+  ctx: ParseContext,
+  closeCondition: (ctx: ParseContext) => boolean,
+): BlockParseResult {
+  const elements: Element[] = [];
+  let consumed = 0;
+  let pos = ctx.pos;
+
+  const { blockRules, inlineRules } = ctx;
+  const checkCtx: ParseContext = { ...ctx, pos };
+  const blockCtx: ParseContext = { ...ctx, pos };
+  const inlineCtx: ParseContext = { ...ctx, pos };
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    checkCtx.pos = pos;
+    if (closeCondition(checkCtx)) {
+      break;
+    }
+
+    if (token.type === "WHITESPACE" && token.lineStart) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    if (token.type === "NEWLINE") {
+      const newlineResult = consumeInlineContentNewlines(ctx, pos);
+      consumed += newlineResult.consumed;
+      pos += newlineResult.consumed;
+      if (newlineResult.addLineBreak) {
+        elements.push({ element: "line-break" });
+      }
+      continue;
+    }
+
+    let matched = false;
+    blockCtx.pos = pos;
+
+    for (const rule of getCandidateBlockRules(blockRules, token)) {
+      const result = rule.parse(blockCtx);
+      if (result.success) {
+        elements.push(...result.elements);
+        consumed += result.consumed;
+        pos += result.consumed;
+        matched = true;
+        break;
+      }
+    }
+
+    if (matched) continue;
+
+    inlineCtx.pos = pos;
+
+    for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
+      const result = rule.parse(inlineCtx);
+      if (result.success) {
+        elements.push(...result.elements);
+        consumed += result.consumed;
+        pos += result.consumed;
+        matched = true;
+        break;
+      }
+    }
+
+    if (!matched) {
+      elements.push({ element: "text", data: token.value });
+      consumed++;
+      pos++;
+    }
+  }
+
+  removeTrailingLineBreaks(elements);
+
+  return { elements, consumed };
+}

package/src/parser/rules/block/parsing/inline-newline.ts

@@ -0,0 +1,41 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { isNonBoundaryBlockToken } from "./non-boundary";
+
+export interface InlineNewlineResult {
+  consumed: number;
+  addLineBreak: boolean;
+}
+
+export function consumeInlineContentNewlines(
+  ctx: ParseContext,
+  startPos: number,
+): InlineNewlineResult {
+  let pos = startPos + 1;
+  let consumed = 1;
+
+  while (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  const nextToken = ctx.tokens[pos];
+  if (!nextToken || nextToken.type === "EOF") {
+    return { consumed, addLineBreak: false };
+  }
+
+  if (nextToken.type === "BLOCK_OPEN" || nextToken.type === "BLOCK_END_OPEN") {
+    const peekCtx: ParseContext = { ...ctx, pos };
+    if (!isNonBoundaryBlockToken(peekCtx, pos)) {
+      return { consumed, addLineBreak: false };
+    }
+  }
+
+  return { consumed, addLineBreak: true };
+}
+
+export function removeTrailingLineBreaks(elements: Element[]): void {
+  while (elements.length > 0 && elements[elements.length - 1]?.element === "line-break") {
+    elements.pop();
+  }
+}

package/src/parser/rules/block/parsing/non-boundary.ts

@@ -0,0 +1,24 @@
+import type { ParseContext } from "../../types";
+import { KNOWN_BLOCK_NAMES } from "../../../constants";
+import { parseBlockName } from "../utils";
+
+/**
+ * Whether the BLOCK_OPEN / BLOCK_END_OPEN token at `pos` opens a block name
+ * that should not end the surrounding paragraph / inline run.
+ */
+export function isNonBoundaryBlockToken(ctx: ParseContext, pos: number): boolean {
+  const token = ctx.tokens[pos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") {
+    return false;
+  }
+
+  const nameResult = parseBlockName(ctx, pos + 1);
+  if (nameResult === null) {
+    return ctx.tokens[pos + 1]?.type !== "EQUALS";
+  }
+
+  if (ctx.scope.excludedBlockNames?.has(nameResult.name)) {
+    return true;
+  }
+  return !KNOWN_BLOCK_NAMES.has(nameResult.name);
+}