@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/paragraph/index.ts

@@ -0,0 +1,60 @@
+/**
+ * Paragraph rule
+ *
+ * Collects inline content until paragraph break (double newline) or end of input.
+ * Line breaks within paragraphs are handled by the newlineLineBreakRule.
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { parseInlineContent } from "./content";
+import { normalizeParagraphElements } from "./normalize";
+
+/**
+ * Paragraph is the fallback block rule.
+ *
+ * Wikidot behavior:
+ * - Single newline -> <br> (handled by newlineLineBreakRule)
+ * - Blank line (double newline) -> new paragraph
+ */
+export const paragraphRule: BlockRule = {
+  name: "paragraph",
+  startTokens: [],
+  requiresLineStart: false,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const result = parseInlineContent(ctx);
+    if (result.elements.length === 0) {
+      return { success: false };
+    }
+
+    const elements = normalizeParagraphElements(result.elements);
+    if (elements.length === 0) {
+      return { success: false };
+    }
+
+    const nextPos = ctx.pos + result.consumed;
+    const nextToken = ctx.tokens[nextPos];
+    if (nextToken?.type === "COLON" && nextToken.lineStart) {
+      return {
+        success: true,
+        elements: [...elements, { element: "line-break" }],
+        consumed: result.consumed,
+      };
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "paragraph",
+            attributes: {},
+            elements,
+          },
+        },
+      ],
+      consumed: result.consumed,
+    };
+  },
+};

package/src/parser/rules/block/paragraph/normalize.ts

@@ -0,0 +1,52 @@
+import type { Element } from "@wdprlib/ast";
+import { processCloseSpanMarkers } from "./span-markers";
+
+type PreservedLineBreak = Extract<Element, { element: "line-break" }> & {
+  _preservedTrailingBreak?: boolean;
+};
+
+export function normalizeParagraphElements(source: Element[]): Element[] {
+  let elements = processCloseSpanMarkers(source);
+
+  removeTrailingLineBreaks(elements);
+  removeTrailingWhitespaceText(elements);
+  elements = removeLeadingLineBreaks(elements);
+
+  return elements;
+}
+
+function removeTrailingLineBreaks(elements: Element[]): void {
+  while (elements.length > 0 && elements[elements.length - 1]?.element === "line-break") {
+    const lastEl = elements[elements.length - 1] as PreservedLineBreak;
+    if (lastEl._preservedTrailingBreak) {
+      delete lastEl._preservedTrailingBreak;
+      break;
+    }
+    elements.pop();
+  }
+}
+
+function removeTrailingWhitespaceText(elements: Element[]): void {
+  while (elements.length > 0) {
+    const last = elements[elements.length - 1];
+    if (
+      last?.element === "text" &&
+      "data" in last &&
+      typeof last.data === "string" &&
+      last.data.trim() === ""
+    ) {
+      elements.pop();
+    } else {
+      break;
+    }
+  }
+}
+
+function removeLeadingLineBreaks(elements: Element[]): Element[] {
+  let first = 0;
+  while (first < elements.length && elements[first]?.element === "line-break") {
+    first++;
+  }
+
+  return first > 0 ? elements.slice(first) : elements;
+}

package/src/parser/rules/block/paragraph/span-markers.ts

@@ -0,0 +1,52 @@
+import type { Element } from "@wdprlib/ast";
+
+/**
+ * Process closeSpan markers in inline content.
+ *
+ * When a `_closeSpan` marker is found, all preceding content is wrapped
+ * in a span and the marker itself is omitted.
+ */
+export function processCloseSpanMarkers(elements: Element[]): Element[] {
+  let result: Element[] | null = null;
+
+  for (let i = 0; i < elements.length; i++) {
+    const elem = elements[i];
+    if (!elem) continue;
+
+    if (isCloseSpanMarker(elem)) {
+      if (result === null) {
+        result = elements.slice(0, i);
+      }
+      if (result.length > 0) {
+        const spanContent = [...result];
+        result.length = 0;
+        result.push({
+          element: "container",
+          data: {
+            type: "span",
+            attributes: {},
+            elements: spanContent,
+          },
+        });
+      }
+    } else {
+      result?.push(elem);
+    }
+  }
+
+  return result ?? elements;
+}
+
+function isCloseSpanMarker(elem: Element): boolean {
+  return (
+    elem.element === "container" &&
+    elem.data &&
+    typeof elem.data === "object" &&
+    "type" in elem.data &&
+    elem.data.type === "span" &&
+    "attributes" in elem.data &&
+    typeof elem.data.attributes === "object" &&
+    elem.data.attributes !== null &&
+    "_closeSpan" in elem.data.attributes
+  );
+}

package/src/parser/rules/block/parsing/attributes/index.ts

@@ -0,0 +1,32 @@
+import type { ParseContext } from "../../../types";
+import { filterUnsafeAttributes } from "../../../common";
+import { scanAttributes, type AttributeParseResult } from "./scanner";
+
+/**
+ * Parses HTML-style attributes from block opening tags.
+ *
+ * Supports quoted values, single-token unquoted values, boolean attributes,
+ * and hyphenated names like `data-paragraph` or `aria-label`.
+ *
+ * Attribute names are lowercased. The result is filtered through
+ * `filterUnsafeAttributes` to strip potentially dangerous HTML attributes.
+ */
+export function parseAttributes(ctx: ParseContext, startPos: number): AttributeParseResult {
+  const result = scanAttributes(ctx, startPos, { hyphenatedNames: true, strikeHyphens: false });
+  return { attrs: filterUnsafeAttributes(result.attrs), consumed: result.consumed };
+}
+
+/**
+ * Parses attributes from block opening tags without safety filtering.
+ *
+ * Use this for block-specific parameters that are not emitted as HTML
+ * attributes. Hyphenated name handling is configurable for legacy parser
+ * contexts that need the first segment only.
+ */
+export function parseAttributesRaw(
+  ctx: ParseContext,
+  startPos: number,
+  hyphenatedNames = true,
+): AttributeParseResult {
+  return scanAttributes(ctx, startPos, { hyphenatedNames, strikeHyphens: true });
+}

package/src/parser/rules/block/parsing/attributes/names.ts

@@ -0,0 +1,93 @@
+import type { Token } from "../../../../../lexer";
+import type { ParseContext } from "../../../types";
+
+export interface AttributeNameOptions {
+  hyphenatedNames: boolean;
+  strikeHyphens: boolean;
+}
+
+export interface AttributeNameResult {
+  name: string;
+  pos: number;
+  consumed: number;
+}
+
+export function consumeAttributeName(
+  ctx: ParseContext,
+  startPos: number,
+  startConsumed: number,
+  startName: string,
+  options: AttributeNameOptions,
+): AttributeNameResult {
+  return options.strikeHyphens
+    ? consumeRawNameSuffix(ctx, startPos, startConsumed, startName, options.hyphenatedNames)
+    : consumeSafeNameSuffix(ctx, startPos, startConsumed, startName);
+}
+
+export function isAttributeNameToken(token: Token | undefined): token is Token {
+  return token?.type === "TEXT" || token?.type === "IDENTIFIER";
+}
+
+function consumeSafeNameSuffix(
+  ctx: ParseContext,
+  startPos: number,
+  startConsumed: number,
+  startName: string,
+): AttributeNameResult {
+  let name = startName;
+  let pos = startPos;
+  let consumed = startConsumed;
+
+  while (
+    ctx.tokens[pos]?.type === "TEXT" &&
+    ctx.tokens[pos]?.value === "-" &&
+    isAttributeNameToken(ctx.tokens[pos + 1])
+  ) {
+    name += "-";
+    pos++;
+    consumed++;
+    name += ctx.tokens[pos]?.value ?? "";
+    pos++;
+    consumed++;
+  }
+
+  return { name, pos, consumed };
+}
+
+function consumeRawNameSuffix(
+  ctx: ParseContext,
+  startPos: number,
+  startConsumed: number,
+  startName: string,
+  hyphenatedNames: boolean,
+): AttributeNameResult {
+  let name = startName;
+  let pos = startPos;
+  let consumed = startConsumed;
+
+  while (isHyphenToken(ctx.tokens[pos])) {
+    while (isHyphenToken(ctx.tokens[pos])) {
+      if (hyphenatedNames) {
+        name += ctx.tokens[pos]?.value ?? "-";
+      }
+      pos++;
+      consumed++;
+    }
+
+    if (!isAttributeNameToken(ctx.tokens[pos])) {
+      break;
+    }
+
+    if (hyphenatedNames) {
+      name += ctx.tokens[pos]?.value ?? "";
+    }
+    pos++;
+    consumed++;
+  }
+
+  return { name, pos, consumed };
+}
+
+function isHyphenToken(token: Token | undefined): boolean {
+  return (token?.type === "TEXT" && token.value === "-") || token?.type === "STRIKE_MARKER";
+}

package/src/parser/rules/block/parsing/attributes/scanner.ts

@@ -0,0 +1,75 @@
+import type { Token } from "../../../../../lexer";
+import type { ParseContext } from "../../../types";
+import { consumeAttributeName, isAttributeNameToken } from "./names";
+import { consumeAttributeValue } from "./values";
+
+export interface AttributeParseResult {
+  attrs: Record<string, string>;
+  consumed: number;
+}
+
+interface AttributeScanOptions {
+  hyphenatedNames: boolean;
+  strikeHyphens: boolean;
+}
+
+export function scanAttributes(
+  ctx: ParseContext,
+  startPos: number,
+  options: AttributeScanOptions,
+): AttributeParseResult {
+  const attrs: Record<string, string> = {};
+  let pos = startPos;
+  let consumed = 0;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || isAttributeTerminator(token)) {
+      break;
+    }
+
+    if (token.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    if (!isAttributeNameToken(token)) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    let name = token.value;
+    pos++;
+    consumed++;
+
+    const nameResult = consumeAttributeName(ctx, pos, consumed, name, options);
+    pos = nameResult.pos;
+    consumed = nameResult.consumed;
+    name = nameResult.name.toLowerCase();
+
+    if (ctx.tokens[pos]?.type !== "EQUALS") {
+      attrs[name] = "true";
+      continue;
+    }
+
+    pos++;
+    consumed++;
+
+    const valueResult = consumeAttributeValue(ctx.tokens[pos]);
+    if (!valueResult) {
+      continue;
+    }
+
+    attrs[name] = valueResult.value;
+    pos++;
+    consumed++;
+  }
+
+  return { attrs, consumed };
+}
+
+function isAttributeTerminator(token: Token): boolean {
+  return token.type === "BLOCK_CLOSE" || token.type === "NEWLINE" || token.type === "EOF";
+}

package/src/parser/rules/block/parsing/attributes/values.ts

@@ -0,0 +1,26 @@
+import type { Token } from "../../../../../lexer";
+import { isAttributeNameToken } from "./names";
+
+export interface AttributeValueResult {
+  value: string;
+}
+
+export function consumeAttributeValue(token: Token | undefined): AttributeValueResult | null {
+  if (!token) {
+    return null;
+  }
+
+  if (token.type === "QUOTED_STRING") {
+    return { value: stripQuotes(token.value) };
+  }
+
+  if (isAttributeNameToken(token)) {
+    return { value: token.value };
+  }
+
+  return null;
+}
+
+function stripQuotes(value: string): string {
+  return value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value;
+}

package/src/parser/rules/block/parsing/block-item.ts

@@ -0,0 +1,29 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { getCandidateBlockRules } from "./rule-dispatch";
+
+export interface BlockItemResult {
+  elements: Element[];
+  consumed: number;
+}
+
+export function parseBlockItem(ctx: ParseContext): BlockItemResult {
+  const token = ctx.tokens[ctx.pos];
+  if (!token) {
+    return { elements: [], consumed: 0 };
+  }
+
+  for (const rule of getCandidateBlockRules(ctx.blockRules, token)) {
+    const result = rule.parse(ctx);
+    if (result.success) {
+      return { elements: result.elements, consumed: result.consumed };
+    }
+  }
+
+  const fallback = ctx.blockFallbackRule.parse(ctx);
+  if (fallback.success && fallback.elements.length > 0) {
+    return { elements: fallback.elements, consumed: fallback.consumed };
+  }
+
+  return { elements: [], consumed: 1 };
+}

package/src/parser/rules/block/parsing/content.ts

@@ -0,0 +1,127 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseBlockItem } from "./block-item";
+
+export interface BlockParseResult {
+  /** The parsed AST elements. */
+  elements: Element[];
+  /** Total number of tokens consumed from the stream. */
+  consumed: number;
+}
+
+const excludedBlockRulesCache = new WeakMap<
+  ParseContext["blockRules"],
+  WeakMap<ReadonlySet<string>, ParseContext["blockRules"]>
+>();
+
+/**
+ * Parses block-level elements from the token stream until a close
+ * condition is satisfied.
+ *
+ * This is the workhorse parser used by container blocks (div, collapsible,
+ * tabview, iftags, align, etc.) to parse their body content. It loops
+ * through tokens, trying each block rule in priority order, and falls back
+ * to the paragraph rule when nothing else matches.
+ *
+ * Whitespace and newline tokens between blocks are silently consumed.
+ * The close condition receives a ParseContext snapshot at the current
+ * position and should return `true` to stop parsing (the close tag
+ * itself is NOT consumed here -- the caller handles that).
+ *
+ * The close condition is also injected into `blockCloseCondition` on
+ * the context so that the paragraph parser can respect the enclosing
+ * block's boundary.
+ *
+ * @param ctx            - Parse context positioned at the start of the body.
+ * @param closeCondition - Predicate that signals the end of the block body.
+ * @param options        - Optional settings.
+ * @param options.excludedBlockNames - Block names that should be excluded
+ *   from both rule dispatch and paragraph-boundary detection. The named
+ *   rules are filtered out of `blockRules`, and the set is propagated to
+ *   the inline parser via `ParseContext.excludedBlockNames` so that
+ *   `BLOCK_OPEN` / `BLOCK_END_OPEN` tokens for these names do not trigger
+ *   paragraph breaks.
+ * @returns Parsed elements and total tokens consumed.
+ */
+export function parseBlocksUntil(
+  ctx: ParseContext,
+  closeCondition: (ctx: ParseContext) => boolean,
+  options?: { excludedBlockNames?: ReadonlySet<string> },
+): BlockParseResult {
+  const elements: Element[] = [];
+  let consumed = 0;
+  let pos = ctx.pos;
+
+  const excluded = options?.excludedBlockNames;
+  const blockRules = excluded ? getExcludedBlockRules(ctx.blockRules, excluded) : ctx.blockRules;
+  const blockScope = {
+    ...ctx.scope,
+    blockCloseCondition: closeCondition,
+    excludedBlockNames: excluded,
+  };
+  const checkCtx: ParseContext = { ...ctx, pos };
+  const blockCtx: ParseContext = {
+    ...ctx,
+    pos,
+    blockRules,
+    scope: blockScope,
+  };
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    // Check close condition
+    checkCtx.pos = pos;
+    if (closeCondition(checkCtx)) {
+      break;
+    }
+
+    // Skip whitespace
+    if (token.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    // Skip newlines
+    if (token.type === "NEWLINE") {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    // Pass close condition and excluded names to context
+    blockCtx.pos = pos;
+    blockCtx.scope = blockScope;
+
+    const result = parseBlockItem(blockCtx);
+    elements.push(...result.elements);
+    consumed += result.consumed;
+    pos += result.consumed;
+  }
+
+  return { elements, consumed };
+}
+
+function getExcludedBlockRules(
+  blockRules: ParseContext["blockRules"],
+  excluded: ReadonlySet<string>,
+): ParseContext["blockRules"] {
+  let byExcluded = excludedBlockRulesCache.get(blockRules);
+  if (!byExcluded) {
+    byExcluded = new WeakMap();
+    excludedBlockRulesCache.set(blockRules, byExcluded);
+  }
+
+  const cached = byExcluded.get(excluded);
+  if (cached) {
+    return cached;
+  }
+
+  const filtered = blockRules.filter((rule) => !excluded.has(rule.name));
+  byExcluded.set(excluded, filtered);
+  return filtered;
+}

package/src/parser/rules/block/parsing/end-condition.ts

@@ -0,0 +1,51 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../utils";
+
+/**
+ * Creates a reusable close-condition function that matches block end tags
+ * (`[[/name]]`) for one or more block names.
+ *
+ * The returned function inspects the tokens at `ctx.pos` and returns both
+ * whether a match was found and how many tokens the closing tag occupies
+ * (including the optional trailing NEWLINE).
+ *
+ * @param blockNames - Array of block names to match (e.g. `["div"]`).
+ * @returns A function suitable for use as a `closeCondition` argument,
+ *          returning `{ matched, consumed }`.
+ */
+export function createBlockEndCondition(
+  blockNames: string[],
+): (ctx: ParseContext) => { matched: boolean; consumed: number } {
+  return (ctx: ParseContext) => {
+    const token = ctx.tokens[ctx.pos];
+    if (token?.type !== "BLOCK_END_OPEN") {
+      return { matched: false, consumed: 0 };
+    }
+
+    const nameResult = parseBlockName(ctx, ctx.pos + 1);
+    if (!nameResult) {
+      return { matched: false, consumed: 0 };
+    }
+
+    if (!blockNames.includes(nameResult.name)) {
+      return { matched: false, consumed: 0 };
+    }
+
+    // Calculate consumed: [[/ + name + ]]
+    let consumed = 1 + nameResult.consumed;
+
+    // Check for closing ]]
+    const closePos = ctx.pos + 1 + nameResult.consumed;
+    if (ctx.tokens[closePos]?.type === "BLOCK_CLOSE") {
+      consumed++;
+    }
+
+    // Check for trailing newline
+    const newlinePos = closePos + 1;
+    if (ctx.tokens[newlinePos]?.type === "NEWLINE") {
+      consumed++;
+    }
+
+    return { matched: true, consumed };
+  };
+}