@wdprlib/parser 3.1.2 → 3.2.0

package/src/parser/rules/block/include.ts

@@ -0,0 +1,179 @@
+import type { Element, PageRef, VariableMap } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../types";
+import { currentToken } from "../types";
+import { parseBlockName } from "./utils";
+
+/**
+ * Parse page reference from include target
+ * Formats:
+ *   - "page" -> { site: null, page: "page" }
+ *   - ":site:page" -> { site: "site", page: "page" }
+ *   - "fragment:name" -> { site: null, page: "fragment:name" }
+ */
+function parsePageRef(target: string): PageRef {
+  // Check for :site:page format
+  if (target.startsWith(":")) {
+    const rest = target.slice(1);
+    const colonIndex = rest.indexOf(":");
+    if (colonIndex !== -1) {
+      return {
+        site: rest.slice(0, colonIndex),
+        page: rest.slice(colonIndex + 1),
+      };
+    }
+  }
+  return { site: null, page: target };
+}
+
+/**
+ * Parse variables from include arguments
+ * Format: key=value separated by |
+ */
+function parseVariables(tokens: string[]): VariableMap {
+  const vars: VariableMap = {};
+
+  let current = "";
+  for (const token of tokens) {
+    if (token === "|") {
+      if (current.trim()) {
+        const eqIndex = current.indexOf("=");
+        if (eqIndex !== -1) {
+          const key = current.slice(0, eqIndex).trim();
+          const value = current.slice(eqIndex + 1).trim();
+          if (key) {
+            vars[key] = value;
+          }
+        }
+      }
+      current = "";
+    } else {
+      current += token;
+    }
+  }
+
+  // Handle last segment
+  if (current.trim()) {
+    const eqIndex = current.indexOf("=");
+    if (eqIndex !== -1) {
+      const key = current.slice(0, eqIndex).trim();
+      const value = current.slice(eqIndex + 1).trim();
+      if (key) {
+        vars[key] = value;
+      }
+    }
+  }
+
+  return vars;
+}
+
+export const includeRule: BlockRule = {
+  name: "include",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    let pos = ctx.pos + 1;
+    let consumed = 1;
+
+    // Parse block name
+    const nameResult = parseBlockName(ctx, pos);
+    if (!nameResult || nameResult.name.toLowerCase() !== "include") {
+      return { success: false };
+    }
+
+    // Page syntax disabled (e.g., forum-post mode)
+    if (!ctx.settings.enablePageSyntax) {
+      return { success: false };
+    }
+
+    pos += nameResult.consumed;
+    consumed += nameResult.consumed;
+
+    // Skip whitespace
+    while (ctx.tokens[pos]?.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+    }
+
+    // Collect target and arguments until ]]
+    const argTokens: string[] = [];
+    let target = "";
+    let inTarget = true;
+
+    while (pos < ctx.tokens.length) {
+      const token = ctx.tokens[pos];
+      if (!token) break;
+
+      if (token.type === "BLOCK_CLOSE") {
+        break;
+      }
+
+      // Newlines are allowed in include arguments
+      if (token.type === "NEWLINE") {
+        pos++;
+        consumed++;
+        continue;
+      }
+
+      if (inTarget) {
+        // First non-whitespace segment is the target
+        if (token.type === "WHITESPACE") {
+          if (target) {
+            inTarget = false;
+          }
+        } else if (token.type === "PIPE") {
+          inTarget = false;
+          argTokens.push("|");
+        } else {
+          target += token.value;
+        }
+      } else {
+        argTokens.push(token.value);
+      }
+
+      pos++;
+      consumed++;
+    }
+
+    // Expect ]]
+    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+      return { success: false };
+    }
+    pos++;
+    consumed++;
+
+    // Skip trailing newline
+    if (ctx.tokens[pos]?.type === "NEWLINE") {
+      pos++;
+      consumed++;
+    }
+
+    if (!target) {
+      return { success: false };
+    }
+
+    const location = parsePageRef(target);
+    const variables = parseVariables(argTokens);
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "include",
+          data: {
+            "paragraph-safe": false,
+            variables,
+            location,
+            elements: [],
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/index.ts

@@ -0,0 +1,127 @@
+/**
+ *
+ * Central registry for all block-level parser rules.
+ *
+ * This module imports every block rule, re-exports each one individually
+ * (for selective use), and assembles them into the ordered `blockRules`
+ * array that the main parser iterates through.
+ *
+ * Rule ordering matters: rules earlier in the array are tried first.
+ * Line-start-only rules (comments, headings, lists, etc.) are naturally
+ * filtered by the `requiresLineStart` flag, but among rules that share
+ * a start token (e.g. BLOCK_OPEN), position in this array determines
+ * priority. For example, `codeBlockRule` is tried before `collapsibleRule`
+ * because both start with BLOCK_OPEN but code blocks should be matched
+ * first.
+ *
+ * The paragraph rule is deliberately excluded from the array -- it serves
+ * as a fallback and is exported separately as `blockFallbackRule`.
+ *
+ * @module
+ */
+import type { BlockRule } from "../types";
+import { headingRule } from "./heading";
+import { horizontalRuleRule } from "./horizontal-rule";
+import { listRule } from "./list";
+import { blockListRule } from "./block-list";
+import { blockquoteRule } from "./blockquote";
+import { definitionListRule } from "./definition-list";
+import { paragraphRule } from "./paragraph";
+import { divRule } from "./div";
+import { codeBlockRule } from "./code";
+import { collapsibleRule } from "./collapsible";
+import { tableRule } from "./table";
+import { tableBlockRule } from "./table-block";
+import { moduleRule } from "./module";
+import { footnoteBlockRule } from "./footnoteblock";
+import { blockCommentRule } from "./comment";
+import { centerRule } from "./center";
+import { contentSeparatorRule } from "./content-separator";
+import { clearFloatRule } from "./clear-float";
+import { alignRule } from "./align";
+import { tabviewRule } from "./tabview";
+import { includeRule } from "./include";
+import { mathBlockRule } from "./math";
+import { htmlBlockRule } from "./html";
+import { embedBlockRule } from "./embed-block";
+import { iframeRule } from "./iframe";
+import { iftagsRule } from "./iftags";
+import { tocRule } from "./toc";
+import { orphanLiRule } from "./orphan-li";
+import { bibliographyRule } from "./bibliography";
+
+export { headingRule } from "./heading";
+export { horizontalRuleRule } from "./horizontal-rule";
+export { listRule } from "./list";
+export { blockListRule } from "./block-list";
+export { blockquoteRule } from "./blockquote";
+export { definitionListRule } from "./definition-list";
+export { paragraphRule } from "./paragraph";
+export { divRule } from "./div";
+export { codeBlockRule } from "./code";
+export { collapsibleRule } from "./collapsible";
+export { tableRule } from "./table";
+export { tableBlockRule } from "./table-block";
+export { moduleRule } from "./module";
+export { footnoteBlockRule } from "./footnoteblock";
+export { blockCommentRule } from "./comment";
+export { centerRule } from "./center";
+export { contentSeparatorRule } from "./content-separator";
+export { clearFloatRule } from "./clear-float";
+export { alignRule } from "./align";
+export { tabviewRule } from "./tabview";
+export { includeRule } from "./include";
+export { mathBlockRule } from "./math";
+export { htmlBlockRule } from "./html";
+export { embedBlockRule } from "./embed-block";
+export { iframeRule } from "./iframe";
+export { iftagsRule } from "./iftags";
+export { tocRule } from "./toc";
+export { orphanLiRule } from "./orphan-li";
+export { bibliographyRule } from "./bibliography";
+
+/**
+ * All block rules in priority order.
+ *
+ * The main parser tries each rule in sequence for the current token.
+ * Rules with `requiresLineStart: true` are only attempted when the token
+ * is at line start, so their position relative to non-line-start rules
+ * is less critical. Among rules that share the same `startTokens` entry,
+ * earlier position wins.
+ *
+ * The paragraph rule is intentionally absent -- it is used as a fallback
+ * when no other rule matches (see `blockFallbackRule`).
+ */
+export const blockRules: BlockRule[] = [
+  blockCommentRule,
+  clearFloatRule,
+  contentSeparatorRule,
+  centerRule,
+  headingRule,
+  horizontalRuleRule,
+  tableRule,
+  tableBlockRule,
+  listRule,
+  blockListRule,
+  orphanLiRule,
+  blockquoteRule,
+  definitionListRule,
+  codeBlockRule,
+  collapsibleRule,
+  tocRule,
+  footnoteBlockRule,
+  moduleRule,
+  alignRule,
+  tabviewRule,
+  includeRule,
+  mathBlockRule,
+  htmlBlockRule,
+  embedBlockRule,
+  iframeRule,
+  iftagsRule,
+  bibliographyRule,
+  divRule,
+  // paragraphRule is not included - used as fallback
+];
+
+export { paragraphRule as blockFallbackRule };

package/src/parser/rules/block/list.ts

@@ -0,0 +1,244 @@
+/**
+ *
+ * Block rule for Wikidot marker-based lists (`* item`, `# item`).
+ *
+ * Wikidot lists use leading `*` (bullet) or `#` (numbered) markers at the
+ * start of a line. Nesting is achieved by prepending spaces:
+ *
+ * ```
+ * * Item 1
+ *  * Nested bullet
+ *  # Nested numbered
+ * * Item 2
+ * ```
+ *
+ * The depth of each item is determined by the number of leading spaces
+ * before the marker. Mixed bullet/numbered lists are supported: when the
+ * list type changes at the same depth, a new sub-list is created.
+ *
+ * The flat depth-annotated items are converted into a recursive tree by
+ * `processDepths()`, then transformed into nested `list` AST elements
+ * by `buildListElement()`.
+ *
+ * Maximum nesting depth is capped at `MAX_LIST_DEPTH` (20).
+ *
+ * @module
+ */
+import type { Element, ListData, ListItem, ListType } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../types";
+import { currentToken } from "../types";
+import { parseInlineUntil } from "../inline/utils";
+import { processDepths, type DepthList } from "../../depth";
+
+/**
+ * Safety limit for list nesting depth.
+ * Items deeper than this are not parsed, preventing stack overflow on
+ * deeply nested or adversarial input.
+ */
+const MAX_LIST_DEPTH = 20;
+
+/** Internal discriminated type for bullet vs numbered items during parsing. */
+type InternalListType = "bullet" | "numbered";
+
+/**
+ * Default list type used as the top-level placeholder in
+ * `processDepths()`. The actual type of each sub-list is determined
+ * by its first item's marker.
+ */
+const GENERIC_LIST_TYPE: InternalListType = "bullet";
+
+/**
+ * Block rule for marker-based lists (`* ` bullet, `# ` numbered).
+ *
+ * Parsing strategy:
+ * 1. Verify the first token is LIST_BULLET or LIST_NUMBER at line start.
+ * 2. Collect consecutive list lines, recording each item's depth (number
+ *    of leading spaces), type (bullet/numbered), and inline content.
+ * 3. Feed the flat depth array into `processDepths()` with type
+ *    comparison, producing a nested tree.
+ * 4. Convert the tree into `list` AST elements via `buildListElement()`.
+ */
+export const listRule: BlockRule = {
+  name: "list",
+  startTokens: ["LIST_BULLET", "LIST_NUMBER"],
+  requiresLineStart: true,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const firstToken = currentToken(ctx);
+
+    if (!firstToken.lineStart) {
+      return { success: false };
+    }
+
+    // Wikidot: list must start with list marker directly (no leading whitespace)
+    if (firstToken.type !== "LIST_BULLET" && firstToken.type !== "LIST_NUMBER") {
+      return { success: false };
+    }
+
+    // Collect depth-annotated items
+    const depths: Array<{
+      depth: number;
+      ltype: InternalListType;
+      value: Element[];
+    }> = [];
+    let pos = ctx.pos;
+    let consumed = 0;
+
+    while (pos < ctx.tokens.length) {
+      const token = ctx.tokens[pos];
+
+      if (!token || !token.lineStart) {
+        break;
+      }
+
+      // Determine depth from leading whitespace
+      let depth = 0;
+      if (token.type === "WHITESPACE") {
+        // Count spaces for depth (each space is 1 level)
+        depth = token.value.length;
+        pos++;
+        consumed++;
+      }
+
+      // Check for list marker
+      const markerToken = ctx.tokens[pos];
+      if (
+        !markerToken ||
+        (markerToken.type !== "LIST_BULLET" && markerToken.type !== "LIST_NUMBER")
+      ) {
+        // Undo whitespace consumption if not followed by list marker
+        if (depth > 0) {
+          pos--;
+          consumed--;
+        }
+        break;
+      }
+
+      // Check maximum depth
+      if (depth > MAX_LIST_DEPTH) {
+        break;
+      }
+
+      // Get list type
+      const ltype: InternalListType = markerToken.type === "LIST_BULLET" ? "bullet" : "numbered";
+
+      // Skip marker
+      pos++;
+      consumed++;
+
+      // Expect whitespace after marker
+      if (ctx.tokens[pos]?.type === "WHITESPACE") {
+        pos++;
+        consumed++;
+      }
+
+      // Parse inline content until newline
+      const inlineCtx: ParseContext = { ...ctx, pos };
+      const inlineResult = parseInlineUntil(inlineCtx, "NEWLINE");
+      const elements: Element[] = inlineResult.elements;
+      consumed += inlineResult.consumed;
+      pos += inlineResult.consumed;
+
+      // Consume newline
+      if (ctx.tokens[pos]?.type === "NEWLINE") {
+        pos++;
+        consumed++;
+      }
+
+      // Add to depths
+      depths.push({
+        depth,
+        ltype,
+        value: elements,
+      });
+    }
+
+    // No items parsed - rule fails
+    if (depths.length === 0) {
+      return { success: false };
+    }
+
+    // Process depths with list type comparison
+    const depthTrees = processDepths<InternalListType, Element[]>(
+      GENERIC_LIST_TYPE,
+      depths,
+      (a, b) => a === b,
+    );
+
+    // Convert depth trees to list elements
+    const lists = depthTrees.map(({ ltype, list }) => buildListElement(ltype, list));
+
+    return {
+      success: true,
+      elements: lists,
+      consumed,
+    };
+  },
+};
+
+/**
+ * Converts the internal list type enum to the AST's {@link ListType}.
+ *
+ * @param ltype - Internal "bullet" or "numbered".
+ * @returns The corresponding AST list type.
+ */
+function toListType(ltype: InternalListType): ListType {
+  return ltype === "numbered" ? "numbered" : "bullet";
+}
+
+/**
+ * Builds a `list` AST element from a depth tree produced by
+ * `processDepths()`.
+ *
+ * @param topLtype - The list type for the top-level list.
+ * @param list     - The depth tree of items and sub-lists.
+ * @returns A `list` element.
+ */
+function buildListElement(
+  topLtype: InternalListType,
+  list: DepthList<InternalListType, Element[]>,
+): Element {
+  return {
+    element: "list",
+    data: buildListData(topLtype, list),
+  };
+}
+
+/**
+ * Recursively builds the {@link ListData} payload from a depth tree.
+ *
+ * Leaf items become `"elements"` list items; nested sub-trees become
+ * `"sub-list"` items with their own recursive {@link ListData}.
+ *
+ * @param topLtype - List type for this level.
+ * @param list     - The depth tree nodes at this level.
+ * @returns Fully constructed {@link ListData}.
+ */
+function buildListData(
+  topLtype: InternalListType,
+  list: DepthList<InternalListType, Element[]>,
+): ListData {
+  const items: ListItem[] = [];
+
+  for (const item of list) {
+    if (item.kind === "item") {
+      items.push({
+        "item-type": "elements",
+        attributes: {},
+        elements: item.value,
+      });
+    } else {
+      items.push({
+        "item-type": "sub-list",
+        element: "list",
+        data: buildListData(item.ltype, item.children),
+      });
+    }
+  }
+
+  return {
+    type: toListType(topLtype),
+    attributes: {},
+    items,
+  };
+}