@wdprlib/ast 4.3.0 → 5.0.0

package/licenses/re2js-MIT.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Oleksii Vasyliev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/ast",
-  "version": "4.3.0",
+  "version": "5.0.0",
   "description": "AST types for Wikidot markup",
   "keywords": [
     "ast",
@@ -18,6 +18,7 @@
     "dist",
     "src",
     "LICENSE",
+    "licenses",
     "THIRD-PARTY-LICENSES.md"
   ],
   "type": "module",
@@ -40,5 +41,8 @@
   },
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "re2js": "2.8.6"
   }
 }

package/src/build-info.generated.ts

@@ -5,7 +5,7 @@ export const buildInfo: Readonly<{
   sha: string | null;
   dirty: boolean | null;
 }> = Object.freeze({
-  version: "4.3.0",
-  sha: "5e221ddeccc585e64dd1a2eb49b14817c3c1be5d",
+  version: "5.0.0",
+  sha: "705ff97e22abd7bfb49d7f8f30087e6f09654c91",
   dirty: false,
 });

package/src/element.ts

@@ -17,6 +17,7 @@
  */
 
 import type { CssLengthUnit } from "./css";
+import type { RateModuleData, CustomRateModuleData } from "./rating";
 
 // ---------------------------------------------------------------------------
 // Primitive types
@@ -499,10 +500,8 @@ export type Module =
       /** Max depth, or null for unlimited */
       depth: number | null;
     }
-  | {
-      /** `[[module Rate]]` — page rating widget */
-      module: "rate";
-    }
+  | RateModuleData
+  | CustomRateModuleData
   | {
       /** `[[module TagCloud]]` — weighted cloud of page tags */
       module: "tag-cloud";
@@ -547,6 +546,7 @@ export type Module =
       "created-at"?: string;
       "updated-at"?: string;
       rating?: string;
+      "rating-axis"?: string;
       votes?: string;
       name?: string;
       fullname?: string;

package/src/index.ts

@@ -11,6 +11,18 @@
  */
 
 export { buildInfo } from "./build-info.generated";
+export { extractReadableText, extractFirstParagraph, countCharacters } from "./readable-text";
+export type { ReadableTextOptions } from "./readable-text";
+export { excerptText, compileTextExcerpt } from "./text-excerpt";
+export type { TextExcerptOptions } from "./text-excerpt";
+export type {
+  RatingRef,
+  RatingVote,
+  RatingAggregate,
+  RatingState,
+  RateModuleData,
+  CustomRateModuleData,
+} from "./rating";
 
 export type { Position, Point } from "./position";
 export { createPoint, createPosition } from "./position";

package/src/rating.ts

@@ -0,0 +1,44 @@
+/** A rating always belongs to the host's displayed page. */
+export type RatingRef = { kind: "main" } | { kind: "custom"; axisKey: string };
+
+/** Zero is a neutral vote, not a cancellation. */
+export type RatingVote = -1 | 0 | 1;
+
+/** Aggregates and the treatment of neutral votes are computed by the host. */
+export interface RatingAggregate {
+  points: number;
+  votes: number;
+  percent: number;
+}
+
+/** An authorized, registered rating supplied by the host for the current viewer. */
+export interface RatingState {
+  ref: RatingRef;
+  label: string;
+  /** Plain-text vote labels, e.g. { 1: "+", 0: "φ", [-1]: "-" }. Omitted entries use + / Ø / –. */
+  voteLabels?: Readonly<Partial<Record<RatingVote, string>>>;
+  /** Host policy: e.g. [1, -1], [1], [-1], or [1, 0, -1]. */
+  allowedVotes: readonly RatingVote[];
+  /** False keeps a visible, read-only widget. Omit the state to hide it entirely. */
+  canVote: boolean;
+  /** Independent of permission to cast a vote. */
+  canCancel: boolean;
+  /** Null means no vote; zero is a saved neutral vote. */
+  currentVote: RatingVote | null;
+  /** Null hides all aggregate values without preventing voting. */
+  aggregate: RatingAggregate | null;
+}
+
+/** `[[module Rate]]` accepts no attributes. A null reference marks an invalid declaration. */
+export interface RateModuleData {
+  module: "rate";
+  ref: Extract<RatingRef, { kind: "main" }> | null;
+  state?: RatingState;
+}
+
+/** `[[module CustomRate key="theme"]]` accepts only key; it never registers or changes policy. */
+export interface CustomRateModuleData {
+  module: "custom-rate";
+  ref: Extract<RatingRef, { kind: "custom" }> | null;
+  state?: RatingState;
+}

package/src/readable-text.ts

@@ -0,0 +1,220 @@
+import type { Element, SyntaxTree } from "./element";
+import { evaluateExpression, formatExprValue, isTruthy } from "./expr-eval";
+
+export interface ReadableTextOptions {
+  /** Exclude a subtree, for example an ACS/license container identified by the host. */
+  exclude?: (element: Element) => boolean;
+  /** Match host-rendered labels or supply text for date/math leaves. Undefined uses the default. */
+  resolveText?: (
+    element: Extract<Element, { element: "link" | "user" | "date" | "math" | "math-inline" }>,
+  ) => string | undefined;
+}
+
+const segmenter = new Intl.Segmenter("und", { granularity: "grapheme" });
+
+/** Count graphemes, including whitespace, in the supplied readable text. */
+export function countCharacters(text: string): number {
+  let count = 0;
+  for (const _segment of segmenter.segment(text)) count++;
+  return count;
+}
+
+/**
+ * Extract semantic text from a resolved document, without executing HTML, CSS or JS.
+ * Paragraph boundaries become blank lines; other whitespace is normalized. Tabs and
+ * collapsibles include all panels. Footnotes are appended once, without reference UI.
+ * Math/date leaves require resolveText; source TeX and timestamps are not prose.
+ */
+export function extractReadableText(ast: SyntaxTree, options: ReadableTextOptions = {}): string {
+  return extractText(ast, options, false);
+}
+
+/** Extract the first nonempty paragraph, excluding headings and appended footnotes. */
+export function extractFirstParagraph(ast: SyntaxTree, options: ReadableTextOptions = {}): string {
+  return extractText(ast, options, true);
+}
+
+function extractText(
+  ast: SyntaxTree,
+  options: ReadableTextOptions,
+  firstParagraphOnly: boolean,
+): string {
+  const parts: string[] = [];
+  const notes = new Set<number>();
+  let footnoteIndex = 0;
+  let firstParagraph: string | undefined;
+  const emit = (value: string, omit: boolean) => {
+    if (!omit) parts.push(value);
+  };
+  const block = (elements: Element[], omit: boolean) => {
+    emit("\n\n", omit);
+    visit(elements, omit);
+    emit("\n\n", omit);
+  };
+  const visitBranch = (elements: Element[], omit: boolean) => {
+    const end =
+      elements.findLastIndex(
+        (element) => element.element !== "text" || element.data.trim() !== "",
+      ) + 1;
+    visit(elements.slice(0, end), omit);
+  };
+  const visit = (elements: Element[], inheritedOmit = false): void => {
+    for (const element of elements) {
+      if (firstParagraphOnly && firstParagraph !== undefined) return;
+      const omit = inheritedOmit || options.exclude?.(element) === true;
+      switch (element.element) {
+        case "text":
+        case "raw":
+        case "email":
+          emit(element.data, omit);
+          break;
+        case "container": {
+          const start = parts.length;
+          const type = element.data.type;
+          const kind: string = typeof type === "string" ? type : "block";
+          const hidden = omit || ["ruby-text", "hidden", "invisible"].includes(kind);
+          if (
+            [
+              "paragraph",
+              "div",
+              "blockquote",
+              "note",
+              "heading",
+              "block",
+              "table-row",
+              "definition-list",
+            ].includes(kind)
+          )
+            block(element.data.elements, hidden);
+          else visit(element.data.elements, hidden);
+          if (firstParagraphOnly && kind === "paragraph" && firstParagraph === undefined) {
+            const text = normalizeText(parts.slice(start).join(""));
+            if (text) firstParagraph = text;
+          }
+          break;
+        }
+        case "color":
+        case "anchor":
+        case "include":
+          visit(element.data.elements, omit);
+          break;
+        case "collapsible":
+          block(element.data.elements, omit);
+          break;
+        case "tab-view":
+          for (const tab of element.data) {
+            emit(`\n\n${tab.label}\n`, omit);
+            block(tab.elements, omit);
+          }
+          break;
+        case "table":
+          emit("\n\n", omit);
+          for (const row of element.data.rows) {
+            for (const cell of row.cells) {
+              visit(cell.elements, omit);
+              emit(" ", omit);
+            }
+            emit("\n", omit);
+          }
+          emit("\n", omit);
+          break;
+        case "definition-list":
+          for (const entry of element.data) {
+            block(entry.key, omit);
+            visit(entry.value, omit);
+          }
+          break;
+        case "list":
+          for (const item of element.data.items) {
+            emit("\n", omit);
+            if (item["item-type"] === "elements") visit(item.elements, omit);
+            else visit([{ element: "list", data: item.data }], omit);
+          }
+          emit("\n", omit);
+          break;
+        case "link": {
+          const label = element.data.label;
+          const destination =
+            typeof element.data.link === "string" ? element.data.link : element.data.link.page;
+          const fallback =
+            label === "page"
+              ? destination
+              : "text" in label
+                ? label.text
+                : (label.url ?? destination);
+          emit(options.resolveText?.(element) ?? fallback, omit);
+          break;
+        }
+        case "user":
+          emit(options.resolveText?.(element) ?? element.data.name, omit);
+          break;
+        case "date":
+        case "math":
+        case "math-inline":
+          emit(options.resolveText?.(element) ?? "", omit);
+          break;
+        case "image":
+          emit(element.data.attributes.alt ?? "", omit);
+          break;
+        case "gallery":
+          if (element.data.content.type === "items")
+            for (const item of element.data.content.items) emit(`${item.alt ?? ""}\n`, omit);
+          break;
+        case "code":
+          emit(`\n\n${element.data.contents}\n\n`, omit);
+          break;
+        case "footnote":
+        case "footnote-ref": {
+          const index = element.element === "footnote" ? footnoteIndex++ : element.data - 1;
+          if (!omit) notes.add(index);
+          break;
+        }
+        case "bibliography-block":
+          for (const entry of element.data.entries) block(entry.value, omit);
+          break;
+        case "if":
+          visitBranch(
+            isTruthy(element.data.condition) ? element.data.then : element.data.else,
+            omit,
+          );
+          break;
+        case "ifexpr": {
+          const result = evaluateExpression(element.data.expression);
+          if (result.success)
+            visitBranch(result.value !== 0 ? element.data.then : element.data.else, omit);
+          break;
+        }
+        case "expr": {
+          const result = evaluateExpression(element.data.expression);
+          if (result.success) emit(formatExprValue(result.value), omit);
+          break;
+        }
+        case "line-break":
+        case "line-breaks":
+          emit("\n", omit);
+          break;
+        case "horizontal-rule":
+        case "content-separator":
+          emit("\n\n", omit);
+          break;
+        default:
+          break;
+      }
+    }
+  };
+  visit(ast.elements);
+  if (firstParagraphOnly) return firstParagraph ?? "";
+  for (const index of notes) {
+    const note = ast.footnotes?.[index];
+    if (note) block(note, false);
+  }
+  return normalizeText(parts.join(""));
+}
+
+function normalizeText(text: string): string {
+  return text
+    .replace(/[^\S\n]+/g, " ")
+    .replace(/ ?\n ?/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}

package/src/text-excerpt.ts

@@ -0,0 +1,109 @@
+import { RE2JS } from "re2js";
+
+export interface TextExcerptOptions {
+  /** RE2 pattern, without / delimiters. Omit to truncate the entire text. */
+  pattern?: string;
+  /** Supported flags: i (case insensitive), m (line anchors), s (dot includes newline). */
+  flags?: string;
+  /** Capture number or name. Defaults to 0, the complete match. */
+  group?: number | string;
+  /** One-based, non-overlapping match number. Defaults to 1; maximum 10,000. */
+  match?: number;
+  /** Grapheme limit, default 200. Invalid, negative, or zero lengths return empty text. */
+  maxLength?: number;
+}
+
+const segmenter = new Intl.Segmenter("und", { granularity: "grapheme" });
+const emptyExcerpt = (_text: string): string => "";
+const MAX_PATTERN_LENGTH = 4_096;
+const MAX_INPUT_LENGTH = 1_000_000;
+const MAX_PROGRAM_SIZE = 4_096;
+const MAX_MATCHES = 10_000;
+const SEARCH_BUDGET = 16_000_000;
+
+/**
+ * Compile once and reuse across pages. Invalid options, unsupported RE2 syntax, missing
+ * matches/groups and exceeded limits return empty text. Regex inputs are limited to
+ * 4,096 pattern code units and 1,000,000 text code units. The compiled program is limited
+ * to 4,096 instructions; searches share a conservative 16,000,000-unit work budget
+ * based on program size, capture count and remaining input length. No native RegExp fallback.
+ */
+export function compileTextExcerpt(options: TextExcerptOptions = {}): (text: string) => string {
+  const limit = options.maxLength ?? 200;
+  if (!Number.isSafeInteger(limit) || limit <= 0) return emptyExcerpt;
+  if (options.pattern === undefined) {
+    if (options.flags !== undefined || options.group !== undefined || options.match !== undefined) {
+      return emptyExcerpt;
+    }
+    return (text) => truncateText(text, limit);
+  }
+  const occurrence = options.match ?? 1;
+  const group = options.group ?? 0;
+  if (
+    options.pattern.length > MAX_PATTERN_LENGTH ||
+    !Number.isSafeInteger(occurrence) ||
+    occurrence < 1 ||
+    occurrence > MAX_MATCHES ||
+    (typeof group === "number" && (!Number.isSafeInteger(group) || group < 0))
+  )
+    return emptyExcerpt;
+  let flags = 0;
+  const seenFlags = new Set<string>();
+  for (const flag of options.flags ?? "") {
+    if (seenFlags.has(flag)) return emptyExcerpt;
+    seenFlags.add(flag);
+    switch (flag) {
+      case "i":
+        flags |= RE2JS.CASE_INSENSITIVE;
+        break;
+      case "m":
+        flags |= RE2JS.MULTILINE;
+        break;
+      case "s":
+        flags |= RE2JS.DOTALL;
+        break;
+      default:
+        return emptyExcerpt;
+    }
+  }
+  let compiled: RE2JS;
+  try {
+    compiled = RE2JS.compile(options.pattern, flags);
+  } catch {
+    return emptyExcerpt;
+  }
+  const programSize = compiled.programSize();
+  const groupCount = compiled.groupCount();
+  if (
+    programSize > MAX_PROGRAM_SIZE ||
+    (typeof group === "number" ? group > groupCount : !Object.hasOwn(compiled.namedGroups(), group))
+  )
+    return emptyExcerpt;
+
+  return (text) => {
+    if (text.length > MAX_INPUT_LENGTH) return "";
+    const matcher = compiled.matcher(text);
+    let budget = SEARCH_BUDGET;
+    let from = 0;
+    for (let index = 1; index <= occurrence; index++) {
+      // Repeated unanchored searches can revisit the remaining input. Charge every search.
+      budget -= programSize * (groupCount + 1) * (text.length - from + 1);
+      if (budget < 0 || !matcher.find()) return "";
+      from = matcher.end();
+    }
+    return truncateText(matcher.group(group) ?? "", limit);
+  };
+}
+
+/** Select a regex match/capture and truncate without splitting a grapheme cluster. */
+export function excerptText(text: string, options: TextExcerptOptions = {}): string {
+  return compileTextExcerpt(options)(text);
+}
+
+function truncateText(text: string, limit: number): string {
+  let count = 0;
+  for (const segment of segmenter.segment(text)) {
+    if (count++ === limit) return text.slice(0, segment.index);
+  }
+  return text;
+}