@refract-org/analyzers 0.2.1

package/src/talk-section-parser.ts

@@ -0,0 +1,151 @@
+import type { EvidenceEvent } from "@refract-org/evidence-graph";
+
+export interface TalkReply {
+  depth: number;
+  text: string;
+  author?: string;
+  timestamp?: string;
+}
+
+export interface TalkThread {
+  heading: string;
+  startedAt?: string;
+  replies: TalkReply[];
+  participants: string[];
+  isResolved: boolean;
+}
+
+export interface TalkThreadChange {
+  type: "opened" | "archived" | "reply_added" | "unchanged";
+  thread: TalkThread;
+}
+
+const RESOLVED_PATTERN = /\{\{(resolved|done|closed|archived)\}\}/i;
+
+function extractSignatures(text: string): { author?: string; timestamp?: string } {
+  const userMatch = text.match(/\[\[[Uu]ser:([^\]|]+)/);
+  const tsMatch = text.match(/(\d{1,2}:\d{2},\s+\d{1,2}\s+\w+\s+\d{4})/);
+  return {
+    author: userMatch?.[1],
+    timestamp: tsMatch?.[1],
+  };
+}
+
+export function parseTalkThreads(wikitext: string): TalkThread[] {
+  const threads: TalkThread[] = [];
+  const headerRegex = /^(={1,3})\s*([^=]+?)\s*\1\s*$/m;
+  const lines = wikitext.split("\n");
+  let currentThread: TalkThread | null = null;
+
+  for (const line of lines) {
+    const headerMatch = headerRegex.exec(line);
+    if (headerMatch) {
+      if (currentThread) {
+        threads.push(currentThread);
+      }
+      currentThread = {
+        heading: headerMatch[2].trim(),
+        startedAt: undefined,
+        replies: [],
+        participants: [],
+        isResolved: false,
+      };
+      continue;
+    }
+
+    if (!currentThread) continue;
+
+    if (line.trim()) {
+      if (RESOLVED_PATTERN.test(line)) {
+        currentThread.isResolved = true;
+      }
+
+      const indent = line.search(/\S/);
+      const depth = indent > 0 ? Math.ceil(indent / 2) + 1 : 1;
+      const { author, timestamp } = extractSignatures(line);
+      const text = line.replace(/^[:*#]+\s*/, "").trim();
+
+      currentThread.replies.push({ depth, text, author, timestamp });
+      if (author && !currentThread.participants.includes(author)) {
+        currentThread.participants.push(author);
+      }
+      if (timestamp && !currentThread.startedAt) {
+        currentThread.startedAt = timestamp;
+      }
+    }
+  }
+
+  if (currentThread) {
+    threads.push(currentThread);
+  }
+
+  return threads;
+}
+
+export function diffTalkThreads(before: TalkThread[], after: TalkThread[]): TalkThreadChange[] {
+  const changes: TalkThreadChange[] = [];
+  const beforeMap = new Map(before.map((t) => [t.heading.toLowerCase(), t]));
+  const afterMap = new Map(after.map((t) => [t.heading.toLowerCase(), t]));
+
+  for (const [key, thread] of afterMap) {
+    const prev = beforeMap.get(key);
+    if (!prev) {
+      changes.push({ type: "opened", thread });
+    } else if (thread.replies.length > prev.replies.length) {
+      changes.push({ type: "reply_added", thread });
+    } else {
+      changes.push({ type: "unchanged", thread });
+    }
+  }
+
+  for (const [key, thread] of beforeMap) {
+    if (!afterMap.has(key)) {
+      changes.push({ type: "archived", thread });
+    }
+  }
+
+  return changes;
+}
+
+export function buildTalkThreadEvents(
+  beforeWikitext: string,
+  afterWikitext: string,
+  fromRevId: number,
+  toRevId: number,
+  timestamp: string,
+): EvidenceEvent[] {
+  const before = parseTalkThreads(beforeWikitext);
+  const after = parseTalkThreads(afterWikitext);
+  const changes = diffTalkThreads(before, after);
+  const events: EvidenceEvent[] = [];
+
+  for (const change of changes) {
+    if (change.type === "unchanged") continue;
+
+    const eventType =
+      change.type === "opened"
+        ? "talk_thread_opened"
+        : change.type === "archived"
+          ? "talk_thread_archived"
+          : "talk_reply_added";
+
+    events.push({
+      eventType: eventType as EvidenceEvent["eventType"],
+      fromRevisionId: fromRevId,
+      toRevisionId: toRevId,
+      section: change.thread.heading || "(unknown)",
+      before: "",
+      after: change.thread.heading,
+      deterministicFacts: [
+        {
+          fact: `talk_thread_${change.type}`,
+          detail: `heading="${change.thread.heading}" participants=${change.thread.participants.length} replies=${change.thread.replies.length} resolved=${change.thread.isResolved}`,
+        },
+      ],
+      layer: "observed",
+      timestamp,
+    });
+  }
+
+  return events;
+}

package/src/template-tracker.ts

@@ -0,0 +1,253 @@
+import type { EvidenceEvent } from "@refract-org/evidence-graph";
+import type { Template, TemplateChange, TemplateTracker, TemplateType } from "./index.js";
+
+const TEMPLATE_TYPE_MAP: Record<string, TemplateType> = {
+  "citation needed": "citation",
+  cn: "citation",
+  citation: "citation",
+  cite: "citation",
+  fact: "citation",
+  unreferenced: "citation",
+  refimprove: "citation",
+  "need citation": "citation",
+  "primary sources": "citation",
+  npov: "neutrality",
+  pov: "neutrality",
+  undue: "neutrality",
+  blp: "blp",
+  "blp sources": "blp",
+  "living persons": "blp",
+  disputed: "dispute",
+  dispute: "dispute",
+  contradict: "dispute",
+  inconsistent: "dispute",
+  cleanup: "cleanup",
+  "copy edit": "cleanup",
+  tone: "cleanup",
+  wikify: "cleanup",
+  merge: "cleanup",
+  split: "cleanup",
+  pp: "protection",
+  protected: "protection",
+  "pp-protected": "protection",
+  "semi-protected": "protection",
+};
+
+export const templateTracker: TemplateTracker = {
+  extractTemplates(wikitext: string): Template[] {
+    const templates: Template[] = [];
+    const seen = new Set<string>();
+    let i = 0;
+    let depth = 0;
+    let start = -1;
+
+    while (i < wikitext.length) {
+      if (wikitext[i] === "{" && wikitext[i + 1] === "{") {
+        if (depth === 0) {
+          start = i;
+        }
+        depth++;
+        i += 2;
+        continue;
+      }
+      if (wikitext[i] === "}" && wikitext[i + 1] === "}") {
+        depth--;
+        if (depth === 0 && start >= 0) {
+          const raw = wikitext.slice(start, i + 2);
+          const inner = raw.slice(2, -2).trim();
+          const firstBar = inner.indexOf("|");
+          const firstName =
+            firstBar >= 0
+              ? inner.slice(0, firstBar).trim()
+              : (() => {
+                  const nl = inner.indexOf("\n");
+                  return nl >= 0 ? inner.slice(0, nl).trim() : inner.trim();
+                })();
+          const name = firstName.toLowerCase().replace(/\s+/g, " ");
+
+          const key = name;
+          if (!seen.has(key)) {
+            seen.add(key);
+            const params = firstBar >= 0 ? parseParams(inner.slice(firstBar + 1)) : undefined;
+            templates.push({
+              name: firstName,
+              type: classifyTemplate(name),
+              params,
+            });
+          }
+        }
+        i += 2;
+        continue;
+      }
+      i++;
+    }
+
+    return templates;
+  },
+
+  diffTemplates(before: Template[], after: Template[]): TemplateChange[] {
+    const changes: TemplateChange[] = [];
+    const beforeMap = new Map<string, Template>();
+    const afterMap = new Map<string, Template>();
+
+    for (const t of before) beforeMap.set(t.name.toLowerCase(), t);
+    for (const t of after) afterMap.set(t.name.toLowerCase(), t);
+
+    for (const [name, t] of afterMap) {
+      if (!beforeMap.has(name)) {
+        changes.push({ type: "added", template: t });
+      } else {
+        changes.push({ type: "unchanged", template: t });
+      }
+    }
+
+    for (const [name, t] of beforeMap) {
+      if (!afterMap.has(name)) {
+        changes.push({ type: "removed", template: t });
+      }
+    }
+
+    return changes;
+  },
+};
+
+function classifyTemplate(name: string): TemplateType {
+  return TEMPLATE_TYPE_MAP[name] ?? "other";
+}
+
+function parseParams(raw: string): Record<string, string> {
+  const params: Record<string, string> = {};
+  const parts = splitParams(raw);
+  let unnamedIndex = 1;
+
+  for (const part of parts) {
+    const eqIndex = part.indexOf("=");
+    if (eqIndex >= 0) {
+      params[part.slice(0, eqIndex).trim()] = part.slice(eqIndex + 1).trim();
+    } else {
+      params[String(unnamedIndex)] = part.trim();
+      unnamedIndex++;
+    }
+  }
+
+  return params;
+}
+
+function splitParams(raw: string): string[] {
+  const parts: string[] = [];
+  let current = "";
+  let braceDepth = 0;
+  let linkDepth = 0;
+
+  for (let i = 0; i < raw.length; i++) {
+    const ch = raw[i];
+    const next = raw[i + 1];
+
+    if (ch === "{" && next === "{") {
+      braceDepth++;
+      current += "{{";
+      i++;
+    } else if (ch === "}" && next === "}" && braceDepth > 0) {
+      braceDepth--;
+      current += "}}";
+      i++;
+    } else if (ch === "[" && next === "[" && linkDepth === 0) {
+      linkDepth++;
+      current += "[[";
+      i++;
+    } else if (ch === "]" && next === "]" && linkDepth > 0) {
+      linkDepth--;
+      current += "]]";
+      i++;
+    } else if (ch === "|" && braceDepth === 0 && linkDepth === 0) {
+      parts.push(current.trim());
+      current = "";
+    } else {
+      current += ch;
+    }
+  }
+
+  if (current.trim()) {
+    parts.push(current.trim());
+  }
+
+  return parts;
+}
+
+export interface ParamChange {
+  templateName: string;
+  paramName: string;
+  oldValue?: string;
+  newValue?: string;
+}
+
+export function diffTemplateParams(before: Template[], after: Template[]): ParamChange[] {
+  const changes: ParamChange[] = [];
+  const beforeMap = new Map<string, Template>();
+  const afterMap = new Map<string, Template>();
+
+  for (const t of before) beforeMap.set(t.name.toLowerCase(), t);
+  for (const t of after) afterMap.set(t.name.toLowerCase(), t);
+
+  for (const [name, afterTmpl] of afterMap) {
+    const beforeTmpl = beforeMap.get(name);
+    if (!beforeTmpl || (!beforeTmpl.params && !afterTmpl.params)) continue;
+    if (!beforeTmpl.params || !afterTmpl.params) continue;
+
+    const beforeParams = normalizeParams(beforeTmpl.params);
+    const afterParams = normalizeParams(afterTmpl.params);
+    const allKeys = new Set([...Object.keys(beforeParams), ...Object.keys(afterParams)]);
+
+    for (const key of allKeys) {
+      const oldVal = beforeParams[key];
+      const newVal = afterParams[key];
+
+      if (oldVal === undefined && newVal !== undefined) {
+        changes.push({ templateName: afterTmpl.name, paramName: key, newValue: newVal });
+      } else if (oldVal !== undefined && newVal === undefined) {
+        changes.push({ templateName: afterTmpl.name, paramName: key, oldValue: oldVal });
+      } else if (oldVal !== newVal) {
+        changes.push({ templateName: afterTmpl.name, paramName: key, oldValue: oldVal, newValue: newVal });
+      }
+    }
+  }
+
+  return changes;
+}
+
+export function buildParamChangeEvents(
+  beforeTemplates: Template[],
+  afterTemplates: Template[],
+  fromRevId: number,
+  toRevId: number,
+  timestamp: string,
+): EvidenceEvent[] {
+  const changes = diffTemplateParams(beforeTemplates, afterTemplates);
+
+  return changes.map((c) => ({
+    eventType: "template_parameter_changed" as EvidenceEvent["eventType"],
+    fromRevisionId: fromRevId,
+    toRevisionId: toRevId,
+    section: "body",
+    before: c.oldValue ?? "",
+    after: c.newValue ?? "",
+    deterministicFacts: [
+      {
+        fact: "template_parameter_changed",
+        detail: `template=${c.templateName} param=${c.paramName}${c.oldValue !== undefined ? ` old=${c.oldValue.slice(0, 100)}` : ""}${c.newValue !== undefined ? ` new=${c.newValue.slice(0, 100)}` : ""}`,
+      },
+    ],
+    layer: "observed",
+    timestamp,
+  }));
+}
+
+function normalizeParams(params: Record<string, string>): Record<string, string | undefined> {
+  const normalized: Record<string, string | undefined> = {};
+  for (const [key, val] of Object.entries(params)) {
+    const nk = key.toLowerCase().trim();
+    if (val === "") continue;
+    normalized[nk] = val;
+  }
+  return normalized;
+}

package/src/wikilink-extractor.ts

@@ -0,0 +1,100 @@
+import type { DeterministicFact, EvidenceEvent } from "@refract-org/evidence-graph";
+
+const WIKILINK_REGEX = /\[\[([^\]]+?)(?:\|([^\]]*))?\]\]/g;
+
+const EXCLUDED_PREFIXES = [
+  "File:",
+  "Image:",
+  "Category:",
+  "wikipedia:",
+  "w:",
+  "mediawiki:",
+  "mw:",
+  "wiktionary:",
+  "wikt:",
+  "commons:",
+  "c:",
+  "d:",
+  "n:",
+  "q:",
+  "s:",
+  "v:",
+  "voy:",
+  "b:",
+];
+
+export function extractWikilinks(wikitext: string): string[] {
+  const links: string[] = [];
+  const seen = new Set<string>();
+  let match: RegExpExecArray | null;
+
+  // biome-ignore lint/suspicious/noAssignInExpressions: Standard regex loop pattern
+  while ((match = WIKILINK_REGEX.exec(wikitext)) !== null) {
+    const target = match[1].trim();
+
+    const isExcluded = EXCLUDED_PREFIXES.some((prefix) => target.toLowerCase().startsWith(prefix.toLowerCase()));
+    if (isExcluded) continue;
+
+    const normalized = target.toLowerCase().replace(/_/g, " ");
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    links.push(normalized);
+  }
+
+  return links;
+}
+
+export function diffWikilinks(before: string[], after: string[]): { added: string[]; removed: string[] } {
+  const beforeSet = new Set(before);
+  const afterSet = new Set(after);
+
+  return {
+    added: after.filter((w) => !beforeSet.has(w)),
+    removed: before.filter((w) => !afterSet.has(w)),
+  };
+}
+
+export function buildWikilinkEvents(
+  beforeWikitext: string,
+  afterWikitext: string,
+  fromRevId: number,
+  toRevId: number,
+  section: string,
+  timestamp: string,
+  extraFacts?: DeterministicFact[],
+): EvidenceEvent[] {
+  const events: EvidenceEvent[] = [];
+  const before = extractWikilinks(beforeWikitext);
+  const after = extractWikilinks(afterWikitext);
+  const { added, removed } = diffWikilinks(before, after);
+
+  for (const link of added) {
+    events.push({
+      eventType: "wikilink_added",
+      fromRevisionId: fromRevId,
+      toRevisionId: toRevId,
+      section,
+      before: "",
+      after: link,
+      deterministicFacts: [{ fact: "wikilink_added", detail: `target=${link}` }, ...(extraFacts ?? [])],
+      layer: "observed",
+      timestamp,
+    });
+  }
+
+  for (const link of removed) {
+    events.push({
+      eventType: "wikilink_removed",
+      fromRevisionId: fromRevId,
+      toRevisionId: toRevId,
+      section,
+      before: link,
+      after: "",
+      deterministicFacts: [{ fact: "wikilink_removed", detail: `target=${link}` }, ...(extraFacts ?? [])],
+      layer: "observed",
+      timestamp,
+    });
+  }
+
+  return events;
+}

package/src/wikitext-parser.ts

@@ -0,0 +1,92 @@
+export function sanitizeWikitext(value: string): string {
+  return value
+    .replace(/<!--[\s\S]*?-->/g, " ")
+    .replace(/<ref[^>/]*?>[\s\S]*?<\/ref>/gi, " ")
+    .replace(/<ref[^>]*/gi, (match) => (match.endsWith("/>") ? " " : match))
+    .replace(/<ref[^>]*\/>/gi, " ")
+    .replace(/\{\{[^}]*\}\}/g, " ")
+    .replace(/\[\[(?:[^|\]]*\|)?([^\]]+)\]\]/g, "$1")
+    .replace(/\[https?:\/\/[^\s\]]+\s*([^\]]*)\]/g, "$1")
+    .replace(/'''/g, "")
+    .replace(/''/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function stripWikitext(wikitext: string): string {
+  let text = wikitext;
+  text = text.replace(/<!--[\s\S]*?-->/g, "");
+  text = text.replace(/<ref\b[^>]*\/\s*>/gi, "");
+  text = text.replace(/<ref\b[^>]*>[\s\S]*?<\/ref\s*>/gi, "");
+  text = text.replace(/<[^>]+>/g, "");
+  text = text.replace(/\{\{[^{}]*?\}\}/g, "");
+  text = text.replace(/'''(.+?)'''/g, "$1");
+  text = text.replace(/''(.+?)''/g, "$1");
+  text = text.replace(/\[\[([^\]|]+?)\]\]/g, "$1");
+  text = text.replace(/\[\[[^\]]+?\|([^\]]+?)\]\]/g, "$1");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  return text.trim();
+}
+
+export interface HeadingPosition {
+  position: number;
+  heading: string;
+}
+
+export function extractHeadingMap(wikitext: string): HeadingPosition[] {
+  return Array.from(wikitext.matchAll(/^==+\s*(.*?)\s*==+\s*$/gm)).map((match) => ({
+    position: match.index ?? 0,
+    heading: match[1]?.trim() ?? "",
+  }));
+}
+
+export function deriveSectionHeading(wikitext: string, position: number): string | null {
+  let selected: string | null = null;
+  for (const heading of extractHeadingMap(wikitext)) {
+    if (heading.position > position) break;
+    selected = heading.heading;
+  }
+  return selected;
+}
+
+export function countCitations(wikitext: string): number {
+  return Array.from(wikitext.matchAll(/<ref\b/gi)).length;
+}
+
+export function countKeywordMentions(
+  wikitext: string,
+  phrases: string[],
+): { totalMentions: number; matchedPhrases: number } {
+  const lowered = wikitext.toLowerCase();
+  let totalMentions = 0;
+  let matchedPhrases = 0;
+  for (const phrase of phrases) {
+    const normalized = phrase.trim().toLowerCase();
+    if (!normalized) continue;
+    let count = 0;
+    let fromIndex = 0;
+    while (fromIndex < lowered.length) {
+      const idx = lowered.indexOf(normalized, fromIndex);
+      if (idx === -1) break;
+      count++;
+      fromIndex = idx + normalized.length;
+    }
+    totalMentions += count;
+    if (count > 0) matchedPhrases += 1;
+  }
+  return { totalMentions, matchedPhrases };
+}
+
+export function extractAnchorSnippet(wikitext: string, phrases: string[], radius = 200): string | null {
+  const lowered = wikitext.toLowerCase();
+  for (const phrase of phrases) {
+    const normalized = phrase.trim().toLowerCase();
+    if (!normalized) continue;
+    const idx = lowered.indexOf(normalized);
+    if (idx === -1) continue;
+    const start = Math.max(0, idx - radius);
+    const end = Math.min(wikitext.length, idx + normalized.length + radius);
+    return wikitext.slice(start, end).trim();
+  }
+  return null;
+}