@refract-org/analyzers 0.2.1

package/src/__tests__/wikitext-parser.test.ts

@@ -0,0 +1,142 @@
+import { describe, expect, it } from "vitest";
+import {
+  countCitations,
+  countKeywordMentions,
+  deriveSectionHeading,
+  extractAnchorSnippet,
+  extractHeadingMap,
+  sanitizeWikitext,
+} from "../wikitext-parser.js";
+
+const SAMPLE = `'''Bold text''' and ''italic''.
+== History ==
+The {{citation needed}} theory was first proposed in 2005.<ref name="smith2005">Smith (2005)</ref>
+
+<!-- This is a comment -->
+
+[[Internal link]] and [[Piped|Display]].
+
+== See also ==
+* [[Related topic]]
+
+== References ==
+{{reflist}}`;
+
+const HEADING_SAMPLE = `Lead text here.
+== First section ==
+Content.
+=== Subsection ===
+Deeper.
+== Second section ==
+More.`;
+
+describe("sanitizeWikitext", () => {
+  it("strips HTML comments", () => {
+    expect(sanitizeWikitext("before <!-- comment --> after")).not.toContain("comment");
+  });
+
+  it("strips ref tags", () => {
+    const result = sanitizeWikitext("text<ref>citation</ref>more");
+    expect(result).not.toContain("citation");
+    expect(result).toContain("text");
+    expect(result).toContain("more");
+  });
+
+  it("strips templates", () => {
+    expect(sanitizeWikitext("{{citation needed}}")).toBe("");
+  });
+
+  it("strips wikilinks, keeping display text", () => {
+    expect(sanitizeWikitext("[[Foo]] and [[Bar|baz]]")).toBe("Foo and baz");
+  });
+
+  it("strips bold and italic markers", () => {
+    expect(sanitizeWikitext("'''bold''' and ''italic''")).toBe("bold and italic");
+  });
+
+  it("collapses all whitespace sequences to single spaces", () => {
+    const result = sanitizeWikitext("a\n\n\n\nb");
+    expect(result).toBe("a b");
+  });
+});
+
+describe("extractHeadingMap", () => {
+  it("extracts headings with positions", () => {
+    const map = extractHeadingMap(HEADING_SAMPLE);
+    expect(map.length).toBeGreaterThanOrEqual(3);
+    expect(map[0].heading).toBe("First section");
+  });
+});
+
+describe("deriveSectionHeading", () => {
+  it("returns heading for a position in a section", () => {
+    const heading = deriveSectionHeading(HEADING_SAMPLE, HEADING_SAMPLE.indexOf("Content"));
+    expect(heading).toBe("First section");
+  });
+
+  it("returns null for lead area", () => {
+    const heading = deriveSectionHeading(HEADING_SAMPLE, HEADING_SAMPLE.indexOf("Lead text"));
+    expect(heading).toBeNull();
+  });
+
+  it("handles positions at the end of content", () => {
+    const heading = deriveSectionHeading(HEADING_SAMPLE, HEADING_SAMPLE.length - 1);
+    expect(heading).toBe("Second section");
+  });
+});
+
+describe("countCitations", () => {
+  it("counts ref tags, returning at least 1", () => {
+    const wikitext = "One<ref>a</ref> two<ref name='b'>c</ref> three<ref>d</ref>";
+    expect(countCitations(wikitext)).toBe(3);
+  });
+
+  it("counts combined refs", () => {
+    expect(countCitations(SAMPLE)).toBeGreaterThan(0);
+  });
+
+  it("returns 0 when there are no refs", () => {
+    expect(countCitations("Plain text with no citations.")).toBe(0);
+  });
+});
+
+describe("countKeywordMentions", () => {
+  it("counts mentions of a single phrase", () => {
+    const result = countKeywordMentions("Earth is the third planet. Earth has one moon.", ["Earth"]);
+    expect(result.totalMentions).toBe(2);
+    expect(result.matchedPhrases).toBe(1);
+  });
+
+  it("counts multiple phrases", () => {
+    const result = countKeywordMentions("Apple and orange are fruits. Apple is red.", ["Apple", "orange"]);
+    expect(result.totalMentions).toBe(3);
+    expect(result.matchedPhrases).toBe(2);
+  });
+
+  it("returns 0 for no matches", () => {
+    const result = countKeywordMentions("No relevant content here.", ["pineapple"]);
+    expect(result.totalMentions).toBe(0);
+    expect(result.matchedPhrases).toBe(0);
+  });
+});
+
+describe("extractAnchorSnippet", () => {
+  it("extracts surrounding context for a keyword", () => {
+    const text = "The Earth is the third planet from the Sun and the only known planet to support life.";
+    const snippet = extractAnchorSnippet(text, ["Earth"]);
+    expect(snippet).toBeTruthy();
+    expect(snippet?.length).toBeGreaterThan(0);
+  });
+
+  it("returns null if keyword not found", () => {
+    const snippet = extractAnchorSnippet("No relevant content here.", ["Nonexistent"]);
+    expect(snippet).toBeNull();
+  });
+
+  it("respects radius parameter", () => {
+    const text = `${"A".repeat(100)}TARGET${"B".repeat(100)}`;
+    const snippet = extractAnchorSnippet(text, ["TARGET"], 10);
+    expect(snippet).toBeTruthy();
+    expect(snippet?.length).toBeLessThan(50);
+  });
+});

package/src/category-tracker.ts

@@ -0,0 +1,75 @@
+import type { DeterministicFact, EvidenceEvent } from "@refract-org/evidence-graph";
+
+const CATEGORY_REGEX = /\[\[Category:([^\]|]+)(?:\|[^\]]*)?\]\]/gi;
+
+export function extractCategories(wikitext: string): string[] {
+  const categories: string[] = [];
+  const seen = new Set<string>();
+  let match: RegExpExecArray | null;
+
+  // biome-ignore lint/suspicious/noAssignInExpressions: Standard regex loop pattern
+  while ((match = CATEGORY_REGEX.exec(wikitext)) !== null) {
+    const name = match[1].trim();
+    if (!name) continue;
+    const normalized = name.toLowerCase().replace(/_/g, " ");
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    categories.push(normalized);
+  }
+
+  return categories;
+}
+
+export function diffCategories(before: string[], after: string[]): { added: string[]; removed: string[] } {
+  const beforeSet = new Set(before);
+  const afterSet = new Set(after);
+
+  return {
+    added: after.filter((c) => !beforeSet.has(c)),
+    removed: before.filter((c) => !afterSet.has(c)),
+  };
+}
+
+export function buildCategoryEvents(
+  beforeWikitext: string,
+  afterWikitext: string,
+  fromRevId: number,
+  toRevId: number,
+  timestamp: string,
+  extraFacts?: DeterministicFact[],
+): EvidenceEvent[] {
+  const events: EvidenceEvent[] = [];
+  const before = extractCategories(beforeWikitext);
+  const after = extractCategories(afterWikitext);
+  const { added, removed } = diffCategories(before, after);
+
+  for (const cat of added) {
+    events.push({
+      eventType: "category_added",
+      fromRevisionId: fromRevId,
+      toRevisionId: toRevId,
+      section: "",
+      before: "",
+      after: cat,
+      deterministicFacts: [{ fact: "category_added", detail: `category=${cat}` }, ...(extraFacts ?? [])],
+      layer: "observed",
+      timestamp,
+    });
+  }
+
+  for (const cat of removed) {
+    events.push({
+      eventType: "category_removed",
+      fromRevisionId: fromRevId,
+      toRevisionId: toRevId,
+      section: "",
+      before: cat,
+      after: "",
+      deterministicFacts: [{ fact: "category_removed", detail: `category=${cat}` }, ...(extraFacts ?? [])],
+      layer: "observed",
+      timestamp,
+    });
+  }
+
+  return events;
+}

package/src/citation-tracker.ts

@@ -0,0 +1,226 @@
+import { createHash } from "node:crypto";
+import type { SourceAuthority, SourceLineage, SourceRecord, SourceType } from "@refract-org/evidence-graph";
+import type { CitationChange, CitationRef, CitationTracker } from "./index.js";
+
+export const citationTracker: CitationTracker = {
+  extractCitations(wikitext: string): CitationRef[] {
+    const refs: CitationRef[] = [];
+    const seen = new Set<string>();
+
+    const refRegex = /<ref\b([^>]*?)>(.*?)<\/ref\s*>/gs;
+    let match: RegExpExecArray | null;
+
+    // biome-ignore lint/suspicious/noAssignInExpressions: Standard regex loop pattern
+    while ((match = refRegex.exec(wikitext)) !== null) {
+      const attrs = match[1];
+      const content = match[2].trim();
+
+      const nameMatch = attrs.match(/name\s*=\s*["']?([^"'\s>]+)/i);
+      const urlMatch = content.match(/url\s*=\s*([^\s|}\]]+)/i);
+      const titleMatch = content.match(/title\s*=\s*([^|}\]]+?)(?:\s*[|}\]])/i);
+
+      const raw = match[0];
+      const key = nameMatch ? nameMatch[1] : raw;
+
+      if (seen.has(key)) continue;
+      seen.add(key);
+
+      refs.push({
+        refName: nameMatch?.[1],
+        url: urlMatch ? urlMatch[1].trim() : undefined,
+        title: titleMatch ? titleMatch[1].trim() : undefined,
+        raw,
+      });
+    }
+
+    const selfClosingRegex = /<ref\b([^>]*?)\/\s*>/g;
+    // biome-ignore lint/suspicious/noAssignInExpressions: Standard regex loop pattern
+    while ((match = selfClosingRegex.exec(wikitext)) !== null) {
+      const attrs = match[1];
+      const nameMatch = attrs.match(/name\s*=\s*["']?([^"'\s>]+)/i);
+      if (!nameMatch) continue;
+
+      const key = nameMatch[1];
+      if (seen.has(key)) continue;
+      seen.add(key);
+
+      refs.push({
+        refName: key,
+        raw: match[0],
+      });
+    }
+
+    return refs;
+  },
+
+  diffCitations(before: CitationRef[], after: CitationRef[]): CitationChange[] {
+    const changes: CitationChange[] = [];
+    const beforeMap = indexByKey(before);
+    const afterMap = indexByKey(after);
+
+    for (const [key, afterRef] of afterMap) {
+      const beforeRef = beforeMap.get(key);
+      if (!beforeRef) {
+        changes.push({ type: "added", after: afterRef });
+      } else if (beforeRef.raw !== afterRef.raw) {
+        changes.push({ type: "replaced", before: beforeRef, after: afterRef });
+      } else {
+        changes.push({ type: "unchanged", after: afterRef });
+      }
+    }
+
+    for (const [key, beforeRef] of beforeMap) {
+      if (!afterMap.has(key)) {
+        changes.push({ type: "removed", before: beforeRef });
+      }
+    }
+
+    return changes;
+  },
+};
+
+function indexByKey(refs: CitationRef[]): Map<string, CitationRef> {
+  const map = new Map<string, CitationRef>();
+  for (const ref of refs) {
+    const key = ref.refName ?? ref.raw;
+    map.set(key, ref);
+  }
+  return map;
+}
+
+export function buildSourceLineage(revisions: { revId: number; timestamp: string; content: string }[]): {
+  sources: SourceRecord[];
+  lineage: SourceLineage[];
+} {
+  const sourceMap = new Map<string, SourceRecord>();
+  const replacementMap = new Map<string, { replacedById: string; atRevisionId: number; atTimestamp: string }[]>();
+
+  function ensureSource(ref: CitationRef, seenAtRevId: number, seenAtTimestamp: string): string {
+    const sourceId = buildSourceId(ref);
+    if (!sourceMap.has(sourceId)) {
+      sourceMap.set(sourceId, {
+        sourceId,
+        url: ref.url,
+        title: ref.title,
+        sourceType: classifySourceType(ref),
+        authority: classifyAuthority(ref),
+        firstSeenRevisionId: seenAtRevId,
+        firstSeenAt: seenAtTimestamp,
+        claimsReferencing: [],
+      });
+    }
+    return sourceId;
+  }
+
+  const allCitations = revisions.map((r) => citationTracker.extractCitations(r.content));
+
+  // Seed sources from the first revision
+  if (revisions.length > 0) {
+    for (const ref of allCitations[0]) {
+      ensureSource(ref, revisions[0].revId, revisions[0].timestamp);
+    }
+  }
+
+  for (let i = 0; i < revisions.length - 1; i++) {
+    const before = revisions[i];
+    const after = revisions[i + 1];
+
+    const beforeRefs = allCitations[i];
+    const afterRefs = allCitations[i + 1];
+    const changes = citationTracker.diffCitations(beforeRefs, afterRefs);
+
+    for (const change of changes) {
+      if (change.after) {
+        const id = ensureSource(change.after, after.revId, after.timestamp);
+        if (change.type === "replaced" && change.before) {
+          const oldId = ensureSource(change.before, after.revId, after.timestamp);
+          const replacements = replacementMap.get(oldId) ?? [];
+          replacements.push({
+            replacedById: id,
+            atRevisionId: after.revId,
+            atTimestamp: after.timestamp,
+          });
+          replacementMap.set(oldId, replacements);
+        }
+      }
+
+      if ((change.type === "removed" || change.type === "replaced") && change.before) {
+        const sourceId = ensureSource(change.before, before.revId, before.timestamp);
+        const record = sourceMap.get(sourceId);
+        if (record) {
+          record.lastSeenRevisionId = before.revId;
+          record.lastSeenAt = before.timestamp;
+        }
+      }
+    }
+  }
+
+  const sources = Array.from(sourceMap.values());
+  const lineage: SourceLineage[] = [];
+  for (const [sourceId, replacements] of replacementMap) {
+    lineage.push({ sourceId, replacements });
+  }
+
+  return { sources, lineage };
+}
+
+export function buildSourceId(ref: CitationRef): string {
+  if (ref.url) {
+    return createHash("sha256").update(ref.url).digest("hex").slice(0, 16);
+  }
+  if (ref.refName) {
+    return createHash("sha256").update(`ref:${ref.refName}`).digest("hex").slice(0, 16);
+  }
+  return createHash("sha256").update(ref.raw).digest("hex").slice(0, 16);
+}
+
+const NEWS_DOMAINS = [
+  "cnn.com",
+  "nytimes.com",
+  "bbc.com",
+  "reuters.com",
+  "apnews.com",
+  "washingtonpost.com",
+  "wsj.com",
+  "theguardian.com",
+  "bloomberg.com",
+  "npr.org",
+  "thehill.com",
+  "politico.com",
+  "foxnews.com",
+  "nbcnews.com",
+  "cbsnews.com",
+  "abcnews.net",
+  "usatoday.com",
+  "latimes.com",
+  "chicagotribune.com",
+  "huffpost.com",
+  "buzzfeednews.com",
+];
+
+function classifySourceType(ref: CitationRef): SourceType {
+  const url = ref.url?.toLowerCase() ?? "";
+  if (!url) return "unknown";
+
+  if (url.includes("doi.org") || /journal|jstor|springer|sciencedirect/i.test(url)) {
+    return "academic";
+  }
+  if (url.includes(".gov")) return "government";
+  if (url.includes(".edu")) return "secondary";
+  if (NEWS_DOMAINS.some((d) => url.includes(d))) return "news";
+
+  return "unknown";
+}
+
+function classifyAuthority(ref: CitationRef): SourceAuthority {
+  const url = ref.url?.toLowerCase() ?? "";
+  if (!url) return "unrated";
+
+  if (url.includes("doi.org") || /journal|jstor|springer/i.test(url)) {
+    return "medium";
+  }
+  if (/\.(edu|gov|org)\b/.test(url)) return "high";
+  if (/\.(com|net)\b/.test(url)) return "medium";
+
+  return "unrated";
+}

package/src/claim-differ.ts

@@ -0,0 +1,4 @@
+// claim-differ has been removed.
+// Text comparison utilities (stripWikitext, fuzzyFindClaim) moved to wikitext-parser.ts.
+// Semantic direction classification (classifyClaimChange) removed — Refract observes mechanical
+// text appearance/disappearance, not semantic interpretation of direction.

package/src/edit-cluster-detector.ts

@@ -0,0 +1,78 @@
+import type { EvidenceEvent, Revision } from "@refract-org/evidence-graph";
+
+const DEFAULT_WINDOW_MS = 60 * 60 * 1000; // 1 hour
+const DEFAULT_MIN_CLUSTER_SIZE = 3;
+
+export interface EditClusterOptions {
+  windowMs?: number;
+  minClusterSize?: number;
+}
+
+export interface EditCluster {
+  revisionIds: number[];
+  timestamp: string;
+  editor?: string;
+  section?: string;
+  eventCount: number;
+}
+
+export function detectEditClusters(revisions: Revision[], options?: EditClusterOptions): EvidenceEvent[] {
+  const windowMs = options?.windowMs ?? DEFAULT_WINDOW_MS;
+  const minSize = options?.minClusterSize ?? DEFAULT_MIN_CLUSTER_SIZE;
+  const events: EvidenceEvent[] = [];
+
+  if (revisions.length < minSize) return events;
+
+  const sorted = [...revisions].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+
+  const clustered = new Set<number>();
+
+  for (let i = 0; i < sorted.length; i++) {
+    if (clustered.has(i)) continue;
+
+    const windowStart = new Date(sorted[i].timestamp).getTime();
+    const windowEnd = windowStart + windowMs;
+    const cluster: number[] = [];
+
+    for (let j = i; j < sorted.length; j++) {
+      const t = new Date(sorted[j].timestamp).getTime();
+      if (t <= windowEnd) {
+        cluster.push(j);
+      } else {
+        break;
+      }
+    }
+
+    if (cluster.length >= minSize) {
+      const hasSingleEditor = singleEditorCluster(sorted, cluster);
+      const _revIds = cluster.map((idx) => sorted[idx].revId);
+
+      for (const idx of cluster) clustered.add(idx);
+
+      events.push({
+        eventType: "edit_cluster_detected",
+        fromRevisionId: sorted[cluster[0]].revId,
+        toRevisionId: sorted[cluster[cluster.length - 1]].revId,
+        section: "",
+        before: "",
+        after: "",
+        deterministicFacts: [
+          {
+            fact: "edit_cluster",
+            detail: `revisions=${cluster.length} window_ms=${windowMs} single_editor=${hasSingleEditor}`,
+          },
+        ],
+        layer: "observed",
+        timestamp: sorted[cluster[0]].timestamp,
+      });
+    }
+  }
+
+  return events;
+}
+
+function singleEditorCluster(revisions: Revision[], indices: number[]): boolean {
+  const firstUser = revisions[indices[0]].user;
+  if (!firstUser) return false;
+  return indices.every((i) => revisions[i].user === firstUser);
+}

package/src/heuristic-classifier.ts

@@ -0,0 +1,59 @@
+export type HeuristicKind =
+  | "revert"
+  | "vandalism"
+  | "major_addition"
+  | "major_removal"
+  | "sourcing"
+  | "cosmetic"
+  | "minor"
+  | "unknown";
+
+export interface HeuristicOptions {
+  majorAdditionThreshold?: number;
+  majorRemovalThreshold?: number;
+  cosmeticThreshold?: number;
+  minorThreshold?: number;
+}
+
+const VANDALISM_PATTERNS = /\b(vandal|vandalism|spam|blanking|test edit)\b/i;
+const SOURCING_PATTERNS = /\b(cite|ref|source|reference|citation|add ref|rm ref)\b/i;
+const REVERT_PATTERNS = /\b(rv|revert|reverted|undo|undid|rollback|rvv)\b/i;
+
+const DEFAULT_MAJOR_ADDITION = 2000;
+const DEFAULT_MAJOR_REMOVAL = -2000;
+const DEFAULT_COSMETIC = 20;
+const DEFAULT_MINOR = 100;
+
+export function classifyHeuristic(comment: string, sizeDelta: number, options?: HeuristicOptions): HeuristicKind {
+  const norm = comment.toLowerCase().trim();
+
+  if (REVERT_PATTERNS.test(norm)) {
+    return "revert";
+  }
+
+  if (VANDALISM_PATTERNS.test(norm)) {
+    return "vandalism";
+  }
+
+  if (SOURCING_PATTERNS.test(norm)) {
+    return "sourcing";
+  }
+
+  if (sizeDelta > (options?.majorAdditionThreshold ?? DEFAULT_MAJOR_ADDITION)) {
+    return "major_addition";
+  }
+
+  if (sizeDelta < (options?.majorRemovalThreshold ?? DEFAULT_MAJOR_REMOVAL)) {
+    return "major_removal";
+  }
+
+  if (Math.abs(sizeDelta) < (options?.cosmeticThreshold ?? DEFAULT_COSMETIC) && !norm) {
+    return "cosmetic";
+  }
+
+  if (Math.abs(sizeDelta) < (options?.minorThreshold ?? DEFAULT_MINOR)) {
+    return "minor";
+  }
+
+  return "unknown";
+}

package/src/index.ts

@@ -0,0 +1,88 @@
+import type { Revision, Section, SectionChange } from "@refract-org/evidence-graph";
+
+export interface SectionDiffer {
+  extractSections(wikitext: string): Section[];
+  diffSections(before: Section[], after: Section[]): SectionChange[];
+}
+
+export interface CitationTracker {
+  extractCitations(wikitext: string): CitationRef[];
+  diffCitations(before: CitationRef[], after: CitationRef[]): CitationChange[];
+}
+
+export interface CitationRef {
+  url?: string;
+  title?: string;
+  refName?: string;
+  raw: string;
+}
+
+export interface CitationChange {
+  type: "added" | "removed" | "replaced" | "unchanged";
+  before?: CitationRef;
+  after?: CitationRef;
+}
+
+export interface RevertDetector {
+  isRevert(comment: string): boolean;
+  detectRevertChain(revisions: Revision[]): RevertChain[];
+}
+
+export interface RevertChain {
+  startRevisionId: number;
+  endRevisionId: number;
+  revertedToRevisionId: number;
+  participants: number;
+}
+
+export interface TemplateTracker {
+  extractTemplates(wikitext: string): Template[];
+  diffTemplates(before: Template[], after: Template[]): TemplateChange[];
+}
+
+export interface Template {
+  name: string; // e.g., "Citation needed", "NPOV", "BLP"
+  type: TemplateType;
+  params?: Record<string, string>;
+}
+
+export type TemplateType = "citation" | "neutrality" | "blp" | "dispute" | "cleanup" | "protection" | "other";
+
+export interface TemplateChange {
+  type: "added" | "removed" | "unchanged";
+  template: Template;
+}
+
+export { buildCategoryEvents, diffCategories, extractCategories } from "./category-tracker.js";
+export { buildSourceId, buildSourceLineage, citationTracker } from "./citation-tracker.js";
+export type { EditClusterOptions } from "./edit-cluster-detector.js";
+export { detectEditClusters } from "./edit-cluster-detector.js";
+export type { HeuristicKind, HeuristicOptions } from "./heuristic-classifier.js";
+export { classifyHeuristic } from "./heuristic-classifier.js";
+export type { ObservationDiff } from "./observation-differ.js";
+export { diffObservations } from "./observation-differ.js";
+export { buildPageMoveEvents } from "./page-move-detector.js";
+export type { ProtectionChange, ProtectionTracker } from "./protection-tracker.js";
+export { protectionTracker } from "./protection-tracker.js";
+export { revertDetector } from "./revert-detector.js";
+export type { SectionEvent, SectionLineage } from "./section-differ.js";
+export { buildSectionLineage, sectionDiffer } from "./section-differ.js";
+export type { TalkActivityOptions, TalkActivityResult } from "./talk-activity-detector.js";
+export { detectTalkActivitySpikes } from "./talk-activity-detector.js";
+export type { TalkCorrelationOptions } from "./talk-correlator.js";
+export { correlateTalkRevisions } from "./talk-correlator.js";
+export type { TalkReply, TalkThread, TalkThreadChange } from "./talk-section-parser.js";
+export { buildTalkThreadEvents, diffTalkThreads, parseTalkThreads } from "./talk-section-parser.js";
+export type { ParamChange } from "./template-tracker.js";
+export { buildParamChangeEvents, diffTemplateParams, templateTracker } from "./template-tracker.js";
+export { buildWikilinkEvents, diffWikilinks, extractWikilinks } from "./wikilink-extractor.js";
+export type { HeadingPosition } from "./wikitext-parser.js";
+export {
+  countCitations,
+  countKeywordMentions,
+  deriveSectionHeading,
+  extractAnchorSnippet,
+  extractHeadingMap,
+  sanitizeWikitext,
+  stripWikitext,
+} from "./wikitext-parser.js";