@refract-org/analyzers 0.2.1

package/src/observation-differ.ts

@@ -0,0 +1,26 @@
+import type { EvidenceEvent } from "@refract-org/evidence-graph";
+
+export interface ObservationDiff {
+  new: EvidenceEvent[];
+  resolved: EvidenceEvent[];
+  unchanged: EvidenceEvent[];
+}
+
+function eventKey(event: EvidenceEvent): string {
+  return `${event.eventType}|${event.fromRevisionId}|${event.toRevisionId}|${event.section}`;
+}
+
+export function diffObservations(prior: EvidenceEvent[], current: EvidenceEvent[]): ObservationDiff {
+  const priorKeys = new Set(prior.map(eventKey));
+  const currentKeys = new Set(current.map(eventKey));
+
+  const newEvents = current.filter((e) => !priorKeys.has(eventKey(e)));
+  const unchangedEvents = current.filter((e) => priorKeys.has(eventKey(e)));
+  const resolvedEvents = prior.filter((e) => !currentKeys.has(eventKey(e)));
+
+  return {
+    new: newEvents,
+    resolved: resolvedEvents,
+    unchanged: unchangedEvents,
+  };
+}

package/src/page-move-detector.ts

@@ -0,0 +1,32 @@
+import type { EvidenceEvent, EvidenceLayer } from "@refract-org/evidence-graph";
+
+interface PageMoveRecord {
+  oldTitle: string;
+  newTitle: string;
+  timestamp: string;
+  revId: number;
+  comment: string;
+}
+
+export function buildPageMoveEvents(moves: PageMoveRecord[]): EvidenceEvent[] {
+  const events: EvidenceEvent[] = [];
+
+  for (const move of moves) {
+    const layer: EvidenceLayer = "observed";
+    events.push({
+      eventType: "page_moved",
+      fromRevisionId: 0,
+      toRevisionId: move.revId,
+      section: "",
+      before: move.oldTitle,
+      after: move.newTitle,
+      deterministicFacts: [
+        { fact: "page_moved", detail: `from=${move.oldTitle} to=${move.newTitle} comment=${move.comment}` },
+      ],
+      layer,
+      timestamp: move.timestamp,
+    });
+  }
+
+  return events;
+}

package/src/protection-tracker.ts

@@ -0,0 +1,103 @@
+export interface ProtectionLogRecord {
+  logId: number;
+  pageTitle: string;
+  timestamp: string;
+  comment: string;
+  action: "protect" | "unprotect" | "modify";
+}
+
+export interface ProtectionState {
+  level: string;
+  sinceTimestamp: string;
+  sinceLogId: number;
+}
+
+export interface ProtectionChange {
+  type: "added" | "removed" | "modified";
+  logEvent: ProtectionLogRecord;
+}
+
+export interface ProtectionTracker {
+  buildState(logs: ProtectionLogRecord[]): Map<string, ProtectionState>;
+  diffState(before: Map<string, ProtectionState>, after: Map<string, ProtectionState>): ProtectionChange[];
+  findLogsBetween(logs: ProtectionLogRecord[], fromTimestamp: string, toTimestamp: string): ProtectionLogRecord[];
+}
+
+export const protectionTracker: ProtectionTracker = {
+  buildState(logs: ProtectionLogRecord[]): Map<string, ProtectionState> {
+    const state = new Map<string, ProtectionState>();
+    const sorted = [...logs].map((l) => ({ l, ts: new Date(l.timestamp).getTime() }));
+    sorted.sort((a, b) => a.ts - b.ts);
+
+    for (const { l: log } of sorted) {
+      if (log.action === "protect" || log.action === "modify") {
+        state.set(log.pageTitle, {
+          level: log.action === "protect" ? "protected" : "modified",
+          sinceTimestamp: log.timestamp,
+          sinceLogId: log.logId,
+        });
+      } else if (log.action === "unprotect") {
+        state.delete(log.pageTitle);
+      }
+    }
+
+    return state;
+  },
+
+  diffState(before: Map<string, ProtectionState>, after: Map<string, ProtectionState>): ProtectionChange[] {
+    const changes: ProtectionChange[] = [];
+    const allTitles = new Set([...before.keys(), ...after.keys()]);
+
+    for (const title of allTitles) {
+      const b = before.get(title);
+      const a = after.get(title);
+
+      if (!b && a) {
+        changes.push({
+          type: "added",
+          logEvent: {
+            logId: a.sinceLogId,
+            pageTitle: title,
+            timestamp: a.sinceTimestamp,
+            comment: "",
+            action: "protect",
+          },
+        });
+      } else if (b && !a) {
+        changes.push({
+          type: "removed",
+          logEvent: {
+            logId: b.sinceLogId,
+            pageTitle: title,
+            timestamp: b.sinceTimestamp,
+            comment: "",
+            action: "unprotect",
+          },
+        });
+      } else if (b && a && b.sinceLogId !== a.sinceLogId) {
+        changes.push({
+          type: "modified",
+          logEvent: {
+            logId: a.sinceLogId,
+            pageTitle: title,
+            timestamp: a.sinceTimestamp,
+            comment: "",
+            action: "modify",
+          },
+        });
+      }
+    }
+
+    return changes;
+  },
+
+  findLogsBetween(logs: ProtectionLogRecord[], fromTimestamp: string, toTimestamp: string): ProtectionLogRecord[] {
+    const from = new Date(fromTimestamp).getTime();
+    const to = new Date(toTimestamp).getTime();
+
+    return logs.filter((l) => {
+      const t = new Date(l.timestamp).getTime();
+      return t > from && t <= to;
+    });
+  },
+};

package/src/revert-detector.ts

@@ -0,0 +1,51 @@
+import type { Revision } from "@refract-org/evidence-graph";
+import type { RevertChain, RevertDetector } from "./index.js";
+
+const REVERT_PATTERNS = [
+  /\brevert/i,
+  /\bundid\s+revision/i,
+  /\brvv\b/i,
+  /\brollback\b/i,
+  /\brestore/i,
+  /\[\[WP:ROLLBACK\]\]/i,
+];
+
+export const revertDetector: RevertDetector = {
+  isRevert(comment: string): boolean {
+    return REVERT_PATTERNS.some((pattern) => pattern.test(comment));
+  },
+
+  detectRevertChain(revisions: Revision[]): RevertChain[] {
+    const chains: RevertChain[] = [];
+    const sorted = [...revisions].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+
+    let i = 0;
+    while (i < sorted.length) {
+      const rev = sorted[i];
+      if (!this.isRevert(rev.comment)) {
+        i++;
+        continue;
+      }
+
+      const existing = chains.find((c) => rev.revId <= c.endRevisionId + 1 && rev.revId >= c.startRevisionId - 1);
+
+      if (existing) {
+        existing.startRevisionId = Math.min(existing.startRevisionId, rev.revId);
+        existing.endRevisionId = Math.max(existing.endRevisionId, rev.revId);
+        existing.participants++;
+      } else {
+        const revertedToMatch = rev.comment.match(/revision\s+(\d+)/i);
+        chains.push({
+          startRevisionId: rev.revId,
+          endRevisionId: rev.revId,
+          revertedToRevisionId: revertedToMatch ? parseInt(revertedToMatch[1], 10) : rev.revId - 1,
+          participants: 1,
+        });
+      }
+
+      i++;
+    }
+
+    return chains;
+  },
+};

package/src/section-differ.ts

@@ -0,0 +1,315 @@
+import type { Section, SectionChange } from "@refract-org/evidence-graph";
+import type { SectionDiffer } from "./index.js";
+
+export const sectionDiffer: SectionDiffer = {
+  extractSections(wikitext: string): Section[] {
+    const sections: Section[] = [];
+    const lines = wikitext.split("\n");
+    const headerRegex = /^(=+)\s*([^=]+?)\s*\1$/;
+
+    // Pre-compute byte offsets of each line from a single TextEncoder pass
+    const bytes = new TextEncoder().encode(wikitext);
+    const lineByteOffsets = [0];
+    for (let i = 0, lineIdx = 0; i < bytes.length; i++) {
+      if (bytes[i] === 0x0a) {
+        lineByteOffsets[++lineIdx] = i + 1;
+      }
+    }
+    while (lineByteOffsets.length <= lines.length) {
+      lineByteOffsets.push(bytes.length);
+    }
+
+    const headerMatches: Array<{
+      index: number;
+      offset: number;
+      level: number;
+      title: string;
+    }> = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      const match = headerRegex.exec(lines[i]);
+      if (match) {
+        headerMatches.push({
+          index: i,
+          offset: lineByteOffsets[i],
+          level: match[1].length,
+          title: match[2].trim(),
+        });
+      }
+    }
+
+    if (headerMatches.length === 0) {
+      sections.push({
+        title: "",
+        level: 1,
+        content: wikitext.trim(),
+        byteOffset: 0,
+      });
+      return sections;
+    }
+
+    const leadEnd = headerMatches[0].offset;
+    const leadContent = wikitext.slice(0, leadEnd).trim();
+    if (leadContent) {
+      sections.push({
+        title: "",
+        level: 1,
+        content: leadContent,
+        byteOffset: 0,
+      });
+    }
+
+    for (let i = 0; i < headerMatches.length; i++) {
+      const header = headerMatches[i];
+      const nextOffset = i + 1 < headerMatches.length ? headerMatches[i + 1].offset : bytes.length;
+      const headerEnd = lineByteOffsets[header.index + 1] ?? bytes.length;
+      const content = wikitext.slice(headerEnd, nextOffset).trim();
+
+      sections.push({
+        title: header.title,
+        level: header.level,
+        content,
+        byteOffset: header.offset,
+      });
+    }
+
+    return sections;
+  },
+
+  diffSections(before: Section[], after: Section[]): SectionChange[] {
+    const changes: SectionChange[] = [];
+    const beforeByTitle = new Map<string, Section>();
+    const afterByTitle = new Map<string, Section>();
+
+    for (const s of before) {
+      const key = sectionKey(s);
+      beforeByTitle.set(key, s);
+    }
+    for (const s of after) {
+      const key = sectionKey(s);
+      afterByTitle.set(key, s);
+    }
+
+    const seenAfter = new Set<string>();
+
+    for (const [key, afterSection] of afterByTitle) {
+      seenAfter.add(key);
+      const beforeSection = beforeByTitle.get(key);
+      if (!beforeSection) {
+        changes.push({
+          section: afterSection.title || "(lead)",
+          changeType: "added",
+          toContent: afterSection.content,
+        });
+      } else if (beforeSection.content !== afterSection.content) {
+        changes.push({
+          section: afterSection.title || "(lead)",
+          changeType: "modified",
+          fromContent: beforeSection.content,
+          toContent: afterSection.content,
+        });
+      } else {
+        changes.push({
+          section: afterSection.title || "(lead)",
+          changeType: "unchanged",
+        });
+      }
+    }
+
+    for (const [key, beforeSection] of beforeByTitle) {
+      if (!seenAfter.has(key)) {
+        changes.push({
+          section: beforeSection.title || "(lead)",
+          changeType: "removed",
+          fromContent: beforeSection.content,
+        });
+      }
+    }
+
+    return changes;
+  },
+};
+
+export interface SectionEvent {
+  revisionId: number;
+  timestamp: string;
+  eventType: "created" | "modified" | "removed" | "renamed";
+  content?: string;
+  oldName?: string;
+  newName?: string;
+}
+
+export interface SectionLineage {
+  sectionName: string;
+  level: number;
+  firstSeenRevisionId: number;
+  firstSeenAt: string;
+  lastSeenRevisionId?: number;
+  lastSeenAt?: string;
+  events: SectionEvent[];
+  isActive: boolean;
+}
+
+export function buildSectionLineage(
+  revisions: Array<{ revId: number; timestamp: string; content: string }>,
+): SectionLineage[] {
+  if (revisions.length === 0) return [];
+
+  const allSections = revisions.map((r) => sectionDiffer.extractSections(r.content));
+
+  const lineages = new Map<string, SectionLineage>();
+
+  const firstSections = allSections[0];
+  for (const section of firstSections) {
+    const key = sectionKey(section);
+    lineages.set(key, {
+      sectionName: section.title || "(lead)",
+      level: section.level,
+      firstSeenRevisionId: revisions[0].revId,
+      firstSeenAt: revisions[0].timestamp,
+      lastSeenRevisionId: revisions[0].revId,
+      lastSeenAt: revisions[0].timestamp,
+      events: [
+        {
+          revisionId: revisions[0].revId,
+          timestamp: revisions[0].timestamp,
+          eventType: "created",
+          content: section.content,
+        },
+      ],
+      isActive: true,
+    });
+  }
+
+  for (let i = 0; i < revisions.length - 1; i++) {
+    const prevRev = revisions[i];
+    const currRev = revisions[i + 1];
+
+    const prevSections = allSections[i];
+    const currSections = allSections[i + 1];
+
+    const prevByKey = new Map<string, Section>();
+    const currByKey = new Map<string, Section>();
+    for (const s of prevSections) prevByKey.set(sectionKey(s), s);
+    for (const s of currSections) currByKey.set(sectionKey(s), s);
+
+    const prevKeys = new Set(prevByKey.keys());
+    const currKeys = new Set(currByKey.keys());
+
+    const removedKeys = [...prevKeys].filter((k) => !currKeys.has(k));
+    const addedKeys = [...currKeys].filter((k) => !prevKeys.has(k));
+
+    const renamedFromTo = new Map<string, string>();
+    const contentToAddKey = new Map<string, string>();
+    for (const addKey of addedKeys) {
+      const section = currByKey.get(addKey);
+      if (!section) continue;
+      contentToAddKey.set(section.content, addKey);
+    }
+    for (const remKey of removedKeys) {
+      const remSection = prevByKey.get(remKey);
+      if (!remSection) continue;
+      const addKey = contentToAddKey.get(remSection.content);
+      if (addKey) {
+        renamedFromTo.set(remKey, addKey);
+      }
+    }
+
+    const renamedToSet = new Set(renamedFromTo.values());
+
+    for (const key of removedKeys) {
+      if (renamedFromTo.has(key)) {
+        const newKey = renamedFromTo.get(key);
+        const oldSection = prevByKey.get(key);
+        const newSection = newKey ? currByKey.get(newKey) : undefined;
+        if (!newKey || !oldSection || !newSection) continue;
+        const lineage = lineages.get(key);
+        if (lineage) {
+          lineage.events.push({
+            revisionId: currRev.revId,
+            timestamp: currRev.timestamp,
+            eventType: "renamed",
+            content: oldSection.content,
+            oldName: oldSection.title || "(lead)",
+            newName: newSection.title || "(lead)",
+          });
+          lineage.sectionName = newSection.title || "(lead)";
+          lineage.level = newSection.level;
+          lineage.lastSeenRevisionId = currRev.revId;
+          lineage.lastSeenAt = currRev.timestamp;
+          lineage.isActive = true;
+          lineages.set(newKey, lineage);
+          lineages.delete(key);
+        }
+      } else {
+        const section = prevByKey.get(key);
+        if (!section) continue;
+        const lineage = lineages.get(key);
+        if (lineage) {
+          lineage.events.push({
+            revisionId: currRev.revId,
+            timestamp: currRev.timestamp,
+            eventType: "removed",
+            content: section.content,
+          });
+          lineage.lastSeenRevisionId = prevRev.revId;
+          lineage.lastSeenAt = prevRev.timestamp;
+          lineage.isActive = false;
+        }
+      }
+    }
+
+    for (const key of addedKeys) {
+      if (!renamedToSet.has(key)) {
+        const section = currByKey.get(key);
+        if (!section) continue;
+        const lineage: SectionLineage = {
+          sectionName: section.title || "(lead)",
+          level: section.level,
+          firstSeenRevisionId: currRev.revId,
+          firstSeenAt: currRev.timestamp,
+          lastSeenRevisionId: currRev.revId,
+          lastSeenAt: currRev.timestamp,
+          events: [
+            {
+              revisionId: currRev.revId,
+              timestamp: currRev.timestamp,
+              eventType: "created",
+              content: section.content,
+            },
+          ],
+          isActive: true,
+        };
+        lineages.set(key, lineage);
+      }
+    }
+
+    for (const key of prevKeys) {
+      if (currKeys.has(key) && !renamedFromTo.has(key)) {
+        const prevSection = prevByKey.get(key);
+        const currSection = currByKey.get(key);
+        if (!prevSection || !currSection) continue;
+        const lineage = lineages.get(key);
+        if (lineage) {
+          if (prevSection.content !== currSection.content) {
+            lineage.events.push({
+              revisionId: currRev.revId,
+              timestamp: currRev.timestamp,
+              eventType: "modified",
+              content: currSection.content,
+            });
+          }
+          lineage.lastSeenRevisionId = currRev.revId;
+          lineage.lastSeenAt = currRev.timestamp;
+          lineage.isActive = true;
+        }
+      }
+    }
+  }
+
+  return [...lineages.values()].sort((a, b) => a.sectionName.localeCompare(b.sectionName));
+}
+
+function sectionKey(s: Section): string {
+  return `${s.level}:${s.title.toLowerCase()}`;
+}

package/src/talk-activity-detector.ts

@@ -0,0 +1,105 @@
+import type { EvidenceEvent, Revision } from "@refract-org/evidence-graph";
+
+const DEFAULT_SPIKE_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+const DEFAULT_SPIKE_FACTOR = 3.0;
+const DEFAULT_MA_PERIODS = 4;
+
+export interface TalkActivityOptions {
+  lookbackWindowMs?: number;
+  spikeFactor?: number;
+  movingAveragePeriods?: number;
+}
+
+export interface TalkActivityResult {
+  spikes: EvidenceEvent[];
+  activityByDay: Array<{ date: string; count: number }>;
+  movingAverage: number;
+}
+
+export function detectTalkActivitySpikes(
+  talkRevisions: Revision[],
+  articleRevisions: Revision[],
+  options?: TalkActivityOptions,
+): TalkActivityResult {
+  const windowMs = options?.lookbackWindowMs ?? DEFAULT_SPIKE_WINDOW_MS;
+  const spikeFactor = options?.spikeFactor ?? DEFAULT_SPIKE_FACTOR;
+  const maPeriods = options?.movingAveragePeriods ?? DEFAULT_MA_PERIODS;
+  const spikes: EvidenceEvent[] = [];
+
+  if (talkRevisions.length === 0) {
+    return { spikes, activityByDay: [], movingAverage: 0 };
+  }
+
+  const dailyCounts = bucketByDay(talkRevisions);
+  const sortedDays = Object.keys(dailyCounts).sort();
+
+  if (sortedDays.length < maPeriods) {
+    return { spikes, activityByDay: dailyActivity(dailyCounts), movingAverage: 0 };
+  }
+
+  const recentEnd = Date.now();
+  const recentStart = recentEnd - windowMs;
+  const recentDays = sortedDays.filter((d) => new Date(d).getTime() >= recentStart);
+
+  const movingAverages: number[] = [];
+  for (let i = maPeriods - 1; i < sortedDays.length; i++) {
+    let sum = 0;
+    for (let j = i - (maPeriods - 1); j <= i; j++) {
+      sum += dailyCounts[sortedDays[j]];
+    }
+    movingAverages.push(sum / maPeriods);
+  }
+
+  const latestMA = movingAverages.length > 0 ? movingAverages[movingAverages.length - 1] : 0;
+  const threshold = Math.max(latestMA * spikeFactor, 3);
+
+  for (const day of recentDays) {
+    const count = dailyCounts[day];
+    if (count >= threshold) {
+      const nearbyArticleEdits = articleRevisions.filter((r) => {
+        const t = new Date(r.timestamp);
+        const dayStart = new Date(day).getTime();
+        const dayEnd = dayStart + 24 * 60 * 60 * 1000;
+        return t.getTime() >= dayStart && t.getTime() < dayEnd;
+      });
+
+      spikes.push({
+        eventType: "talk_activity_spike",
+        fromRevisionId: 0,
+        toRevisionId: 0,
+        section: "",
+        before: "",
+        after: "",
+        deterministicFacts: [
+          {
+            fact: "talk_activity_spike",
+            detail: `date=${day} talk_edits=${count} moving_average=${latestMA.toFixed(1)} threshold=${threshold.toFixed(1)} nearby_article_edits=${nearbyArticleEdits.length}`,
+          },
+        ],
+        layer: "observed",
+        timestamp: new Date(day).toISOString(),
+      });
+    }
+  }
+
+  return {
+    spikes,
+    activityByDay: dailyActivity(dailyCounts),
+    movingAverage: latestMA,
+  };
+}
+
+function bucketByDay(revisions: Revision[]): Record<string, number> {
+  const buckets: Record<string, number> = {};
+  for (const r of revisions) {
+    const day = r.timestamp.slice(0, 10);
+    buckets[day] = (buckets[day] ?? 0) + 1;
+  }
+  return buckets;
+}
+
+function dailyActivity(counts: Record<string, number>): Array<{ date: string; count: number }> {
+  return Object.entries(counts)
+    .map(([date, count]) => ({ date, count }))
+    .sort((a, b) => a.date.localeCompare(b.date));
+}

package/src/talk-correlator.ts

@@ -0,0 +1,70 @@
+import type { EvidenceEvent, Revision } from "@refract-org/evidence-graph";
+
+const DEFAULT_WINDOW_BEFORE_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_WINDOW_AFTER_MS = 3 * 24 * 60 * 60 * 1000;
+
+export interface TalkCorrelationOptions {
+  windowBeforeMs?: number;
+  windowAfterMs?: number;
+}
+
+export function correlateTalkRevisions(
+  articleRevs: Revision[],
+  talkRevs: Revision[],
+  options?: TalkCorrelationOptions,
+): EvidenceEvent[] {
+  const windowBefore = options?.windowBeforeMs ?? DEFAULT_WINDOW_BEFORE_MS;
+  const windowAfter = options?.windowAfterMs ?? DEFAULT_WINDOW_AFTER_MS;
+  const events: EvidenceEvent[] = [];
+
+  if (articleRevs.length === 0 || talkRevs.length === 0) return events;
+
+  const sortedTalk = [...talkRevs].map((r) => ({ r, ts: new Date(r.timestamp).getTime() }));
+  sortedTalk.sort((a, b) => a.ts - b.ts);
+
+  const articleTimes = articleRevs.map((r) => new Date(r.timestamp).getTime());
+
+  for (let a = 0; a < articleRevs.length; a++) {
+    const article = articleRevs[a];
+    const articleTime = articleTimes[a];
+    const windowStart = articleTime - windowBefore;
+    const windowEnd = articleTime + windowAfter;
+
+    let closest: Revision | null = null;
+    let closestDelta = Infinity;
+
+    for (let t = 0; t < sortedTalk.length; t++) {
+      const talkTime = sortedTalk[t].ts;
+      if (talkTime < windowStart) continue;
+      if (talkTime > windowEnd) break;
+
+      const delta = Math.abs(talkTime - articleTime);
+      if (delta < closestDelta) {
+        closestDelta = delta;
+        closest = sortedTalk[t].r;
+      }
+    }
+
+    if (closest) {
+      const deltaHours = Math.round((closestDelta / (1000 * 60 * 60)) * 10) / 10;
+      events.push({
+        eventType: "talk_page_correlated",
+        fromRevisionId: article.revId,
+        toRevisionId: closest.revId,
+        section: "",
+        before: "",
+        after: "",
+        deterministicFacts: [
+          {
+            fact: "talk_page_correlated",
+            detail: `time_delta_hours=${deltaHours} talk_comment=${closest.comment.slice(0, 200)}`,
+          },
+        ],
+        layer: "observed",
+        timestamp: closest.timestamp,
+      });
+    }
+  }
+
+  return events;
+}