@slicedearth/whoisleuth-cli 1.45.0 → 1.47.4

package/frontend/src/lib/analysis/visualization-matrix.js

@@ -2,7 +2,14 @@ import { scaleBand } from 'd3-scale';
 import { boundedVisualizationId as boundedId, boundedVisualizationText as boundedText, } from "./visualization-bounds.js";
 export const MAX_VISUAL_MATRIX_ROWS = 24;
 export const MAX_VISUAL_MATRIX_COLUMNS = 6;
-function normalizedMatrixState(value) {
+// A row is one source-pair lane, so it can publish at most one cell for each
+// allowed column. This keeps output at 24 × 6 = 144 cells and prevents hostile
+// duplicate inputs from creating sub-pixel marks or unbounded cell arrays.
+export const MAX_VISUAL_MATRIX_CELLS = MAX_VISUAL_MATRIX_ROWS * MAX_VISUAL_MATRIX_COLUMNS;
+const MAX_VISUAL_MATRIX_COLUMN_INPUTS = MAX_VISUAL_MATRIX_COLUMNS * 4;
+const MAX_VISUAL_MATRIX_ROW_INPUTS = MAX_VISUAL_MATRIX_ROWS * 4;
+const MAX_VISUAL_MATRIX_CELL_INPUTS_PER_ROW = MAX_VISUAL_MATRIX_COLUMNS * 4;
+function normalizedMatrixTone(value) {
     const state = boundedText(value, 30).toLowerCase();
     if (state === 'equal' || state === 'equivalent' || state === 'same')
         return 'equal';
@@ -22,45 +29,79 @@ function normalizedMatrixState(value) {
     return 'unknown';
 }
 export function projectEvidenceMatrix(rawColumns, rawRows) {
-    const columns = [...new Set((Array.isArray(rawColumns) ? rawColumns : [])
-            .map((column) => boundedText(column, 40))
-            .filter(Boolean))]
-        .slice(0, MAX_VISUAL_MATRIX_COLUMNS);
+    const columnInputs = Array.isArray(rawColumns) ? rawColumns : [];
+    const columns = [];
+    const seenColumns = new Set();
+    for (const rawColumn of columnInputs.slice(0, MAX_VISUAL_MATRIX_COLUMN_INPUTS)) {
+        const column = boundedText(rawColumn, 40);
+        if (!column || seenColumns.has(column) || columns.length >= MAX_VISUAL_MATRIX_COLUMNS)
+            continue;
+        seenColumns.add(column);
+        columns.push(column);
+    }
     const columnSet = new Set(columns);
     const seenRows = new Set();
-    const candidates = (Array.isArray(rawRows) ? rawRows : [])
-        .map((row, index) => {
+    const rowInputs = Array.isArray(rawRows) ? rawRows : [];
+    const rows = [];
+    let discardedCellInputs = 0;
+    const scannedRows = rowInputs.slice(0, MAX_VISUAL_MATRIX_ROW_INPUTS);
+    for (const [index, row] of scannedRows.entries()) {
+        if (rows.length >= MAX_VISUAL_MATRIX_ROWS)
+            break;
         const id = boundedId(row?.id) || `row-${index}`;
         const label = boundedText(row?.label, 56);
         if (!label || seenRows.has(id))
-            return null;
+            continue;
         seenRows.add(id);
         const rowCells = (Array.isArray(row.cells) ? row.cells : []);
-        const byColumn = new Map(rowCells
-            .map((cell) => {
+        const scannedCells = rowCells.slice(0, MAX_VISUAL_MATRIX_CELL_INPUTS_PER_ROW);
+        discardedCellInputs += Math.max(0, rowCells.length - scannedCells.length);
+        const byColumn = new Map();
+        for (const cell of scannedCells) {
             const column = boundedText(cell?.column, 40);
-            return [column, {
-                    state: normalizedMatrixState(cell?.state),
-                    detail: boundedText(cell?.detail, 120),
+            if (!columnSet.has(column) || byColumn.has(column)) {
+                discardedCellInputs += 1;
+                continue;
+            }
+            const exactState = boundedText(cell?.state, 40).toLowerCase() || 'unknown';
+            byColumn.set(column, {
+                column,
+                state: exactState,
+                tone: normalizedMatrixTone(cell?.tone ?? exactState),
+                detail: boundedText(cell?.detail, 320),
+            });
+        }
+        const cells = columns.flatMap((column) => {
+            const value = byColumn.get(column);
+            if (value)
+                return [value];
+            if (row.sparse === true)
+                return [];
+            return [{
+                    column,
+                    state: 'not_collected',
+                    tone: 'not_collected',
+                    detail: '',
                 }];
-        })
-            .filter(([column]) => columnSet.has(column)));
-        return {
+        });
+        rows.push({
             id,
             label,
-            cells: columns.map((column) => ({
-                column,
-                state: byColumn.get(column)?.state ?? 'not_collected',
-                detail: byColumn.get(column)?.detail ?? '',
-            })),
-        };
-    })
-        .filter((row) => Boolean(row));
-    const rows = candidates.slice(0, MAX_VISUAL_MATRIX_ROWS);
+            context: boundedText(row.context, 90),
+            status: boundedText(row.status, 40).toLowerCase() || 'unknown',
+            assessment: boundedText(row.assessment, 80),
+            cells,
+        });
+    }
+    const discarded = {
+        columnInputs: Math.max(0, columnInputs.length - columns.length),
+        rowInputs: Math.max(0, rowInputs.length - rows.length),
+        cellInputs: discardedCellInputs,
+    };
     const width = 900;
     const top = 54;
-    const left = 210;
-    const rowHeight = 30;
+    const left = 250;
+    const rowHeight = 40;
     const x = scaleBand().domain(columns).range([left, 870]).padding(0.12);
     const y = scaleBand().domain(rows.map((row) => row.id))
         .range([top, top + Math.max(1, rows.length) * rowHeight])
@@ -83,6 +124,8 @@ export function projectEvidenceMatrix(rawColumns, rawRows) {
                 width: x.bandwidth(),
             })),
         })),
-        truncated: candidates.length > rows.length || rawColumns.length > columns.length,
+        discarded,
+        projectedCellCount: rows.reduce((sum, row) => sum + row.cells.length, 0),
+        truncated: discarded.columnInputs > 0 || discarded.rowInputs > 0 || discarded.cellInputs > 0,
     };
 }

package/frontend/src/lib/analysis/visualization-models.js

@@ -1,21 +1,15 @@
 import { scaleBand, scaleLinear, scalePoint } from 'd3-scale';
 import { boundedVisualizationId as boundedId, boundedVisualizationNumber as boundedNumber, boundedVisualizationText as boundedText, validVisualizationDate as validDate, } from "./visualization-bounds.js";
 export { FORCE_GRAPH_LINK_KINDS, MAX_FORCE_GRAPH_LINKS, MAX_FORCE_GRAPH_NODES, projectBoundedForceGraph, } from "./visualization-force-graph.js";
-export { MAX_VISUAL_MATRIX_COLUMNS, MAX_VISUAL_MATRIX_ROWS, projectEvidenceMatrix, } from "./visualization-matrix.js";
+export { MAX_VISUAL_MATRIX_CELLS, MAX_VISUAL_MATRIX_COLUMNS, MAX_VISUAL_MATRIX_ROWS, projectEvidenceMatrix, } from "./visualization-matrix.js";
 export { MAX_COLLECTION_TIMING_SOURCES, MAX_LIFECYCLE_EVENTS, MAX_TREND_POINTS, projectCollectionTiming, projectLifecycleEvents, projectTrendPoints, } from "./visualization-time-series.js";
 export const MAX_REDIRECT_NODES = 9;
-export const MAX_TRIAGE_PLOT_POINTS = 300;
 export const WATCHLIST_ACTIVITY_DAYS = 28;
 export const MAX_SCORE_FACTORS = 16;
-export const MAX_COVERAGE_BAR_GROUPS = 18;
+export const MAX_SCORE_FACTOR_LABEL_LENGTH = 160;
+export const MAX_PROFILE_LISTING_BAR_GROUPS = 18;
 export const MAX_MONITOR_TIMELINE_EVENTS = 12;
 export const MAX_MONITOR_TIMELINE_LANES = 6;
-function boundedScore(value) {
-    if (value === null || value === undefined || value === '')
-        return null;
-    const score = Number(value);
-    return Number.isFinite(score) ? Math.max(0, Math.min(100, score)) : null;
-}
 function redirectLabel(value) {
     const text = boundedText(value, 320);
     try {
@@ -75,80 +69,21 @@ export function projectRedirectPath(rawRedirects) {
         truncated: rawCount > redirects.length,
     };
 }
-function deterministicSample(items, limit) {
-    const boundedLimit = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
-    if (boundedLimit === 0)
-        return [];
-    if (items.length <= boundedLimit)
-        return items;
-    if (boundedLimit === 1)
-        return items.slice(0, 1);
-    return Array.from({ length: boundedLimit }, (_, index) => {
-        const sourceIndex = Math.round(index * (items.length - 1) / (boundedLimit - 1));
-        return items[sourceIndex];
-    }).filter((item) => item !== undefined);
-}
-export function projectTriagePoints(rawPoints) {
-    const eligible = (Array.isArray(rawPoints) ? rawPoints : [])
-        .map((point) => ({
-        domain: boundedText(point?.domain, 253).toLowerCase(),
-        risk: boundedScore(point?.risk),
-        opportunity: boundedScore(point?.opportunity),
-        availability: boundedText(point?.availability, 30).toLowerCase() || 'unknown',
-        trusted: Boolean(point?.trusted),
-    }))
-        .filter((point) => point.domain && point.risk !== null && point.opportunity !== null)
-        .sort((a, b) => a.domain.localeCompare(b.domain));
-    const sampled = deterministicSample(eligible, MAX_TRIAGE_PLOT_POINTS);
-    const quadrants = eligible.reduce((counts, point) => {
-        if (point.risk >= 50) {
-            if (point.opportunity >= 50)
-                counts.priorityReview += 1;
-            else
-                counts.riskLedReview += 1;
-        }
-        else if (point.opportunity >= 50) {
-            counts.availableReview += 1;
-        }
-        else {
-            counts.lowerScores += 1;
-        }
-        return counts;
-    }, {
-        availableReview: 0,
-        priorityReview: 0,
-        lowerScores: 0,
-        riskLedReview: 0,
-    });
-    const x = scaleLinear().domain([0, 100]).range([58, 842]).clamp(true);
-    const y = scaleLinear().domain([0, 100]).range([308, 28]).clamp(true);
-    const points = sampled.map((point) => ({
-        ...point,
-        risk: point.risk,
-        opportunity: point.opportunity,
-        x: x(point.risk),
-        y: y(point.opportunity),
-        tone: point.trusted
-            ? 'trusted'
-            : point.availability === 'available'
-                ? 'available'
-                : point.availability === 'error'
-                    ? 'error'
-                    : 'registered',
-    }));
-    return {
-        width: 900,
-        height: 360,
-        points,
-        eligibleCount: eligible.length,
-        omittedCount: Math.max(0, (Array.isArray(rawPoints) ? rawPoints.length : 0) - eligible.length),
-        sampled: eligible.length > sampled.length,
-        quadrants,
-    };
-}
 function utcDayKey(milliseconds) {
     return new Date(milliseconds).toISOString().slice(0, 10);
 }
+function utcWeekLabel(startMilliseconds, endMilliseconds) {
+    const start = new Date(startMilliseconds);
+    const end = new Date(endMilliseconds);
+    const startDay = start.getUTCDate();
+    const endDay = end.getUTCDate();
+    const startMonth = start.toLocaleDateString('en-AU', { month: 'short', timeZone: 'UTC' });
+    const endMonth = end.toLocaleDateString('en-AU', { month: 'short', timeZone: 'UTC' });
+    const endYear = end.getUTCFullYear();
+    return startMonth === endMonth
+        ? `${startDay}–${endDay} ${endMonth} ${endYear}`
+        : `${startDay} ${startMonth}–${endDay} ${endMonth} ${endYear}`;
+}
 export function projectWatchlistActivity(rawEvents) {
     const valid = (Array.isArray(rawEvents) ? rawEvents : [])
         .map((event) => {
@@ -162,7 +97,19 @@ export function projectWatchlistActivity(rawEvents) {
     })
         .filter((event) => Boolean(event));
     if (!valid.length) {
-        return { width: 620, height: 190, days: [], maxChanges: 0, totalChecks: 0, totalChanges: 0 };
+        return {
+            width: 660,
+            height: 235,
+            days: [],
+            weekdayLabels: [],
+            weekLabels: [],
+            timeBasis: 'UTC calendar days',
+            windowStart: null,
+            windowEnd: null,
+            maxChanges: 0,
+            totalChecks: 0,
+            totalChanges: 0,
+        };
     }
     const latest = Math.max(...valid.map((event) => event.milliseconds));
     const latestDay = Date.UTC(new Date(latest).getUTCFullYear(), new Date(latest).getUTCMonth(), new Date(latest).getUTCDate());
@@ -179,8 +126,8 @@ export function projectWatchlistActivity(rawEvents) {
         current.conclusive += event.conclusiveCount;
         byDay.set(key, current);
     }
-    const weekScale = scaleBand().domain([0, 1, 2, 3]).range([72, 580]).padding(0.12);
-    const dayScale = scaleBand().domain([0, 1, 2, 3, 4, 5, 6]).range([26, 165]).padding(0.12);
+    const weekScale = scaleBand().domain([0, 1, 2, 3]).range([92, 638]).padding(0.12);
+    const dayScale = scaleBand().domain([0, 1, 2, 3, 4, 5, 6]).range([42, 190]).padding(0.12);
     const dayWidth = weekScale.bandwidth();
     const dayHeight = dayScale.bandwidth();
     const days = Array.from({ length: WATCHLIST_ACTIVITY_DAYS }, (_, index) => {
@@ -199,10 +146,31 @@ export function projectWatchlistActivity(rawEvents) {
             ...activity,
         };
     });
+    const weekdayLabels = Array.from({ length: 7 }, (_, index) => ({
+        index,
+        label: new Date(start + index * 86_400_000).toLocaleDateString('en-AU', { weekday: 'short', timeZone: 'UTC' }),
+        y: (dayScale(index) ?? 0) + dayHeight / 2 + 3,
+    }));
+    const weekLabels = Array.from({ length: 4 }, (_, index) => {
+        const weekStart = start + index * 7 * 86_400_000;
+        const weekEnd = weekStart + 6 * 86_400_000;
+        return {
+            index,
+            label: utcWeekLabel(weekStart, weekEnd),
+            startDate: utcDayKey(weekStart),
+            endDate: utcDayKey(weekEnd),
+            x: (weekScale(index) ?? 0) + dayWidth / 2,
+        };
+    });
     return {
-        width: 620,
-        height: 190,
+        width: 660,
+        height: 235,
         days,
+        weekdayLabels,
+        weekLabels,
+        timeBasis: 'UTC calendar days',
+        windowStart: utcDayKey(start),
+        windowEnd: utcDayKey(latestDay),
         maxChanges: Math.max(0, ...days.map((day) => day.changes)),
         totalChecks: windowEvents.length,
         totalChanges: windowEvents.reduce((sum, event) => sum + event.changeCount, 0),
@@ -212,7 +180,7 @@ export function projectScoreFactors(rawFactors) {
     const candidates = (Array.isArray(rawFactors) ? rawFactors : [])
         .map((factor, index) => ({
         id: `${boundedId(factor?.label) || 'factor'}-${index}`,
-        label: boundedText(factor?.label, 56) || `Factor ${index + 1}`,
+        label: boundedText(factor?.label, MAX_SCORE_FACTOR_LABEL_LENGTH) || `Factor ${index + 1}`,
         delta: boundedNumber(factor?.delta, -100, 100),
     }));
     const factors = candidates.slice(0, MAX_SCORE_FACTORS);
@@ -237,34 +205,35 @@ export function projectScoreFactors(rawFactors) {
         truncated: candidates.length > factors.length,
     };
 }
-export function projectCoverageBars(rawGroups) {
+export function projectProfileListingBars(rawGroups) {
     const candidates = (Array.isArray(rawGroups) ? rawGroups : [])
         .map((group, index) => {
-        const values = {
-            protected: Math.trunc(boundedNumber(group?.protected, 0, 100_000)),
+        const outcomes = {
             registered: Math.trunc(boundedNumber(group?.registered, 0, 100_000)),
             available: Math.trunc(boundedNumber(group?.available, 0, 100_000)),
             unknown: Math.trunc(boundedNumber(group?.unknown, 0, 100_000)),
         };
+        const total = Object.values(outcomes).reduce((sum, value) => sum + value, 0);
         return {
             id: boundedId(group?.id) || `group-${index}`,
             label: boundedText(group?.label, 56) || `Group ${index + 1}`,
-            ...values,
-            total: Object.values(values).reduce((sum, value) => sum + value, 0),
+            ...outcomes,
+            total,
+            profileListed: Math.min(total, Math.trunc(boundedNumber(group?.profileListed, 0, 100_000))),
         };
     })
         .filter((group) => group.total > 0)
         .sort((a, b) => b.total - a.total || a.label.localeCompare(b.label));
-    const groups = candidates.slice(0, MAX_COVERAGE_BAR_GROUPS);
+    const groups = candidates.slice(0, MAX_PROFILE_LISTING_BAR_GROUPS);
     const maximum = Math.max(1, ...groups.map((group) => group.total));
     const x = scaleLinear().domain([0, maximum]).range([210, 860]).clamp(true);
-    const rowHeight = 31;
+    const rowHeight = 40;
     return {
         width: 900,
         height: Math.max(76, 40 + groups.length * rowHeight),
         groups: groups.map((group, index) => {
             let offset = 0;
-            const segments = ['protected', 'registered', 'available', 'unknown'].map((state) => {
+            const segments = ['registered', 'available', 'unknown'].map((state) => {
                 const value = group[state];
                 const start = offset;
                 offset += value;

package/frontend/src/lib/analysis/visualization-time-series.js

@@ -116,28 +116,69 @@ export function projectTrendPoints(rawPoints) {
         .filter((point) => Boolean(point))
         .sort((a, b) => a.milliseconds - b.milliseconds || a.id.localeCompare(b.id));
     const points = candidates.slice(-MAX_TREND_POINTS);
-    const x = scalePoint().domain(points.map((point) => point.id)).range([64, 850]).padding(0.45);
+    const firstMilliseconds = points[0]?.milliseconds ?? 0;
+    const latestMilliseconds = points.at(-1)?.milliseconds ?? firstMilliseconds;
+    const x = scaleLinear()
+        .domain(firstMilliseconds === latestMilliseconds
+        ? [firstMilliseconds - 1, latestMilliseconds + 1]
+        : [firstMilliseconds, latestMilliseconds])
+        .range([64, 850])
+        .clamp(true);
     const maximum = Math.max(1, ...points.map((point) => point.total));
     const y = scaleLinear().domain([0, maximum]).range([190, 24]).nice().clamp(true);
     const first = points[0] ?? null;
     const latest = points.at(-1) ?? null;
+    const projectedPoints = points.map((point, sequenceIndex) => ({
+        ...point,
+        sequenceIndex,
+        x: points.length > 1 ? x(point.milliseconds) : 450,
+        y: y(point.total),
+        addedY: y(point.added),
+    }));
+    // A capped point is a lower bound, so neither adjacent slope is measured.
+    // Retain only directly adjacent complete-to-complete segments and leave a
+    // visible gap around every capped point.
+    const segments = projectedPoints.slice(1).flatMap((to, index) => {
+        const from = projectedPoints[index];
+        return from && !from.partial && !to.partial ? [{
+                fromId: from.id,
+                toId: to.id,
+                x1: from.x,
+                y1: from.y,
+                x2: to.x,
+                y2: to.y,
+            }] : [];
+    });
+    const partialChecks = points.filter((point) => point.partial).length;
+    const hasPartialChecks = partialChecks > 0;
+    const summaryMetric = (value, lowerBound) => ({
+        value,
+        lowerBound,
+        label: lowerBound ? `At least ${value}` : String(value),
+    });
+    const peakTotal = points.reduce((peak, point) => Math.max(peak, point.total), 0);
+    const newlyObserved = points.reduce((total, point) => total + point.added, 0);
     return {
         width: 900,
         height: 225,
         maximum,
+        spacing: 'elapsed_time',
+        elapsed: {
+            firstAt: points[0]?.date ?? null,
+            latestAt: points.at(-1)?.date ?? null,
+            milliseconds: Math.max(0, latestMilliseconds - firstMilliseconds),
+        },
         ticks: y.ticks(4).map((value) => ({ value, y: y(value) })),
-        points: points.map((point) => ({
-            ...point,
-            x: x(point.id) ?? 450,
-            y: y(point.total),
-            addedY: y(point.added),
-        })),
+        points: projectedPoints,
+        segments,
         summary: {
-            firstTotal: first?.total ?? 0,
-            latestTotal: latest?.total ?? 0,
-            peakTotal: points.reduce((peak, point) => Math.max(peak, point.total), 0),
-            newlyObserved: points.reduce((total, point) => total + point.added, 0),
-            partialChecks: points.filter((point) => point.partial).length,
+            first: summaryMetric(first?.total ?? 0, Boolean(first?.partial)),
+            latest: summaryMetric(latest?.total ?? 0, Boolean(latest?.partial)),
+            // Any capped total can exceed the displayed observed maximum.
+            peak: summaryMetric(peakTotal, hasPartialChecks),
+            // Every capped new-count is itself a lower bound, including observed 0.
+            newlyObserved: summaryMetric(newlyObserved, hasPartialChecks),
+            partialChecks,
         },
         truncated: candidates.length > points.length,
     };

package/frontend/src/lib/analysis/website-snapshot-model.js

@@ -1,6 +1,7 @@
 import { normalizeDomain } from "./case-model.js";
 export const WEBSITE_SNAPSHOT_SCHEMA = 'whoisleuth.website-profile-snapshots';
 export const WEBSITE_SNAPSHOT_SCHEMA_VERSION = 4;
+export const SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS = Object.freeze([1, 2, 3, WEBSITE_SNAPSHOT_SCHEMA_VERSION]);
 export const MAX_WEBSITE_SNAPSHOTS = 60;
 export const MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN = 12;
 export const MAX_WEBSITE_SNAPSHOT_STORE_BYTES = 512 * 1024;
@@ -188,6 +189,11 @@ export function normalizeWebsiteSnapshotStore(raw) {
         && Number(value?.version) > WEBSITE_SNAPSHOT_SCHEMA_VERSION) {
         throw new Error('This website-snapshot collection uses a newer schema and cannot be read safely.');
     }
+    if (value?.schema === WEBSITE_SNAPSHOT_SCHEMA
+        && Number.isSafeInteger(value.version)
+        && !SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS.includes(Number(value.version))) {
+        throw new Error('This website-snapshot collection uses an unsupported schema and cannot be read safely.');
+    }
     const sourceValues = Array.isArray(raw) ? raw : Array.isArray(value?.snapshots) ? value.snapshots : [];
     const snapshots = values(sourceValues, MAX_WEBSITE_SNAPSHOTS * 2, normalizeWebsiteProfileSnapshot);
     const perDomain = new Map();