hippo-memory 1.60.0 → 1.61.0

package/dist/dashboard-queries.js

@@ -0,0 +1,355 @@
+// Read models over a snapshot: overview, project, memory page, search, and the fresh single-memory detail.
+import { calculateStrength } from './memory.js';
+import { listMemoryConflicts } from './store/conflicts.js';
+import { readEntry } from './store/entry-reads.js';
+import { BANDS, DAY_MS, LAYERS, isLiveMemory, memoryFacts, projectIdentity, } from './dashboard-snapshot.js';
+const SERIES_DAYS = 90;
+/** The server words each delta for this range; the client re-slices the series for 7d and 90d. */
+const DELTA_DAYS = 30;
+const POINTS_MAX = 4_000;
+const GRID_COLS = 64;
+const GRID_ROWS = 32;
+const NAME_MATCHES = 3;
+function round1(value) {
+    return Math.round(value * 10) / 10;
+}
+function round3(value) {
+    return Math.round(value * 1000) / 1000;
+}
+/** Cumulative counts at 91 daily points, oldest first: point 0 is the base at the window start, point 90 is now. */
+function cumulativeSeries(times, nowMs) {
+    const days = Array.from({ length: SERIES_DAYS + 1 }, () => 0);
+    let base = 0;
+    for (const t of times) {
+        const ago = Math.floor((nowMs - t) / DAY_MS);
+        if (ago >= SERIES_DAYS)
+            base++;
+        else
+            days[SERIES_DAYS - Math.max(0, ago)]++;
+    }
+    let running = base;
+    return days.map((n) => (running += n));
+}
+function addedWithin(series) {
+    return series[SERIES_DAYS] - series[SERIES_DAYS - DELTA_DAYS];
+}
+function pct(share) {
+    return `${(share * 100).toFixed(1)}%`;
+}
+function plural(n, one, many) {
+    return `${n} ${n === 1 ? one : many}`;
+}
+/** Named projects by size, then Global, then Unassigned, as the treemap orders them. */
+function orderedProjects(snap) {
+    const rank = (p) => (p.summary.kind === 'project' ? 0 : p.summary.kind === 'global' ? 1 : 2);
+    return [...snap.projects].sort((a, b) => rank(a) - rank(b) || b.summary.live - a.summary.live || (a.summary.name < b.summary.name ? -1 : 1));
+}
+function buildKpis(snap, projects) {
+    const total = snap.facts.length;
+    const named = projects.filter((p) => p.summary.kind === 'project');
+    const totalSeries = cumulativeSeries(snap.facts.map((f) => f.createdMs), snap.nowMs);
+    const projectSeries = cumulativeSeries(named.map((p) => p.firstCreatedMs), snap.nowMs);
+    const atRisk = projects.reduce((sum, p) => sum + p.summary.atRisk, 0);
+    const weakNow = snap.facts.reduce((sum, f) => sum + (f.strength < 0.2 ? 1 : 0), 0);
+    const withConflicts = projects.filter((p) => p.summary.openConflicts > 0).length;
+    const embedded = snap.embeddingCoverage === null ? 0 : Math.round(snap.embeddingCoverage * total);
+    return [
+        {
+            id: 'total', label: 'Total memories', value: total, series: totalSeries,
+            delta: `+${addedWithin(totalSeries)} created in the last ${DELTA_DAYS}d`,
+        },
+        {
+            id: 'projects', label: 'Projects', value: named.length, series: projectSeries,
+            delta: `+${addedWithin(projectSeries)} new in ${DELTA_DAYS}d`,
+        },
+        {
+            id: 'atRiskShare', label: 'At-risk share, 30d', value: total > 0 ? atRisk / total : 0, series: null,
+            delta: `${pct(total > 0 ? weakNow / total : 0)} now at strength under 0.2`,
+        },
+        {
+            id: 'openConflicts', label: 'Open conflicts', value: snap.openConflicts, series: null,
+            delta: `in ${plural(withConflicts, 'project', 'projects')}`,
+        },
+        {
+            id: 'embeddingCoverage', label: 'Embedding coverage', value: snap.embeddingCoverage ?? 0, series: null,
+            delta: snap.embeddingCoverage === null ? 'no vector table yet' : `${total - embedded} not embedded`,
+        },
+    ];
+}
+function topKeys(projects, count, score) {
+    return projects
+        .filter((p) => score(p) > 0)
+        .sort((a, b) => score(b) - score(a) || b.summary.live - a.summary.live || (a.summary.key < b.summary.key ? -1 : 1))
+        .slice(0, count)
+        .map((p) => p.summary.key);
+}
+/** The overview of one snapshot, built once and reused until the snapshot is replaced. */
+export function buildOverview(snap) {
+    if (snap.memo.overview !== null)
+        return snap.memo.overview;
+    const ordered = orderedProjects(snap);
+    const overview = {
+        snapshotId: snap.id,
+        generatedAt: new Date(snap.nowMs).toISOString(),
+        total: snap.facts.length,
+        excluded: snap.excluded,
+        embeddingCoverage: snap.embeddingCoverage,
+        kpis: buildKpis(snap, ordered),
+        projects: ordered.map((p) => p.summary),
+        mostAtRisk: topKeys(ordered, 5, (p) => p.summary.atRisk),
+        mostConflicts: topKeys(ordered, 4, (p) => p.summary.openConflicts),
+    };
+    snap.memo.overview = overview;
+    return overview;
+}
+function buildScatter(snap, project) {
+    const cached = snap.memo.scatter.get(project.summary.key);
+    if (cached)
+        return cached;
+    let maxAge = 0;
+    for (const i of project.members)
+        maxAge = Math.max(maxAge, snap.facts[i].ageDays);
+    const maxAgeDays = Math.max(1, Math.ceil(maxAge));
+    let scatter;
+    if (project.members.length <= POINTS_MAX) {
+        const points = [];
+        const ids = [];
+        for (const i of project.members) {
+            const f = snap.facts[i];
+            points.push([round1(f.ageDays), round3(f.strength), f.layer, f.band]);
+            ids.push(f.id);
+        }
+        scatter = { mode: 'points', maxAgeDays, points, ids };
+    }
+    else {
+        const cells = [[], [], [], []];
+        for (const layer of cells)
+            for (let k = 0; k < GRID_COLS * GRID_ROWS; k++)
+                layer.push(0);
+        const maxLog = Math.log10(maxAgeDays + 1);
+        for (const i of project.members) {
+            const f = snap.facts[i];
+            const col = Math.min(GRID_COLS - 1, Math.floor((Math.log10(f.ageDays + 1) / maxLog) * GRID_COLS));
+            const row = Math.min(GRID_ROWS - 1, Math.floor(f.strength * GRID_ROWS));
+            cells[f.layer][row * GRID_COLS + col]++;
+        }
+        scatter = { mode: 'grid', maxAgeDays, cols: GRID_COLS, rows: GRID_ROWS, cells };
+    }
+    snap.memo.scatter.set(project.summary.key, scatter);
+    return scatter;
+}
+/** Summary and scatter for one project key, or null when the key is unknown. */
+export function buildProjectDetail(snap, key) {
+    const project = snap.byKey.get(key);
+    if (!project)
+        return null;
+    return { snapshotId: snap.id, summary: project.summary, scatter: buildScatter(snap, project) };
+}
+function numericSort(sort) {
+    switch (sort) {
+        case 'layer': return (f) => f.layer;
+        case 'strength': return (f) => f.strength;
+        case 'retrievals': return (f) => f.retrievals;
+        case 'last': return (f) => f.lastRetrievedDays;
+        case 'age': return (f) => f.ageDays;
+        case 'confidence': return (f) => f.confidenceRank;
+        case 'content':
+        case 'scope': return null;
+    }
+}
+function comparator(facts, sort, dir) {
+    const sign = dir === 'asc' ? 1 : -1;
+    const tie = (a, b) => (facts[a].id < facts[b].id ? -1 : facts[a].id > facts[b].id ? 1 : 0);
+    const num = numericSort(sort);
+    if (num)
+        return (a, b) => (num(facts[a]) - num(facts[b])) * sign || tie(a, b);
+    // Content and scope compare precomputed strings with `<`; localeCompare per pair is far slower at 50k rows.
+    const text = sort === 'content' ? (f) => f.lower : (f) => f.scope ?? '';
+    return (a, b) => {
+        const x = text(facts[a]);
+        const y = text(facts[b]);
+        return (x < y ? -sign : x > y ? sign : 0) || tie(a, b);
+    };
+}
+function sortedMembers(snap, project, sort, dir) {
+    const key = `${project.summary.key}|${sort}|${dir}`;
+    const hit = snap.memo.sorted.get(key);
+    if (hit)
+        return hit;
+    const sorted = Int32Array.from(project.members).sort(comparator(snap.facts, sort, dir));
+    snap.memo.sorted.set(key, sorted);
+    return sorted;
+}
+function bucket(value) {
+    return value >= 0.5 ? 0 : value >= 0.2 ? 1 : 2;
+}
+function filterMembers(snap, sorted, query) {
+    const layerOn = [false, false, false, false];
+    for (const l of query.layers)
+        layerOn[l] = true;
+    const { brush, q } = query;
+    const kept = new Int32Array(sorted.length);
+    let n = 0;
+    const counts = { all: 0, risk: 0, pinned: 0, conflict: 0 };
+    const decay = { now: [0, 0, 0], in7d: [0, 0, 0], in30d: [0, 0, 0], pinned: 0 };
+    for (const i of sorted) {
+        const f = snap.facts[i];
+        if (!layerOn[f.layer])
+            continue;
+        if (f.ageDays < brush.amin || f.ageDays > brush.amax || f.strength < brush.smin || f.strength > brush.smax)
+            continue;
+        if (q !== null && !f.lower.includes(q) && !f.tagsLower.includes(q))
+            continue;
+        kept[n++] = i;
+        decay.now[bucket(f.strength)]++;
+        decay.in7d[bucket(f.strength7d)]++;
+        decay.in30d[bucket(f.strength30d)]++;
+        if (f.pinned) {
+            decay.pinned++;
+            counts.pinned++;
+        }
+        if (f.band === 3)
+            counts.risk++;
+        if (f.inConflict)
+            counts.conflict++;
+        counts.all++;
+    }
+    return { idx: kept.subarray(0, n), counts, decay, byChip: new Map() };
+}
+function filteredSet(snap, project, query) {
+    const { brush } = query;
+    const key = [
+        project.summary.key, query.sort, query.dir, query.layers.join(''),
+        brush.amin, brush.amax, brush.smin, brush.smax, query.q ?? '',
+    ].join('|');
+    const hit = snap.memo.filtered.get(key);
+    if (hit)
+        return hit;
+    const set = filterMembers(snap, sortedMembers(snap, project, query.sort, query.dir), query);
+    snap.memo.filtered.set(key, set);
+    return set;
+}
+function chipMembers(snap, set, chip) {
+    if (chip === 'all')
+        return set.idx;
+    const hit = set.byChip.get(chip);
+    if (hit)
+        return hit;
+    const pick = (f) => (chip === 'risk' ? f.band === 3 : chip === 'pinned' ? f.pinned : f.inConflict);
+    const out = set.idx.filter((i) => pick(snap.facts[i]));
+    set.byChip.set(chip, out);
+    return out;
+}
+function toRow(f) {
+    return {
+        id: f.id,
+        content: f.head,
+        truncated: f.truncated,
+        layer: LAYERS[f.layer],
+        band: BANDS[f.band],
+        strength: round3(f.strength),
+        retrievals: f.retrievals,
+        lastRetrievedDays: round1(f.lastRetrievedDays),
+        ageDays: round1(f.ageDays),
+        confidence: f.confidence,
+        scope: f.scope,
+        pinned: f.pinned,
+        wrong: f.wrong,
+        inConflict: f.inConflict,
+    };
+}
+/** One page of a project's memories under the query's sort and filters, or null when the key is unknown. */
+export function buildMemoryPage(snap, key, query) {
+    const project = snap.byKey.get(key);
+    if (!project)
+        return null;
+    const set = filteredSet(snap, project, query);
+    const shown = chipMembers(snap, set, query.chip);
+    const rows = [];
+    for (let k = query.offset; k < Math.min(shown.length, query.offset + query.limit); k++) {
+        rows.push(toRow(snap.facts[shown[k]]));
+    }
+    const decay = set.decay;
+    return { snapshotId: snap.id, total: shown.length, offset: query.offset, limit: query.limit, counts: set.counts, decay, rows };
+}
+/** Substring search over content and tags across every project; it reads the snapshot and never touches recall. */
+export function buildSearch(snap, text) {
+    const needle = text.toLowerCase();
+    const perProject = Array.from({ length: snap.projects.length }, () => 0);
+    let total = 0;
+    for (const f of snap.facts) {
+        if (f.lower.includes(needle) || f.tagsLower.includes(needle)) {
+            perProject[f.project]++;
+            total++;
+        }
+    }
+    const hits = {};
+    snap.projects.forEach((p, i) => {
+        if (perProject[i] > 0)
+            hits[p.summary.key] = perProject[i];
+    });
+    const nameMatches = snap.projects
+        .filter((p) => p.summary.name.toLowerCase().includes(needle))
+        .sort((a, b) => b.summary.live - a.summary.live)
+        .slice(0, NAME_MATCHES)
+        .map((p) => ({ key: p.summary.key, name: p.summary.name, live: p.summary.live }));
+    return { snapshotId: snap.id, q: text, total, hits, nameMatches };
+}
+function openConflictsOf(hippoRoot, tenantId, entry, nowMs) {
+    const out = [];
+    for (const c of listMemoryConflicts(hippoRoot, 'open', tenantId)) {
+        if (c.memory_a_id !== entry.id && c.memory_b_id !== entry.id)
+            continue;
+        const otherId = c.memory_a_id === entry.id ? c.memory_b_id : c.memory_a_id;
+        const other = readEntry(hippoRoot, otherId, tenantId);
+        if (!other || !isLiveMemory(other))
+            continue;
+        out.push({
+            id: c.id,
+            reason: c.reason,
+            score: c.score,
+            other: {
+                id: other.id,
+                content: other.content,
+                strength: round3(calculateStrength(other, new Date(nowMs))),
+                retrievals: other.retrieval_count,
+            },
+        });
+    }
+    return out;
+}
+/** Full detail read fresh from the store, never from the snapshot, so the drawer is current inside the coalescing window. */
+export function buildMemoryDetail(hippoRoot, tenantId, entry, snapshotId, nowMs, embedded) {
+    const f = memoryFacts(entry, nowMs);
+    const project = projectIdentity(entry.origin_project);
+    return {
+        snapshotId,
+        id: entry.id,
+        content: entry.content,
+        layer: LAYERS[f.layer],
+        band: BANDS[f.band],
+        tags: entry.tags,
+        strength: round3(f.strength),
+        strength7d: round3(f.strength7d),
+        strength30d: round3(f.strength30d),
+        halfLifeDays: entry.half_life_days,
+        retrievals: entry.retrieval_count,
+        lastRetrieved: entry.last_retrieved,
+        created: entry.created,
+        ageDays: round1(f.ageDays),
+        schemaFit: entry.schema_fit,
+        valence: entry.emotional_valence,
+        confidence: f.confidence,
+        agedOut: f.agedOut,
+        pinned: entry.pinned,
+        wrong: f.wrong,
+        projectKey: project.key,
+        projectName: project.name,
+        scope: entry.scope,
+        tenant: entry.tenantId,
+        kind: entry.kind,
+        embedded,
+        conflicts: openConflictsOf(hippoRoot, tenantId, entry, nowMs),
+    };
+}
+//# sourceMappingURL=dashboard-queries.js.map

package/dist/dashboard-snapshot.d.ts

@@ -0,0 +1,138 @@
+import { Layer as MemoryLayer, type MemoryEntry } from './memory.js';
+import type { MemoryConflict } from './store/rows.js';
+import type { Band, ChipCounts, Layer, Overview, ProjectKind, ProjectSummary, ScatterGrid, ScatterPoints } from './dashboard-types.js';
+export declare const DAY_MS = 86400000;
+export declare const LAYERS: readonly Layer[];
+export declare const BANDS: readonly Band[];
+/** An outside commit rebuilds the snapshot at most this often. */
+export declare const COALESCE_MS = 10000;
+/** Projected strengths track the clock, so the snapshot is rebuilt at least this often. */
+export declare const TTL_MS: number;
+/** The one predicate for "shown on the dashboard": not superseded, not archived, not quarantined. */
+export declare function isLiveMemory(entry: Pick<MemoryEntry, 'superseded_by' | 'kind' | 'scope'>): boolean;
+/** Wire key, display name and kind of a project. */
+export interface ProjectIdentity {
+    key: string;
+    name: string;
+    kind: ProjectKind;
+}
+/** Wire key, display name and kind for an `origin_project` value. */
+export declare function projectIdentity(origin: string | null | undefined): ProjectIdentity;
+/** Index of a layer in the wire order `LAYERS`. */
+export declare function layerIndex(layer: MemoryLayer): number;
+/** Per-memory numbers shared by the snapshot build and the single-memory detail route. */
+export interface MemoryFacts {
+    strength: number;
+    strength7d: number;
+    strength30d: number;
+    band: number;
+    layer: number;
+    ageDays: number;
+    lastRetrievedDays: number;
+    confidence: string;
+    agedOut: boolean;
+    wrong: boolean;
+}
+/** Strength now and at +7d and +30d, band, age and the confidence split for one entry at `nowMs`. */
+export declare function memoryFacts(entry: MemoryEntry, nowMs: number): MemoryFacts;
+/** One live memory as the snapshot holds it: numbers and short strings only, never the full entry. */
+export interface Fact {
+    id: string;
+    head: string;
+    truncated: boolean;
+    lower: string;
+    tagsLower: string;
+    layer: number;
+    band: number;
+    strength: number;
+    strength7d: number;
+    strength30d: number;
+    retrievals: number;
+    lastRetrievedDays: number;
+    ageDays: number;
+    confidence: string;
+    confidenceRank: number;
+    scope: string | null;
+    pinned: boolean;
+    wrong: boolean;
+    inConflict: boolean;
+    embedded: boolean;
+    project: number;
+    createdMs: number;
+}
+export interface ProjectAgg {
+    summary: ProjectSummary;
+    /** Indices into `Snapshot.facts`, in snapshot order. */
+    members: Int32Array;
+    firstCreatedMs: number;
+}
+export type Chip = 'all' | 'risk' | 'pinned' | 'conflict';
+/** A filtered memory set over one project: indices in sort order, chip counts, decay buckets, and per-chip lists built on demand. */
+export interface FilteredSet {
+    idx: Int32Array;
+    counts: ChipCounts;
+    decay: {
+        now: [number, number, number];
+        in7d: [number, number, number];
+        in30d: [number, number, number];
+        pinned: number;
+    };
+    byChip: Map<Chip, Int32Array>;
+}
+/** Small least-recently-used map; the oldest entry leaves when it is full. */
+export declare class Lru<V> {
+    private readonly capacity;
+    private readonly entries;
+    constructor(capacity: number);
+    get(key: string): V | undefined;
+    set(key: string, value: V): void;
+}
+/** Per-snapshot memo of query results; it dies with the snapshot, so no key needs a snapshot id. */
+export interface SnapshotMemo {
+    sorted: Lru<Int32Array>;
+    filtered: Lru<FilteredSet>;
+    scatter: Map<string, ScatterPoints | ScatterGrid>;
+    overview: Overview | null;
+}
+export interface Snapshot {
+    id: number;
+    tenantId: string;
+    nowMs: number;
+    facts: Fact[];
+    projects: ProjectAgg[];
+    byKey: Map<string, ProjectAgg>;
+    excluded: {
+        superseded: number;
+        archived: number;
+        quarantined: number;
+    };
+    /** Embedded share of live memories, or null when the store has no vector table yet. */
+    embeddingCoverage: number | null;
+    /** Open conflicts with both members live, each counted once. */
+    openConflicts: number;
+    memo: SnapshotMemo;
+}
+export interface SnapshotInput {
+    id: number;
+    tenantId: string;
+    nowMs: number;
+    entries: readonly MemoryEntry[];
+    openConflicts: readonly Pick<MemoryConflict, 'memory_a_id' | 'memory_b_id'>[];
+    embeddedIds: ReadonlySet<string> | null;
+}
+/** Builds the snapshot from already-loaded rows; it touches no store, so tests can feed it directly. */
+export declare function buildSnapshot(input: SnapshotInput): Snapshot;
+export interface SnapshotService {
+    /** The cached snapshot, or a new build when the cache key changed, the coalescing window allows it, or `fresh` is set. */
+    get(tenantId: string, fresh?: boolean): Snapshot;
+    /** Drops the cache after a dashboard write commits, so the next read rebuilds. */
+    invalidate(): void;
+    /** Id of the most recent build, which is what a client holding the last response still has. */
+    currentId(): number;
+    /** Ids with a vector as of the last build (a direct read before any build), so a detail read never rescans the vector table. */
+    embeddedIds(): ReadonlySet<string> | null;
+    close(): void;
+}
+/** Holds one read-only connection for its commit signal (`PRAGMA data_version`) and the snapshot cache; `now` dates the strength projections, `cacheClock` ages the cache. */
+export declare function createSnapshotService(hippoRoot: string, now: () => number, cacheClock?: () => number): SnapshotService;
+//# sourceMappingURL=dashboard-snapshot.d.ts.map