@atomicmemory/core 1.0.0

package/dist/db/ppr.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Personalized PageRank (PPR) for adaptive link expansion.
+ *
+ * Replaces fixed-depth BFS link expansion with a principled algorithm that
+ * propagates relevance scores through the memory graph. PPR naturally handles
+ * variable-hop reasoning: densely-connected subgraphs get deeper traversal,
+ * while sparse connections receive less signal.
+ *
+ * Algorithm: Power iteration on the PPR equation:
+ *   r(t+1) = damping * A_norm * r(t) + (1 - damping) * seed
+ *
+ * where:
+ *   seed = initial retrieval scores (normalized to sum=1)
+ *   A_norm = column-normalized adjacency matrix from memory_links
+ *   damping = probability of following a link (vs teleporting back to seed)
+ *
+ * Adapted from HippoRAG (NeurIPS 2024) for AtomicMemory's pgvector + link table
+ * architecture.
+ */
+import pg from 'pg';
+export interface PPRConfig {
+    /** Probability of following a link vs teleporting back to seed nodes. */
+    damping?: number;
+    /** Maximum power-iteration rounds. */
+    maxIterations?: number;
+    /** Convergence threshold (L1 norm of score delta). */
+    tolerance?: number;
+}
+export interface PPRResult {
+    /** Memory ID → PPR score, sorted descending. Excludes seed nodes. */
+    scores: Map<string, number>;
+    /** Number of iterations until convergence. */
+    iterations: number;
+}
+/**
+ * Run Personalized PageRank over the memory_links graph.
+ *
+ * @param pool  - Postgres connection pool
+ * @param seedScores - Map of memory ID → relevance score from initial retrieval
+ * @param config - PPR tuning parameters
+ * @returns PPR scores for all reachable memories (excluding seeds)
+ */
+/**
+ * Run PPR power iteration over an in-memory adjacency list.
+ *
+ * Exported separately from the DB-loading wrapper so it can be unit-tested
+ * without a database connection.
+ */
+export declare function runPPR(adjacency: Map<string, Set<string>>, seedScores: Map<string, number>, pprConfig?: PPRConfig): PPRResult;
+/**
+ * Run PPR with adjacency loaded from the database.
+ *
+ * Loads the 2-hop subgraph around seed nodes from memory_links, then
+ * delegates to the pure runPPR function.
+ */
+export declare function personalizedPageRank(pool: pg.Pool, seedScores: Map<string, number>, pprConfig?: PPRConfig): Promise<PPRResult>;

package/dist/db/ppr.js

@@ -0,0 +1,178 @@
+/**
+ * Personalized PageRank (PPR) for adaptive link expansion.
+ *
+ * Replaces fixed-depth BFS link expansion with a principled algorithm that
+ * propagates relevance scores through the memory graph. PPR naturally handles
+ * variable-hop reasoning: densely-connected subgraphs get deeper traversal,
+ * while sparse connections receive less signal.
+ *
+ * Algorithm: Power iteration on the PPR equation:
+ *   r(t+1) = damping * A_norm * r(t) + (1 - damping) * seed
+ *
+ * where:
+ *   seed = initial retrieval scores (normalized to sum=1)
+ *   A_norm = column-normalized adjacency matrix from memory_links
+ *   damping = probability of following a link (vs teleporting back to seed)
+ *
+ * Adapted from HippoRAG (NeurIPS 2024) for AtomicMemory's pgvector + link table
+ * architecture.
+ */
+const DEFAULT_DAMPING = 0.5;
+const DEFAULT_MAX_ITERATIONS = 20;
+const DEFAULT_TOLERANCE = 1e-6;
+/**
+ * Run Personalized PageRank over the memory_links graph.
+ *
+ * @param pool  - Postgres connection pool
+ * @param seedScores - Map of memory ID → relevance score from initial retrieval
+ * @param config - PPR tuning parameters
+ * @returns PPR scores for all reachable memories (excluding seeds)
+ */
+/**
+ * Run PPR power iteration over an in-memory adjacency list.
+ *
+ * Exported separately from the DB-loading wrapper so it can be unit-tested
+ * without a database connection.
+ */
+export function runPPR(adjacency, seedScores, pprConfig = {}) {
+    if (seedScores.size === 0 || adjacency.size === 0) {
+        return { scores: new Map(), iterations: 0 };
+    }
+    const damping = pprConfig.damping ?? DEFAULT_DAMPING;
+    const maxIterations = pprConfig.maxIterations ?? DEFAULT_MAX_ITERATIONS;
+    const tolerance = pprConfig.tolerance ?? DEFAULT_TOLERANCE;
+    const allNodes = collectNodes(adjacency, seedScores);
+    const seed = normalizeSeed(seedScores, allNodes);
+    const outDegree = computeOutDegree(adjacency, allNodes);
+    const { scores, iterations } = iteratePPR(allNodes, seed, adjacency, outDegree, damping, maxIterations, tolerance);
+    return { scores: extractExpansionScores(scores, seedScores), iterations };
+}
+/** Power iteration loop for PPR convergence. */
+function iteratePPR(allNodes, seed, adjacency, outDegree, damping, maxIterations, tolerance) {
+    let scores = new Map(seed);
+    let iterations = 0;
+    for (let i = 0; i < maxIterations; i++) {
+        iterations = i + 1;
+        const next = computeNextScores(allNodes, seed, adjacency, outDegree, scores, damping);
+        const delta = computeDelta(allNodes, next, scores);
+        scores = next;
+        if (delta < tolerance)
+            break;
+    }
+    return { scores, iterations };
+}
+/** Compute next-step scores from seed teleportation + link propagation. */
+function computeNextScores(allNodes, seed, adjacency, outDegree, scores, damping) {
+    const next = new Map();
+    for (const node of allNodes) {
+        next.set(node, (1 - damping) * (seed.get(node) ?? 0));
+    }
+    for (const [src, neighbors] of adjacency) {
+        const contribution = damping * (scores.get(src) ?? 0) / (outDegree.get(src) ?? 1);
+        for (const neighbor of neighbors) {
+            next.set(neighbor, (next.get(neighbor) ?? 0) + contribution);
+        }
+    }
+    return next;
+}
+/** Compute L1 norm of score delta between iterations. */
+function computeDelta(allNodes, next, scores) {
+    let delta = 0;
+    for (const node of allNodes) {
+        delta += Math.abs((next.get(node) ?? 0) - (scores.get(node) ?? 0));
+    }
+    return delta;
+}
+/** Extract non-seed scores above zero for the expansion result. */
+function extractExpansionScores(scores, seedScores) {
+    const seedIds = new Set(seedScores.keys());
+    const expansion = new Map();
+    for (const [id, score] of scores) {
+        if (!seedIds.has(id) && score > 0) {
+            expansion.set(id, score);
+        }
+    }
+    return expansion;
+}
+/**
+ * Run PPR with adjacency loaded from the database.
+ *
+ * Loads the 2-hop subgraph around seed nodes from memory_links, then
+ * delegates to the pure runPPR function.
+ */
+export async function personalizedPageRank(pool, seedScores, pprConfig = {}) {
+    if (seedScores.size === 0)
+        return { scores: new Map(), iterations: 0 };
+    const adjacency = await loadAdjacency(pool, seedScores);
+    return runPPR(adjacency, seedScores, pprConfig);
+}
+/**
+ * Load the adjacency list for all nodes reachable within 2 hops from seeds.
+ *
+ * We limit to 2 hops from seeds to bound the graph size while still allowing
+ * PPR to propagate through intermediate nodes. The power iteration handles
+ * deeper propagation via the iterative score updates.
+ */
+async function loadAdjacency(pool, seedScores) {
+    const seedIds = [...seedScores.keys()];
+    if (seedIds.length === 0)
+        return new Map();
+    const result = await pool.query(`WITH seed_links AS (
+       SELECT source_id, target_id FROM memory_links
+       WHERE source_id = ANY($1) OR target_id = ANY($1)
+     ),
+     hop1_nodes AS (
+       SELECT source_id AS id FROM seed_links
+       UNION
+       SELECT target_id AS id FROM seed_links
+     ),
+     hop2_links AS (
+       SELECT ml.source_id, ml.target_id FROM memory_links ml
+       INNER JOIN hop1_nodes h ON ml.source_id = h.id OR ml.target_id = h.id
+     )
+     SELECT source_id, target_id FROM seed_links
+     UNION
+     SELECT source_id, target_id FROM hop2_links`, [seedIds]);
+    const adjacency = new Map();
+    for (const row of result.rows) {
+        const src = row.source_id;
+        const tgt = row.target_id;
+        if (!adjacency.has(src))
+            adjacency.set(src, new Set());
+        adjacency.get(src).add(tgt);
+        if (!adjacency.has(tgt))
+            adjacency.set(tgt, new Set());
+        adjacency.get(tgt).add(src);
+    }
+    return adjacency;
+}
+function collectNodes(adjacency, seedScores) {
+    const nodes = new Set();
+    for (const id of seedScores.keys())
+        nodes.add(id);
+    for (const [src, neighbors] of adjacency) {
+        nodes.add(src);
+        for (const neighbor of neighbors)
+            nodes.add(neighbor);
+    }
+    return nodes;
+}
+function normalizeSeed(seedScores, allNodes) {
+    let total = 0;
+    for (const score of seedScores.values())
+        total += score;
+    if (total === 0)
+        total = 1;
+    const normalized = new Map();
+    for (const node of allNodes) {
+        normalized.set(node, (seedScores.get(node) ?? 0) / total);
+    }
+    return normalized;
+}
+function computeOutDegree(adjacency, allNodes) {
+    const degree = new Map();
+    for (const node of allNodes) {
+        degree.set(node, adjacency.get(node)?.size ?? 0);
+    }
+    return degree;
+}

package/dist/db/query-helpers.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Shared query-building helpers for vector search and hybrid search backends.
+ *
+ * Centralizes scoring weight preparation, parameter setup, and site filtering
+ * to eliminate duplication between repository-vector-search.ts and
+ * repository-representations.ts.
+ */
+export declare const RRF_K = 60;
+export interface HybridQueryParams {
+    /** Positional params array for pg.query. */
+    params: unknown[];
+    /** SQL clause fragment for optional source_site filter. Uses the alias prefix if provided. */
+    siteFilter: string;
+    /** ISO string for the reference timestamp. */
+    refTime: string;
+    /** Scoring weights from config. */
+    wSim: number;
+    wImp: number;
+    wRec: number;
+    rankingMinSimilarity: number;
+}
+/**
+ * Build shared query parameters for a hybrid (vector + FTS) search.
+ *
+ * Returns an array of positional params, the site filter clause, and
+ * scoring weights. The caller must set the first params in a known order:
+ *   $1=embedding, $2=userId, $3=queryText, $4=limit, $5=wSim, $6=wImp, $7=wRec, $8=refTime, $9=rankingMinSimilarity, [$10=sourceSite]
+ */
+export declare function buildHybridSearchParams(queryEmbedding: number[], userId: string, queryText: string, limit: number, siteFilterColumn: string, sourceSite?: string, referenceTime?: Date): HybridQueryParams;
+/**
+ * Build shared query parameters for a vector-only scored search.
+ *
+ * Returns params in order: $1=embedding, $2=userId, $3=limit, $4=wSim, $5=wImp, $6=wRec, $7=refTime, $8=rankingMinSimilarity, [$9=sourceSite]
+ */
+export declare function buildVectorSearchParams(queryEmbedding: number[], userId: string, limit: number, sourceSite?: string, referenceTime?: Date): {
+    params: unknown[];
+    siteClause: string;
+    wSim: number;
+    wImp: number;
+    wRec: number;
+    rankingMinSimilarity: number;
+    refTime: string;
+};
+export declare function clampUnit(value: number): number;

package/dist/db/query-helpers.js

@@ -0,0 +1,60 @@
+/**
+ * Shared query-building helpers for vector search and hybrid search backends.
+ *
+ * Centralizes scoring weight preparation, parameter setup, and site filtering
+ * to eliminate duplication between repository-vector-search.ts and
+ * repository-representations.ts.
+ */
+import pgvector from 'pgvector/pg';
+import { config } from '../config.js';
+export const RRF_K = 60;
+/**
+ * Build shared query parameters for a hybrid (vector + FTS) search.
+ *
+ * Returns an array of positional params, the site filter clause, and
+ * scoring weights. The caller must set the first params in a known order:
+ *   $1=embedding, $2=userId, $3=queryText, $4=limit, $5=wSim, $6=wImp, $7=wRec, $8=refTime, $9=rankingMinSimilarity, [$10=sourceSite]
+ */
+export function buildHybridSearchParams(queryEmbedding, userId, queryText, limit, siteFilterColumn, sourceSite, referenceTime) {
+    const wSim = config.scoringWeightSimilarity;
+    const wImp = config.scoringWeightImportance;
+    const wRec = config.scoringWeightRecency;
+    const rankingMinSimilarity = clampUnit(config.retrievalProfileSettings.rankingMinSimilarity);
+    const refTime = (referenceTime ?? new Date()).toISOString();
+    const siteFilter = sourceSite ? `AND ${siteFilterColumn} = $10` : '';
+    const params = [
+        pgvector.toSql(queryEmbedding),
+        userId,
+        queryText,
+        Math.max(1, limit),
+        wSim, wImp, wRec, refTime, rankingMinSimilarity,
+    ];
+    if (sourceSite)
+        params.push(sourceSite);
+    return { params, siteFilter, refTime, wSim, wImp, wRec, rankingMinSimilarity };
+}
+/**
+ * Build shared query parameters for a vector-only scored search.
+ *
+ * Returns params in order: $1=embedding, $2=userId, $3=limit, $4=wSim, $5=wImp, $6=wRec, $7=refTime, $8=rankingMinSimilarity, [$9=sourceSite]
+ */
+export function buildVectorSearchParams(queryEmbedding, userId, limit, sourceSite, referenceTime) {
+    const wSim = config.scoringWeightSimilarity;
+    const wImp = config.scoringWeightImportance;
+    const wRec = config.scoringWeightRecency;
+    const rankingMinSimilarity = clampUnit(config.retrievalProfileSettings.rankingMinSimilarity);
+    const refTime = (referenceTime ?? new Date()).toISOString();
+    const siteClause = sourceSite ? 'AND source_site = $9' : '';
+    const params = [
+        pgvector.toSql(queryEmbedding), userId, Math.max(1, Math.min(100, limit)),
+        wSim, wImp, wRec, refTime, rankingMinSimilarity,
+    ];
+    if (sourceSite)
+        params.push(sourceSite);
+    return { params, siteClause, wSim, wImp, wRec, rankingMinSimilarity, refTime };
+}
+export function clampUnit(value) {
+    if (!Number.isFinite(value))
+        return 0;
+    return Math.max(0, Math.min(1, value));
+}

package/dist/db/raw-doc-artifact-sync.d.ts

@@ -0,0 +1,128 @@
+/**
+ * @file Sync hook keeping `storage_artifacts.status` in lockstep
+ * with `raw_documents.raw_storage_status`.
+ *
+ * Step 7 of the storage-sibling plan. Every existing raw-storage
+ * transition (upload finalize, reconciler success/failure, cleanup
+ * success/failure) calls this hook after mutating `raw_documents`
+ * so the paired `storage_artifacts` row tracks the same lifecycle.
+ *
+ * v1 keeps `raw_documents` as the source of truth for document-
+ * backed uploads — the artifact row is a synchronized projection.
+ * The follow-up Webapp Storage UI PR (source plan Phase 8) makes
+ * `storage_artifacts` the source of truth and switches the document
+ * formatter to a join-projection. Until then, the helper guarantees
+ * both rows agree.
+ *
+ * Status mapping (plan rev 6 §Step 7):
+ *
+ *   pointer_recorded (with external_uri / pointer artifact) → 'stored'
+ *   pointer_recorded (no artifact linked)                   → no-op
+ *   blob_stored                                              → 'stored'
+ *   blob_pending                                             → 'pending'
+ *   blob_uploading                                           → 'pending' (transient)
+ *   blob_available                                           → 'available'
+ *   blob_archival_failed                                     → 'failed'
+ *   raw_storage_failed                                       → 'failed'
+ *   blob_tombstoned                                          → 'deleted'
+ *   blob_deleted                                             → 'deleted'
+ *   inline_text_stored                                       → no-op (inline; no artifact)
+ */
+import type pg from 'pg';
+import type { RawStorageStatus } from './raw-document-types.js';
+/**
+ * Canonical raw_storage cleanup-failure envelope. Persisted to both
+ * `raw_documents.last_error` (via `markCleanupFailedAndSyncArtifact`)
+ * and the linked `storage_artifacts.last_error`. The provider's raw
+ * exception message is funneled through `buildLastError` so the
+ * central `sanitizeLastErrorMessage` cap (1 KiB) and control-char
+ * collapse apply uniformly — multi-line stack traces, sensitive
+ * URLs in error blobs, and oversized provider envelopes never reach
+ * the JSONB column unfiltered. The `storage_provider` field is
+ * appended after sanitization so the envelope shape stays
+ * additive on top of the canonical `LastError`.
+ */
+export declare function buildRawStorageCleanupFailureEnvelope(message: string, storageProvider: string): Record<string, unknown>;
+export interface SyncArtifactStatusInput {
+    rawDocumentId: string;
+    newRawStatus: RawStorageStatus;
+    /**
+     * Optional failure envelope to record on the artifact's
+     * `last_error` column. Pass only on failure transitions; the
+     * helper clears `last_error` on success transitions (`stored` /
+     * `pending` / `available` / `deleted`).
+     */
+    lastError?: Record<string, unknown>;
+}
+/**
+ * Map a single `raw_documents` row's new status onto its linked
+ * `storage_artifacts` row. Resolves the link inside the same query
+ * pair (no separate read) so the hook is safe to call inside the
+ * existing transactions that wrap the raw-storage transitions.
+ *
+ * No-op (silently) when:
+ *   - the document is not linked to an artifact (`storage_artifact_id IS NULL`),
+ *   - the new raw status maps to no artifact state (e.g. `inline_text_stored`).
+ *
+ * The function deliberately does NOT scope by `user_id` — the
+ * caller has already vouched for the row by holding the row's
+ * advisory lock or having authenticated the request that triggered
+ * the transition.
+ */
+export declare function syncArtifactStatusFromRawDocument(client: pg.PoolClient | pg.Pool, args: SyncArtifactStatusInput): Promise<void>;
+/**
+ * Reconciler paired promote: flip the document to `blob_available`
+ * AND sync the artifact in one transaction. Two-row drift is
+ * impossible — either both rows commit or both roll back.
+ */
+export declare function promoteAndSyncArtifact(pool: pg.Pool, args: {
+    rowId: string;
+    claimId: string;
+    provider: string;
+    providerFields: Record<string, unknown>;
+}): Promise<number>;
+/**
+ * Reconciler paired archival-fail: flip the document to
+ * `blob_archival_failed` AND sync the artifact (with the
+ * `last_error` envelope) in one transaction.
+ */
+export declare function markArchivalFailedAndSyncArtifact(pool: pg.Pool, args: {
+    rowId: string;
+    claimId: string;
+    lastError: Record<string, unknown>;
+    provider: string;
+}): Promise<number>;
+/**
+ * Document-keyed paired marker for the cleanup path. Each blob
+ * cleanup carries its `rawDocumentId` (see
+ * `ManagedBlobRefRow.rawDocumentId`) so the sync never needs to
+ * disambiguate by URI. The raw_documents UPDATE is owner-scoped;
+ * if zero rows match (cross-user mismatch or already-terminal),
+ * the artifact sync is skipped — preventing a mismatched caller
+ * from flipping another user's artifact.
+ */
+export declare function markCleanupSuccessAndSyncArtifact(pool: pg.Pool, args: {
+    userId: string;
+    rawDocumentId: string;
+    storageUri: string;
+    semantics: 'deleted' | 'unpinned' | 'tombstoned';
+}): Promise<void>;
+/**
+ * Document-keyed paired failure marker. Mirrors
+ * `markCleanupSuccessAndSyncArtifact` but for the failure branch
+ * (`raw_storage_failed`). Same owner-scope gating.
+ *
+ * Retry-friendly: the UPDATE matches on `(user_id, id)` only — a row
+ * that is ALREADY `raw_storage_failed` is re-marked so a fresh
+ * provider error replaces the prior envelope on both
+ * `raw_documents.last_error` AND the linked
+ * `storage_artifacts.last_error`. Mirrors the failure-marker pattern
+ * elsewhere in the codebase, where retries land the newest, most-
+ * useful error rather than discarding it. Cross-user mismatches
+ * still roll back via the owner-scoped WHERE.
+ */
+export declare function markCleanupFailedAndSyncArtifact(pool: pg.Pool, args: {
+    userId: string;
+    rawDocumentId: string;
+    lastError?: Record<string, unknown>;
+}): Promise<void>;