@atomicmemory/core 1.0.0

package/dist/db/raw-storage-reconciliation-repository.js

@@ -0,0 +1,200 @@
+/**
+ * Raw-storage reconciler DB layer. The reconciler promotes
+ * `raw_storage_status='blob_pending'` rows on eventual providers once the
+ * adapter's `head()` confirms gateway retrievability (→
+ * `blob_available`) or, after exhausted retries, marks them
+ * `blob_archival_failed`.
+ *
+ * Concurrency model (rev-2 §3 + rev-6 §1): claim-then-release. Phase
+ * A grabs a batch in a short transaction with `FOR UPDATE SKIP
+ * LOCKED` so multiple reconciler instances partition the work
+ * deterministically without coordinator overhead. Slow
+ * network calls run with NO DB locks held. failure-transition's per-row UPDATE
+ * is guarded on `raw_storage_claim_id` so a stale claim cannot
+ * clobber a row another worker has since reclaimed.
+ *
+ * Eligibility predicate gates on TYPED columns only — no JSONB casts
+ * (rev-6 §2): a malformed JSONB shape can't masquerade as a fresh
+ * claim. The predicate also explicitly excludes `blob_uploading` rows
+ * — those recover through `uploadRaw`'s same-bytes idempotent retry,
+ * not the reconciler.
+ */
+const RECONCILABLE_PROVIDERS = new Set(['filecoin']);
+/**
+ * claim step: claim a batch of eligible eventual-provider `blob_pending` rows by
+ * writing `raw_storage_claim_id` + `raw_storage_claimed_at`. Skips
+ * rows another worker holds a fresh claim on. Returns the typed row
+ * state the reconciler will probe; the network call runs after this
+ * function returns + the caller commits the transaction.
+ *
+ * Eligibility predicate (rev-7 §3 — two-axis: claim ownership AND
+ * scheduled backoff):
+ *   - `raw_storage_status = 'blob_pending'`
+ *   - `storage_provider = args.provider`
+ *   - `storage_uri IS NOT NULL` (rev-7 §1: the claim step-only rows with no
+ *     URI recover via uploadRaw idempotency, NOT the reconciler)
+ *   - `deleted_at IS NULL`
+ *   - backoff elapsed: `next_check_at IS NULL OR next_check_at <= NOW()`
+ *   - no live owner: claim_id NULL OR claimed_at older than staleAfterMs
+ */
+export async function claimReconcileBatch(pool, args) {
+    assertReconciliableProvider(args.provider);
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const result = await client.query(
+        // Capture the prior `raw_storage_claim_id` in the CTE so the
+        // RETURNING clause can flag stale-claim recoveries — RETURNING
+        // sees the NEW row only, but joining against the captured CTE
+        // (`UPDATE … FROM claimed`) makes the pre-update value
+        // visible. The reconciler emits
+        // `filecoin.reconcile.stale_claim_recovered` based on this flag.
+        `WITH claimed AS (
+         SELECT id, raw_storage_claim_id AS prior_claim_id
+          FROM raw_documents
+          WHERE raw_storage_status = 'blob_pending'
+            AND storage_provider = $4
+            AND storage_uri IS NOT NULL
+            AND deleted_at IS NULL
+            AND (raw_storage_next_check_at IS NULL OR raw_storage_next_check_at <= NOW())
+            AND (
+              raw_storage_claim_id IS NULL
+              OR raw_storage_claimed_at < NOW() - ($1::bigint * INTERVAL '1 millisecond')
+            )
+          ORDER BY COALESCE(raw_storage_next_check_at, raw_storage_claimed_at, updated_at) ASC
+          LIMIT $2
+          FOR UPDATE SKIP LOCKED
+       )
+       UPDATE raw_documents rd
+          SET raw_storage_claim_id = $3::text,
+              raw_storage_claimed_at = NOW(),
+              updated_at = NOW()
+         FROM claimed
+        WHERE rd.id = claimed.id
+        RETURNING rd.id, rd.user_id, rd.storage_uri, rd.storage_provider,
+                  rd.content_hash, rd.raw_storage_status, rd.raw_storage_metadata,
+                  rd.raw_storage_reconcile_attempts, rd.raw_storage_pending_since,
+                  (claimed.prior_claim_id IS NOT NULL) AS recovered_stale_claim`, [args.staleAfterMs, args.batchSize, args.claimId, args.provider]);
+        await client.query('COMMIT');
+        return result.rows.map(toClaimedRow);
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw err;
+    }
+    finally {
+        client.release();
+    }
+}
+function assertReconciliableProvider(provider) {
+    if (!RECONCILABLE_PROVIDERS.has(provider)) {
+        throw new Error(`raw-storage reconciler does not support provider '${provider}'`);
+    }
+}
+function toClaimedRow(row) {
+    return {
+        id: row.id,
+        userId: row.user_id,
+        storageUri: row.storage_uri,
+        storageProvider: row.storage_provider,
+        contentHash: row.content_hash ?? null,
+        rawStorageStatus: 'blob_pending',
+        rawStorageMetadata: row.raw_storage_metadata ?? {},
+        rawStorageReconcileAttempts: typeof row.raw_storage_reconcile_attempts === 'number'
+            ? row.raw_storage_reconcile_attempts
+            : 0,
+        rawStoragePendingSince: parseTimestamp(row.raw_storage_pending_since),
+        recoveredStaleClaim: row.recovered_stale_claim === true,
+    };
+}
+function parseTimestamp(value) {
+    if (value === null || value === undefined)
+        return null;
+    if (value instanceof Date)
+        return value;
+    if (typeof value === 'string') {
+        const parsed = new Date(value);
+        return Number.isNaN(parsed.getTime()) ? null : parsed;
+    }
+    return null;
+}
+/**
+ * failure-transition — success: promote to `blob_available`. Guarded on claim_id
+ * + status='blob_pending' so a stale claim cannot promote a row
+ * another worker has since failed. Deep-merges the new
+ * provider keys into the existing sibling (rev-4 §3) so a status-
+ * only probe doesn't drop `cid`/`piece_cid`/`deals` the original
+ * upload wrote. Layer-scoped `last_error` clear (rev-4 §2): drops
+ * the envelope only when it was scoped to `raw_storage`.
+ */
+export async function promoteToAvailableWithClient(pool, args) {
+    assertReconciliableProvider(args.provider);
+    const result = await pool.query(`UPDATE raw_documents
+        SET raw_storage_status = 'blob_available',
+            raw_storage_metadata = raw_storage_metadata ||
+              jsonb_build_object(
+                $1::text,
+                COALESCE(raw_storage_metadata->$1::text, '{}'::jsonb) || $2::jsonb
+              ),
+            raw_storage_claim_id = NULL,
+            raw_storage_claimed_at = NULL,
+            raw_storage_last_checked_at = NOW(),
+            raw_storage_next_check_at = NULL,
+            raw_storage_reconcile_attempts = 0,
+            raw_storage_pending_since = NULL,
+            last_error = CASE
+              WHEN last_error IS NOT NULL AND last_error->>'layer' = 'raw_storage'
+              THEN NULL
+              ELSE last_error
+            END,
+            updated_at = NOW()
+      WHERE id = $3
+        AND raw_storage_status = 'blob_pending'
+        AND storage_provider = $1
+        AND raw_storage_claim_id = $4`, [args.provider, JSON.stringify(args.providerFields), args.rowId, args.claimId]);
+    return result.rowCount ?? 0;
+}
+/**
+ * failure-transition — still-pending probe: clear the claim (release ownership)
+ * + increment attempts + advance `next_check_at` per the
+ * caller-computed backoff. Status STAYS `blob_pending` and
+ * `pending_since` is preserved.
+ */
+export async function markStillPendingWithClient(pool, args) {
+    assertReconciliableProvider(args.provider);
+    const result = await pool.query(`UPDATE raw_documents
+        SET raw_storage_claim_id = NULL,
+            raw_storage_claimed_at = NULL,
+            raw_storage_last_checked_at = NOW(),
+            raw_storage_next_check_at = NOW() + ($1::bigint * INTERVAL '1 millisecond'),
+            raw_storage_reconcile_attempts = raw_storage_reconcile_attempts + 1,
+            updated_at = NOW()
+      WHERE id = $2
+        AND raw_storage_status = 'blob_pending'
+        AND storage_provider = $4
+        AND raw_storage_claim_id = $3`, [args.nextCheckAtMs, args.rowId, args.claimId, args.provider]);
+    return result.rowCount ?? 0;
+}
+/**
+ * failure-transition — terminal failure: status → `blob_archival_failed`. Sets
+ * a fresh raw-storage `last_error` envelope, clears claim/pending
+ * state, resets attempts. Guarded on claim_id.
+ */
+export async function markArchivalFailedWithClient(pool, args) {
+    assertReconciliableProvider(args.provider);
+    const result = await pool.query(`UPDATE raw_documents
+        SET raw_storage_status = 'blob_archival_failed',
+            raw_storage_claim_id = NULL,
+            raw_storage_claimed_at = NULL,
+            raw_storage_last_checked_at = NOW(),
+            raw_storage_next_check_at = NULL,
+            raw_storage_reconcile_attempts = 0,
+            raw_storage_pending_since = NULL,
+            last_error = $1::jsonb,
+            updated_at = NOW()
+      WHERE id = $2
+        AND raw_storage_status = 'blob_pending'
+        AND storage_provider = $4
+        AND raw_storage_claim_id = $3`, [JSON.stringify(args.lastError), args.rowId, args.claimId, args.provider]);
+    return result.rowCount ?? 0;
+}

package/dist/db/reflection-jobs-repository.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Postgres-backed work queue for the async Reflect step.
+ *
+ * Idempotent enqueue: a unique partial index on (user_id, conversation_id)
+ * WHERE status IN ('pending','in_progress') guarantees one in-flight job per
+ * conversation at a time. Re-enqueue after completion creates a new job (the
+ * unique index excludes 'completed' and 'failed').
+ *
+ * The worker (services/reflect-jobs.ts) drives the lifecycle: fetchPending →
+ * markInProgress → run reflect → markCompleted | markFailed.
+ */
+import pg from 'pg';
+export type JobStatus = 'pending' | 'in_progress' | 'completed' | 'failed';
+export interface ReflectionJob {
+    id: string;
+    userId: string;
+    conversationId: string;
+    status: JobStatus;
+    attempts: number;
+    lastError: string | null;
+    createdAt: Date;
+    lastTriedAt: Date | null;
+}
+export declare class ReflectionJobsRepository {
+    private readonly pool;
+    constructor(pool: pg.Pool);
+    enqueue(userId: string, conversationId: string): Promise<void>;
+    fetchPending(limit: number): Promise<ReflectionJob[]>;
+    markInProgress(id: string): Promise<void>;
+    markCompleted(id: string): Promise<void>;
+    markFailed(id: string, error: string): Promise<void>;
+    findById(id: string): Promise<ReflectionJob | null>;
+}

package/dist/db/reflection-jobs-repository.js

@@ -0,0 +1,48 @@
+export class ReflectionJobsRepository {
+    pool;
+    constructor(pool) {
+        this.pool = pool;
+    }
+    async enqueue(userId, conversationId) {
+        await this.pool.query(`INSERT INTO reflection_jobs (user_id, conversation_id) VALUES ($1, $2)
+       ON CONFLICT DO NOTHING`, [userId, conversationId]);
+    }
+    async fetchPending(limit) {
+        const { rows } = await this.pool.query(`SELECT id, user_id, conversation_id, status, attempts, last_error,
+              created_at, last_tried_at
+       FROM reflection_jobs
+       WHERE status = 'pending'
+       ORDER BY created_at ASC
+       LIMIT $1`, [limit]);
+        return rows.map(mapJob);
+    }
+    async markInProgress(id) {
+        await this.pool.query(`UPDATE reflection_jobs
+       SET status = 'in_progress', attempts = attempts + 1, last_tried_at = now()
+       WHERE id = $1`, [id]);
+    }
+    async markCompleted(id) {
+        await this.pool.query(`UPDATE reflection_jobs SET status = 'completed' WHERE id = $1`, [id]);
+    }
+    async markFailed(id, error) {
+        await this.pool.query(`UPDATE reflection_jobs SET status = 'failed', last_error = $2 WHERE id = $1`, [id, error]);
+    }
+    async findById(id) {
+        const { rows } = await this.pool.query(`SELECT id, user_id, conversation_id, status, attempts, last_error,
+              created_at, last_tried_at
+       FROM reflection_jobs WHERE id = $1`, [id]);
+        return rows[0] ? mapJob(rows[0]) : null;
+    }
+}
+function mapJob(r) {
+    return {
+        id: r.id,
+        userId: r.user_id,
+        conversationId: r.conversation_id,
+        status: r.status,
+        attempts: r.attempts,
+        lastError: r.last_error,
+        createdAt: r.created_at,
+        lastTriedAt: r.last_tried_at,
+    };
+}

package/dist/db/reflections-repository.d.ts

@@ -0,0 +1,41 @@
+/**
+ * ReflectionsRepository — CRUD plus cosine-similarity search for the
+ * session_reflections table. Each row is an LLM-synthesized observation about
+ * a conversation, with citations to the supporting memory ids and an embedding
+ * for retrieval-side similarity search.
+ *
+ * Pure SQL via pg.Pool. No ORM. Mutations fail closed: caller catches errors,
+ * we propagate them with the original error attached.
+ */
+import pg from 'pg';
+export type ObservationType = 'entity_state' | 'event_summary' | 'preference' | 'contradiction' | 'decision' | 'numeric_value';
+export interface NewReflection {
+    userId: string;
+    conversationId: string;
+    observation: string;
+    observationType: ObservationType;
+    evidenceMemoryIds: string[];
+    embedding: number[];
+}
+export interface Reflection extends NewReflection {
+    id: string;
+    createdAt: Date;
+}
+export declare class ReflectionsRepository {
+    private readonly pool;
+    constructor(pool: pg.Pool);
+    /**
+     * Insert multiple reflections in a single transaction. On any error,
+     * the entire transaction is rolled back.
+     */
+    insertMany(rows: readonly NewReflection[]): Promise<void>;
+    /**
+     * Find all reflections for a (userId, conversationId) pair, ordered by creation time.
+     */
+    findByConversation(userId: string, conversationId: string): Promise<Reflection[]>;
+    /**
+     * Find the top-K reflections most similar to queryEmbedding for a userId,
+     * ordered by cosine distance (closest first).
+     */
+    findSimilar(userId: string, queryEmbedding: number[], topK: number): Promise<Reflection[]>;
+}

package/dist/db/reflections-repository.js

@@ -0,0 +1,83 @@
+import pgvector from 'pgvector/pg';
+export class ReflectionsRepository {
+    pool;
+    constructor(pool) {
+        this.pool = pool;
+    }
+    /**
+     * Insert multiple reflections in a single transaction. On any error,
+     * the entire transaction is rolled back.
+     */
+    async insertMany(rows) {
+        if (rows.length === 0)
+            return;
+        const sql = `
+      INSERT INTO session_reflections
+        (user_id, conversation_id, observation, observation_type, evidence_memory_ids, embedding)
+      VALUES ($1, $2, $3, $4, $5, $6::vector)
+    `;
+        const client = await this.pool.connect();
+        try {
+            await client.query('BEGIN');
+            for (const r of rows) {
+                await client.query(sql, [
+                    r.userId,
+                    r.conversationId,
+                    r.observation,
+                    r.observationType,
+                    r.evidenceMemoryIds,
+                    pgvector.toSql(r.embedding),
+                ]);
+            }
+            await client.query('COMMIT');
+        }
+        catch (e) {
+            await client.query('ROLLBACK');
+            throw e;
+        }
+        finally {
+            client.release();
+        }
+    }
+    /**
+     * Find all reflections for a (userId, conversationId) pair, ordered by creation time.
+     */
+    async findByConversation(userId, conversationId) {
+        const { rows } = await this.pool.query(`SELECT id, user_id, conversation_id, observation, observation_type,
+              evidence_memory_ids, created_at
+       FROM session_reflections
+       WHERE user_id = $1 AND conversation_id = $2
+       ORDER BY created_at ASC`, [userId, conversationId]);
+        return rows.map(mapRow);
+    }
+    /**
+     * Find the top-K reflections most similar to queryEmbedding for a userId,
+     * ordered by cosine distance (closest first).
+     */
+    async findSimilar(userId, queryEmbedding, topK) {
+        const { rows } = await this.pool.query(`SELECT id, user_id, conversation_id, observation, observation_type,
+              evidence_memory_ids, created_at
+       FROM session_reflections
+       WHERE user_id = $1
+       ORDER BY embedding <=> $2::vector
+       LIMIT $3`, [userId, pgvector.toSql(queryEmbedding), topK]);
+        return rows.map(mapRow);
+    }
+}
+/**
+ * Map a Postgres query result row to a Reflection object.
+ * Excludes the embedding vector from the returned object (not needed for
+ * app logic, only for similarity search on insertion/retrieval).
+ */
+function mapRow(r) {
+    return {
+        id: r.id,
+        userId: r.user_id,
+        conversationId: r.conversation_id,
+        observation: r.observation,
+        observationType: r.observation_type,
+        evidenceMemoryIds: r.evidence_memory_ids,
+        embedding: [],
+        createdAt: r.created_at,
+    };
+}

package/dist/db/repository-claims.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Structured claim/version history repository.
+ */
+import pg from 'pg';
+import { type ClaimRow, type ClaimVersionRow, type MutationSummary, type MutationType, type RelationType, type SearchResult } from './repository-types.js';
+export interface MutationProvenance {
+    mutationType: MutationType;
+    mutationReason?: string;
+    previousVersionId?: string;
+    actorModel?: string;
+    contradictionConfidence?: number;
+}
+export interface ClaimSlotInput {
+    slotKey: string;
+    subjectEntityId: string;
+    relationType: RelationType;
+    objectEntityId: string;
+}
+export interface ClaimSlotTarget {
+    claimId: string;
+    versionId: string;
+    memoryId: string;
+}
+export interface SlotBackfillCandidate {
+    claimId: string;
+    userId: string;
+    memoryId: string;
+}
+export declare class ClaimRepository {
+    private pool;
+    constructor(pool: pg.Pool);
+    createClaim(userId: string, claimType: string, validAt?: Date, slot?: ClaimSlotInput | null): Promise<string>;
+    createClaimWithClient(client: pg.PoolClient, userId: string, claimType: string, validAt?: Date, slot?: ClaimSlotInput | null): Promise<string>;
+    getClaim(id: string, userId: string): Promise<ClaimRow | null>;
+    getActiveClaimTargetBySlot(userId: string, slotKey: string): Promise<ClaimSlotTarget | null>;
+    listClaimsMissingSlots(userId: string): Promise<SlotBackfillCandidate[]>;
+    updateClaimSlot(userId: string, claimId: string, slot: ClaimSlotInput): Promise<void>;
+    setClaimCurrentVersion(claimId: string, versionId: string | null, status?: string, validAt?: Date): Promise<void>;
+    setClaimCurrentVersionWithClient(client: pg.PoolClient, claimId: string, versionId: string | null, status?: string, validAt?: Date): Promise<void>;
+    invalidateClaim(userId: string, claimId: string, invalidAt?: Date, invalidatedByVersionId?: string | null, status?: string): Promise<void>;
+    invalidateClaimWithClient(client: pg.PoolClient, userId: string, claimId: string, invalidAt?: Date, invalidatedByVersionId?: string | null, status?: string): Promise<void>;
+    createClaimVersion(input: {
+        claimId: string;
+        userId: string;
+        memoryId?: string;
+        content: string;
+        embedding: number[];
+        importance: number;
+        sourceSite: string;
+        sourceUrl?: string;
+        episodeId?: string;
+        validFrom?: Date;
+        provenance?: MutationProvenance;
+    }): Promise<string>;
+    createClaimVersionWithClient(client: pg.PoolClient, input: {
+        claimId: string;
+        userId: string;
+        memoryId?: string;
+        content: string;
+        embedding: number[];
+        importance: number;
+        sourceSite: string;
+        sourceUrl?: string;
+        episodeId?: string;
+        validFrom?: Date;
+        provenance?: MutationProvenance;
+    }): Promise<string>;
+    getClaimVersionByMemoryId(userId: string, memoryId: string): Promise<ClaimVersionRow | null>;
+    getClaimVersionByMemoryIdWithClient(client: pg.PoolClient, userId: string, memoryId: string): Promise<ClaimVersionRow | null>;
+    getClaimVersion(versionId: string, userId: string): Promise<ClaimVersionRow | null>;
+    getClaimVersionAtTime(claimId: string, userId: string, asOf: string): Promise<ClaimVersionRow | null>;
+    listClaimVersions(claimId: string): Promise<ClaimVersionRow[]>;
+    /**
+     * Create a new version for an UPDATE mutation, preserving the old version
+     * in the version history. The old version gets valid_to set and the new
+     * version becomes current.
+     */
+    createUpdateVersion(input: {
+        oldVersionId: string;
+        claimId: string;
+        userId: string;
+        memoryId: string;
+        content: string;
+        embedding: number[];
+        importance: number;
+        sourceSite: string;
+        sourceUrl?: string;
+        episodeId?: string;
+        validFrom?: Date;
+        mutationReason?: string;
+        actorModel?: string;
+    }): Promise<string>;
+    supersedeClaimVersion(userId: string, versionId: string, supersededByVersionId: string | null, validTo?: Date): Promise<void>;
+    supersedeClaimVersionWithClient(client: pg.PoolClient, userId: string, versionId: string, supersededByVersionId: string | null, validTo?: Date): Promise<void>;
+    addEvidence(input: {
+        claimVersionId: string;
+        episodeId?: string;
+        memoryId?: string;
+        quoteText?: string;
+        speaker?: string;
+    }): Promise<void>;
+    addEvidenceWithClient(client: pg.PoolClient, input: {
+        claimVersionId: string;
+        episodeId?: string;
+        memoryId?: string;
+        quoteText?: string;
+        speaker?: string;
+    }): Promise<void>;
+    searchClaimVersions(userId: string, queryEmbedding: number[], limit: number, asOf: string, sourceSite?: string): Promise<SearchResult[]>;
+    countClaims(userId: string): Promise<number>;
+    countOpenClaimVersions(claimId: string): Promise<number>;
+    /**
+     * Get the full mutation history for a claim, ordered chronologically.
+     * Returns all versions including superseded ones, with provenance metadata.
+     */
+    getMutationHistory(claimId: string): Promise<ClaimVersionRow[]>;
+    /**
+     * Find the claim associated with a memory ID (even if the memory is deleted).
+     */
+    findClaimByMemoryId(userId: string, memoryId: string): Promise<{
+        claimId: string;
+        versions: ClaimVersionRow[];
+    } | null>;
+    /**
+     * Get aggregate mutation stats for a user: counts by mutation type,
+     * total versions, active vs superseded counts. Used for mutation audit trail.
+     */
+    getUserMutationSummary(userId: string): Promise<MutationSummary>;
+    /**
+     * Trace the supersession chain forward from a given version ID.
+     * Returns the sequence of versions that replaced each other,
+     * enabling "why did this memory change?" debugging.
+     */
+    getReversalChain(userId: string, startVersionId: string, maxDepth?: number): Promise<ClaimVersionRow[]>;
+    /**
+     * Get recent mutations across all claims for a user, ordered newest first.
+     * Useful for "what changed recently?" debugging.
+     */
+    getRecentMutations(userId: string, limit?: number): Promise<ClaimVersionRow[]>;
+    deleteAll(userId?: string): Promise<void>;
+}