cogmem 3.6.2

package/dist/batch/IngestionCursorStore.d.ts

@@ -0,0 +1,52 @@
+import type { SourceAdapterKind, SourceDefinition } from '../adapters/types.js';
+export interface IngestionSourceCursor {
+    sourceId: string;
+    sourcePath: string;
+    sourceType: SourceAdapterKind;
+    projectId?: string;
+    enabled: boolean;
+    lastProcessedAt?: number;
+    lastSeenHash?: string;
+    lastSeenMtime?: number;
+    contentWindowStart?: number;
+    contentWindowEnd?: number;
+    updatedAt: number;
+}
+export interface ProcessedSourceRecord {
+    recordHash: string;
+    sourceId: string;
+    sourcePath: string;
+    sourceType: SourceAdapterKind;
+    contentHash: string;
+    contentWindowStart: number;
+    contentWindowEnd: number;
+    processedAt: number;
+    neuronId?: string;
+}
+export declare class IngestionCursorStore {
+    private db;
+    private closed;
+    constructor(dbPath?: string);
+    private initializeSchema;
+    registerSource(source: SourceDefinition): void;
+    listRegisteredSources(): IngestionSourceCursor[];
+    getCursor(sourceId: string): IngestionSourceCursor | null;
+    hasProcessedRecord(recordHash: string): boolean;
+    markRecordProcessed(record: ProcessedSourceRecord): void;
+    updateCursor(input: {
+        sourceId: string;
+        sourcePath: string;
+        sourceType: SourceAdapterKind;
+        projectId?: string;
+        lastProcessedAt: number;
+        lastSeenHash: string;
+        lastSeenMtime: number;
+        contentWindowStart: number;
+        contentWindowEnd: number;
+    }): void;
+    listProcessedRecordHashes(sourceId: string, windowStart: number, windowEnd: number): Set<string>;
+    listRecentUnprocessedSources(since: number): IngestionSourceCursor[];
+    close(): void;
+    private mapCursor;
+}
+//# sourceMappingURL=IngestionCursorStore.d.ts.map

package/dist/batch/IngestionCursorStore.js

@@ -0,0 +1,153 @@
+import Database from 'bun:sqlite';
+export class IngestionCursorStore {
+    db;
+    closed = false;
+    constructor(dbPath = ':memory:') {
+        this.db = new Database(dbPath);
+        this.initializeSchema();
+    }
+    initializeSchema() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ingestion_source_cursors (
+        source_id TEXT PRIMARY KEY,
+        source_path TEXT NOT NULL,
+        source_type TEXT NOT NULL,
+        project_id TEXT,
+        enabled INTEGER NOT NULL DEFAULT 1,
+        last_processed_at INTEGER,
+        last_seen_hash TEXT,
+        last_seen_mtime INTEGER,
+        content_window_start INTEGER,
+        content_window_end INTEGER,
+        updated_at INTEGER NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS ingestion_processed_records (
+        record_hash TEXT PRIMARY KEY,
+        source_id TEXT NOT NULL,
+        source_path TEXT NOT NULL,
+        source_type TEXT NOT NULL,
+        content_hash TEXT NOT NULL,
+        content_window_start INTEGER NOT NULL,
+        content_window_end INTEGER NOT NULL,
+        processed_at INTEGER NOT NULL,
+        neuron_id TEXT
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_ingestion_processed_source_window
+        ON ingestion_processed_records(source_id, content_window_start, content_window_end, processed_at DESC);
+    `);
+    }
+    registerSource(source) {
+        const now = Date.now();
+        this.db.prepare(`
+      INSERT INTO ingestion_source_cursors (
+        source_id, source_path, source_type, project_id, enabled, updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT(source_id) DO UPDATE SET
+        source_path = excluded.source_path,
+        source_type = excluded.source_type,
+        project_id = excluded.project_id,
+        enabled = excluded.enabled,
+        updated_at = excluded.updated_at
+    `).run(source.sourceId, source.sourcePath, source.adapterKind, source.projectId || null, source.enabled === false ? 0 : 1, now);
+    }
+    listRegisteredSources() {
+        const rows = this.db.prepare(`
+      SELECT *
+      FROM ingestion_source_cursors
+      WHERE enabled = 1
+      ORDER BY updated_at DESC, source_id ASC
+    `).all();
+        return rows.map((row) => this.mapCursor(row));
+    }
+    getCursor(sourceId) {
+        const row = this.db.prepare(`
+      SELECT *
+      FROM ingestion_source_cursors
+      WHERE source_id = ?
+    `).get(sourceId);
+        return row ? this.mapCursor(row) : null;
+    }
+    hasProcessedRecord(recordHash) {
+        const row = this.db.prepare(`
+      SELECT 1
+      FROM ingestion_processed_records
+      WHERE record_hash = ?
+      LIMIT 1
+    `).get(recordHash);
+        return Boolean(row);
+    }
+    markRecordProcessed(record) {
+        this.db.prepare(`
+      INSERT OR REPLACE INTO ingestion_processed_records (
+        record_hash, source_id, source_path, source_type, content_hash,
+        content_window_start, content_window_end, processed_at, neuron_id
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(record.recordHash, record.sourceId, record.sourcePath, record.sourceType, record.contentHash, record.contentWindowStart, record.contentWindowEnd, record.processedAt, record.neuronId || null);
+    }
+    updateCursor(input) {
+        this.db.prepare(`
+      INSERT INTO ingestion_source_cursors (
+        source_id, source_path, source_type, project_id, enabled,
+        last_processed_at, last_seen_hash, last_seen_mtime,
+        content_window_start, content_window_end, updated_at
+      ) VALUES (?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(source_id) DO UPDATE SET
+        source_path = excluded.source_path,
+        source_type = excluded.source_type,
+        project_id = excluded.project_id,
+        last_processed_at = excluded.last_processed_at,
+        last_seen_hash = excluded.last_seen_hash,
+        last_seen_mtime = excluded.last_seen_mtime,
+        content_window_start = excluded.content_window_start,
+        content_window_end = excluded.content_window_end,
+        updated_at = excluded.updated_at
+    `).run(input.sourceId, input.sourcePath, input.sourceType, input.projectId || null, input.lastProcessedAt, input.lastSeenHash, input.lastSeenMtime, input.contentWindowStart, input.contentWindowEnd, Date.now());
+    }
+    listProcessedRecordHashes(sourceId, windowStart, windowEnd) {
+        const rows = this.db.prepare(`
+      SELECT record_hash
+      FROM ingestion_processed_records
+      WHERE source_id = ?
+        AND content_window_start = ?
+        AND content_window_end = ?
+    `).all(sourceId, windowStart, windowEnd);
+        return new Set(rows.map((row) => row.record_hash));
+    }
+    listRecentUnprocessedSources(since) {
+        const rows = this.db.prepare(`
+      SELECT *
+      FROM ingestion_source_cursors
+      WHERE enabled = 1
+        AND (
+          last_processed_at IS NULL
+          OR content_window_end IS NULL
+          OR content_window_end < ?
+        )
+      ORDER BY updated_at DESC
+    `).all(since);
+        return rows.map((row) => this.mapCursor(row));
+    }
+    close() {
+        if (this.closed)
+            return;
+        this.db.close();
+        this.closed = true;
+    }
+    mapCursor(row) {
+        return {
+            sourceId: row.source_id,
+            sourcePath: row.source_path,
+            sourceType: row.source_type,
+            projectId: row.project_id || undefined,
+            enabled: row.enabled === 1,
+            lastProcessedAt: row.last_processed_at || undefined,
+            lastSeenHash: row.last_seen_hash || undefined,
+            lastSeenMtime: row.last_seen_mtime || undefined,
+            contentWindowStart: row.content_window_start || undefined,
+            contentWindowEnd: row.content_window_end || undefined,
+            updatedAt: row.updated_at
+        };
+    }
+}

package/dist/batch/InstalledBatchProcessor.d.ts

@@ -0,0 +1,88 @@
+import type { IngestInput, Neuron } from '../types/index.js';
+import { type BatchEpisodeEnvelope, type SourceAdapterDiagnostic, type SourceDefinition } from '../adapters/index.js';
+import type { IngestionCursorStore } from './IngestionCursorStore.js';
+import type { OfflineConsolidationOutput } from '../engine/OfflineConsolidationPipeline.js';
+import type { MemoryEvent } from '../types/index.js';
+export interface BatchConsolidationWindow {
+    start: number;
+    end: number;
+    label: string;
+}
+export interface BatchConsolidationRunOptions {
+    window: BatchConsolidationWindow;
+    sources?: SourceDefinition[];
+}
+export interface BatchConsolidationSummary {
+    window: BatchConsolidationWindow;
+    sourcesScanned: number;
+    sourcesChanged: number;
+    recordsParsed: number;
+    recordsIngested: number;
+    skippedRecords: number;
+    processedSourceIds: string[];
+    adapterDiagnostics: SourceAdapterDiagnostic[];
+    sourceResults: BatchSourceResult[];
+    offline: OfflineConsolidationOutput;
+}
+export interface BatchSourceResult {
+    sourceId: string;
+    sourcePath: string;
+    adapterKind: SourceDefinition['adapterKind'];
+    recordsParsed: number;
+    recordsIngested: number;
+    skippedRecords: number;
+    diagnostics: SourceAdapterDiagnostic[];
+}
+export type BatchProgressEvent = {
+    stage: 'source:start';
+    sourceIndex: number;
+    totalSources: number;
+    sourcePath: string;
+    adapterKind: SourceDefinition['adapterKind'];
+} | {
+    stage: 'source:parsed';
+    sourceIndex: number;
+    totalSources: number;
+    sourcePath: string;
+    adapterKind: SourceDefinition['adapterKind'];
+    recordsParsed: number;
+    pendingRecords: number;
+    skippedRecords: number;
+} | {
+    stage: 'source:ingest:start';
+    sourceIndex: number;
+    totalSources: number;
+    sourcePath: string;
+    adapterKind: SourceDefinition['adapterKind'];
+    pendingRecords: number;
+} | {
+    stage: 'source:ingest:complete';
+    sourceIndex: number;
+    totalSources: number;
+    sourcePath: string;
+    adapterKind: SourceDefinition['adapterKind'];
+    ingestedRecords: number;
+    totalRecordsIngested: number;
+} | {
+    stage: 'offline:start';
+    recordsIngested: number;
+} | {
+    stage: 'offline:complete';
+    recordsIngested: number;
+};
+interface InstalledBatchProcessorDependencies {
+    cursorStore: IngestionCursorStore;
+    ingestBatch: (inputs: IngestInput[]) => Promise<Neuron[]>;
+    recordRawEvidence?: (envelope: BatchEpisodeEnvelope) => MemoryEvent | undefined | Promise<MemoryEvent | undefined>;
+    runOfflineWindow: (window: BatchConsolidationWindow) => Promise<OfflineConsolidationOutput>;
+    onProgress?: (event: BatchProgressEvent) => void;
+}
+export declare class InstalledBatchProcessor {
+    private readonly deps;
+    private readonly loader;
+    private readonly adapters;
+    constructor(deps: InstalledBatchProcessorDependencies);
+    runOnce(options: BatchConsolidationRunOptions): Promise<BatchConsolidationSummary>;
+}
+export {};
+//# sourceMappingURL=InstalledBatchProcessor.d.ts.map

package/dist/batch/InstalledBatchProcessor.js

@@ -0,0 +1,179 @@
+import { ConversationMarkdownAdapter, HermesStateDbAdapter, MarkdownSourceLoader, OpenClawDailyMemoryAdapter, OpenClawMemoryIndexAdapter, OpenClawPersonaAdapter, OpenClawSessionAdapter, OpenClawUserProfileAdapter, SoulMarkdownAdapter, buildEpisodeEnvelope } from '../adapters/index.js';
+export class InstalledBatchProcessor {
+    deps;
+    loader = new MarkdownSourceLoader();
+    adapters = new Map([
+        ['conversation_markdown', new ConversationMarkdownAdapter()],
+        ['hermes_state_db', new HermesStateDbAdapter()],
+        ['soul_markdown', new SoulMarkdownAdapter()],
+        ['openclaw_daily_memory', new OpenClawDailyMemoryAdapter()],
+        ['openclaw_session', new OpenClawSessionAdapter()],
+        ['openclaw_memory_index', new OpenClawMemoryIndexAdapter()],
+        ['openclaw_user_profile', new OpenClawUserProfileAdapter()],
+        ['openclaw_persona', new OpenClawPersonaAdapter()]
+    ]);
+    constructor(deps) {
+        this.deps = deps;
+    }
+    async runOnce(options) {
+        const registered = options.sources || [];
+        for (const source of registered) {
+            this.deps.cursorStore.registerSource(source);
+        }
+        const sources = registered.length > 0
+            ? registered
+            : this.deps.cursorStore.listRegisteredSources().map((cursor) => ({
+                sourceId: cursor.sourceId,
+                adapterKind: cursor.sourceType,
+                sourcePath: cursor.sourcePath,
+                projectId: cursor.projectId
+            }));
+        let sourcesChanged = 0;
+        let recordsParsed = 0;
+        let skippedRecords = 0;
+        let recordsIngested = 0;
+        const processedSourceIds = [];
+        const adapterDiagnostics = [];
+        const sourceResults = [];
+        for (const [sourceOffset, source] of sources.entries()) {
+            const adapter = this.adapters.get(source.adapterKind);
+            if (!adapter)
+                continue;
+            const sourceIndex = sourceOffset + 1;
+            this.deps.onProgress?.({
+                stage: 'source:start',
+                sourceIndex,
+                totalSources: sources.length,
+                sourcePath: source.sourcePath,
+                adapterKind: source.adapterKind
+            });
+            const snapshot = this.loader.read(source);
+            const cursor = this.deps.cursorStore.getCursor(source.sourceId);
+            const adapted = adapter.adapt(source, snapshot, source.tags?.includes('ingest:profile_only')
+                ? undefined
+                : {
+                    start: options.window.start,
+                    end: options.window.end
+                });
+            adapterDiagnostics.push(...(adapted.diagnostics || []));
+            const seenHashes = this.deps.cursorStore.listProcessedRecordHashes(source.sourceId, options.window.start, options.window.end);
+            const pending = adapted.records.filter((record) => !seenHashes.has(record.provenance.recordHash));
+            recordsParsed += adapted.records.length;
+            skippedRecords += adapted.records.length - pending.length;
+            this.deps.onProgress?.({
+                stage: 'source:parsed',
+                sourceIndex,
+                totalSources: sources.length,
+                sourcePath: source.sourcePath,
+                adapterKind: source.adapterKind,
+                recordsParsed: adapted.records.length,
+                pendingRecords: pending.length,
+                skippedRecords: adapted.records.length - pending.length
+            });
+            if (!cursor || cursor.lastSeenHash !== snapshot.fileHash || pending.length > 0) {
+                sourcesChanged += 1;
+            }
+            if (pending.length > 0) {
+                this.deps.onProgress?.({
+                    stage: 'source:ingest:start',
+                    sourceIndex,
+                    totalSources: sources.length,
+                    sourcePath: source.sourcePath,
+                    adapterKind: source.adapterKind,
+                    pendingRecords: pending.length
+                });
+                const envelopes = pending.map((record) => buildEpisodeEnvelope(source, record));
+                const inputs = await Promise.all(envelopes.map(async (item) => {
+                    const rawEvent = await this.deps.recordRawEvidence?.(item);
+                    if (!rawEvent)
+                        return item.ingestInput;
+                    const sourceRefs = item.ingestInput.sourceRefs || [];
+                    const rawRef = {
+                        ...(sourceRefs[0] || {}),
+                        eventId: rawEvent.eventId,
+                        eventType: rawEvent.eventType,
+                        contentHash: rawEvent.contentHash,
+                        threadId: rawEvent.threadId,
+                        sessionId: rawEvent.sessionId,
+                        turnId: rawEvent.turnId,
+                        role: rawEvent.role,
+                        threadSeq: rawEvent.threadSeq,
+                        turnSeq: rawEvent.turnSeq,
+                        eventOrdinal: rawEvent.eventOrdinal,
+                        parentEventId: rawEvent.parentEventId,
+                        prevEventId: rawEvent.prevEventId,
+                        nextEventId: rawEvent.nextEventId,
+                        causalityType: rawEvent.causalityType,
+                        orderingConfidence: rawEvent.orderingConfidence,
+                    };
+                    return {
+                        ...item.ingestInput,
+                        sourceEventId: rawEvent.eventId,
+                        sourceRefs: [...sourceRefs, rawRef],
+                    };
+                }));
+                const neurons = await this.deps.ingestBatch(inputs);
+                recordsIngested += envelopes.length;
+                envelopes.forEach((item, index) => {
+                    const neuron = neurons[index];
+                    this.deps.cursorStore.markRecordProcessed({
+                        recordHash: item.record.provenance.recordHash,
+                        sourceId: source.sourceId,
+                        sourcePath: source.sourcePath,
+                        sourceType: source.adapterKind,
+                        contentHash: item.record.provenance.fileHash,
+                        contentWindowStart: options.window.start,
+                        contentWindowEnd: options.window.end,
+                        processedAt: Date.now(),
+                        neuronId: neuron?.id
+                    });
+                });
+                this.deps.onProgress?.({
+                    stage: 'source:ingest:complete',
+                    sourceIndex,
+                    totalSources: sources.length,
+                    sourcePath: source.sourcePath,
+                    adapterKind: source.adapterKind,
+                    ingestedRecords: envelopes.length,
+                    totalRecordsIngested: recordsIngested
+                });
+            }
+            this.deps.cursorStore.updateCursor({
+                sourceId: source.sourceId,
+                sourcePath: source.sourcePath,
+                sourceType: source.adapterKind,
+                projectId: source.projectId,
+                lastProcessedAt: Date.now(),
+                lastSeenHash: snapshot.fileHash,
+                lastSeenMtime: snapshot.fileMtimeMs,
+                contentWindowStart: options.window.start,
+                contentWindowEnd: options.window.end
+            });
+            processedSourceIds.push(source.sourceId);
+            sourceResults.push({
+                sourceId: source.sourceId,
+                sourcePath: source.sourcePath,
+                adapterKind: source.adapterKind,
+                recordsParsed: adapted.records.length,
+                recordsIngested: pending.length,
+                skippedRecords: adapted.records.length - pending.length,
+                diagnostics: adapted.diagnostics || []
+            });
+        }
+        this.deps.onProgress?.({ stage: 'offline:start', recordsIngested });
+        const offline = await this.deps.runOfflineWindow(options.window);
+        this.deps.onProgress?.({ stage: 'offline:complete', recordsIngested });
+        return {
+            window: options.window,
+            sourcesScanned: sources.length,
+            sourcesChanged,
+            recordsParsed,
+            recordsIngested,
+            skippedRecords,
+            processedSourceIds,
+            adapterDiagnostics,
+            sourceResults,
+            offline
+        };
+    }
+}

package/dist/batch/SourceOnboarding.d.ts

@@ -0,0 +1,56 @@
+import { type NormalizationFamily, type NormalizedSourceFamily } from '../utils/ConversationMarkdownNormalization.js';
+import type { SourceAdapterDiagnostic, SourceAdapterKind, SourceDefinition } from '../adapters/index.js';
+export type OnboardingLane = 'conversation_source' | 'soul_like_source' | 'user_profile' | 'agent_persona' | 'memory_index' | 'unknown';
+export interface SourcePreflightReport {
+    sourceId: string;
+    sourcePath: string;
+    adapterKind: SourceAdapterKind;
+    lane: OnboardingLane;
+    recordsParsed: number;
+    recordsPending: number;
+    diagnostics: SourceAdapterDiagnostic[];
+    shouldContinue: boolean;
+    rerouteSuggestion?: '--conversation' | '--soul';
+    normalizationRequired: boolean;
+    isNormalizedOutput: boolean;
+    normalizationContract?: 'v1';
+    normalizedSourceFamily?: NormalizedSourceFamily;
+    originalInputFamily?: NormalizationFamily;
+    normalizedFrom?: NormalizationFamily;
+    routeStatus: 'supported' | 'warning' | 'reroute' | 'normalize' | 'blocked';
+    recommendations: string[];
+}
+export interface PreflightWindow {
+    start: number;
+    end: number;
+    label: string;
+}
+export interface BatchPreflightSummary {
+    window: PreflightWindow;
+    sourcesScanned: number;
+    conversationSources: string[];
+    soulLikeSources: string[];
+    userProfileSources: string[];
+    agentPersonaSources: string[];
+    memoryIndexSources: string[];
+    contractMismatchFiles: string[];
+    rerouteToConversation: string[];
+    rerouteToSoul: string[];
+    normalizationRequired: string[];
+    shouldProceed: boolean;
+    sourceReports: SourcePreflightReport[];
+}
+export interface StrictFailure {
+    sourcePath: string;
+    reason: string;
+}
+export declare function classifyLane(source: SourceDefinition): OnboardingLane;
+export declare function inspectSources(sources: SourceDefinition[], window: PreflightWindow, seenHashesBySourceId?: Map<string, Set<string>>): BatchPreflightSummary;
+export declare function evaluateStrictPreflightFailures(sources: SourceDefinition[], preflight: BatchPreflightSummary): StrictFailure[];
+export declare function evaluateStrictPostIngestFailures(sources: SourceDefinition[], preflight: BatchPreflightSummary, sourceResults: Array<{
+    sourceId: string;
+    recordsIngested: number;
+    diagnostics: SourceAdapterDiagnostic[];
+}>): StrictFailure[];
+export declare function formatPreflightSummary(preflight: BatchPreflightSummary): string;
+//# sourceMappingURL=SourceOnboarding.d.ts.map