@xopcai/xopc 0.0.180 → 0.0.187

package/dist/src/agent/memory/dreaming/light-sweep.js

@@ -1,233 +1,115 @@
-import { DREAMING_DIR_RELATIVE } from "./constants.js";
-import { buildEntryKey, normalizeMemoryPath } from "./utils.js";
-import { bumpEntryPhaseSignal, loadDreamingStore, saveDreamingStore, withDreamingStoreLock } from "./short-term-store.js";
-import "./last-run.js";
-import path from "node:path";
-import fs from "node:fs/promises";
+import { recordDreamingDecision } from "../../../storage/sqlite/dreaming-repository.js";
+import { appendMemorySignal, appendMemoryTraceEvent, listMemoryRecords, listMemorySignals } from "../../../storage/sqlite/memory-records-repository.js";
+import "../../../storage/sqlite/index.js";
 //#region src/agent/memory/dreaming/light-sweep.ts
 function resolveConfig(overrides) {
 	return {
 		enabled: overrides?.enabled === true,
-		cron: typeof overrides?.cron === "string" ? overrides.cron : "0 */6 * * *",
 		lookbackDays: Math.max(1, Math.floor(Number(overrides?.lookbackDays) || 2)),
-		limit: Math.max(0, Math.floor(Number(overrides?.limit) || 100)),
-		dedupeSimilarity: Math.max(0, Math.min(1, Number(overrides?.dedupeSimilarity) || .9))
+		limit: Math.max(0, Math.floor(Number(overrides?.limit) || 100))
 	};
 }
+/** Stage recent structured records as consolidation observations. */
 async function runLightSweep(params) {
 	const cfg = resolveConfig(params.config);
 	const now = params.now ?? /* @__PURE__ */ new Date();
-	const startedAt = now.toISOString();
-	const runId = `light:${startedAt}:${process.pid}`;
-	const startMs = Date.now();
+	const started = Date.now();
 	if (!cfg.enabled) {
-		const reason = "light sweep disabled";
-		await writeLastRun(params.dreamingRoot, {
-			runId,
-			startedAt,
-			cfg,
-			ok: true,
-			reason,
-			startMs,
-			light: {
-				scannedEntries: 0,
-				newSignals: 0,
-				deduped: 0
-			}
-		});
+		trace(params.workspaceDir, "light sweep disabled", 0, [], started);
 		return {
 			ok: true,
-			reason,
+			reason: "light sweep disabled",
 			scannedEntries: 0,
 			newSignals: 0,
 			deduped: 0
 		};
 	}
 	try {
-		const result = await withDreamingStoreLock(params.dreamingRoot, async () => {
-			const { store } = await loadDreamingStore({ dreamingRoot: params.dreamingRoot });
-			const candidates = await scanRecentMemoryLines(params.workspaceDir, cfg.lookbackDays, now);
-			let scannedEntries = 0;
-			let newSignals = 0;
-			let deduped = 0;
-			for (const candidate of candidates.slice(0, cfg.limit)) {
-				scannedEntries += 1;
-				const key = buildEntryKey(candidate);
-				const existing = store.entries[key];
-				if (existing) {
-					if (observedAtOrZero(existing) >= candidate.observedAtMs) {
-						deduped += 1;
-						continue;
-					}
-					existing.snippet = candidate.snippet;
-					existing.lastObservedAt = candidate.observedAt;
-					bumpEntryPhaseSignal(existing, "sourceCount");
-					bumpEntryPhaseSignal(existing, "lightHits");
-					continue;
-				}
-				const duplicate = Object.values(store.entries).find((entry) => similarity(entry.snippet, candidate.snippet) >= cfg.dedupeSimilarity);
-				if (duplicate) {
-					if (observedAtOrZero(duplicate) < candidate.observedAtMs) {
-						duplicate.lastObservedAt = candidate.observedAt;
-						bumpEntryPhaseSignal(duplicate, "sourceCount");
-						bumpEntryPhaseSignal(duplicate, "lightHits");
-					}
-					deduped += 1;
-					continue;
-				}
-				store.entries[key] = newLightEntry(candidate);
-				newSignals += 1;
-			}
-			if (scannedEntries > 0) {
-				store.updatedAt = startedAt;
-				await saveDreamingStore({
-					dreamingRoot: params.dreamingRoot,
-					store
+		const cutoff = now.getTime() - cfg.lookbackDays * 864e5;
+		const records = listMemoryRecords({
+			providerId: "local",
+			workspaceId: params.workspaceDir,
+			limit: 500
+		}).filter((record) => Date.parse(record.updatedAt) >= cutoff).slice(0, cfg.limit);
+		const existing = listMemorySignals({
+			workspaceId: params.workspaceDir,
+			limit: 500
+		});
+		const staged = new Set(existing.filter((signal) => signal.source === "dreaming" && signal.metadata.phase === "light" && Date.parse(signal.createdAt) >= cutoff).map((signal) => signal.recordId).filter((id) => Boolean(id)));
+		let newSignals = 0;
+		for (const record of records) {
+			if (staged.has(record.id)) {
+				recordDreamingDecision({
+					runId: params.runId,
+					recordId: record.id,
+					action: "skip",
+					reasonCode: "already_staged"
 				});
+				continue;
 			}
-			return {
-				scannedEntries,
-				newSignals,
-				deduped
-			};
-		});
-		await writeLastRun(params.dreamingRoot, {
-			runId,
-			startedAt,
-			cfg,
-			ok: true,
-			reason: "light sweep complete",
-			startMs,
-			light: result
-		});
+			appendMemorySignal({
+				signal: {
+					source: "dreaming",
+					recordId: record.id,
+					score: record.importance,
+					content: record.content,
+					metadata: {
+						phase: "light",
+						recordStatus: record.status ?? "active"
+					}
+				},
+				providerId: "local",
+				sourceAgentId: record.provenance.sourceAgentId,
+				workspaceId: params.workspaceDir
+			});
+			newSignals += 1;
+			recordDreamingDecision({
+				runId: params.runId,
+				recordId: record.id,
+				action: "observe",
+				reasonCode: "recent_record_staged",
+				score: record.importance
+			});
+		}
+		const deduped = records.length - newSignals;
+		trace(params.workspaceDir, "light sweep complete", newSignals, records.map((record) => record.id), started);
 		return {
 			ok: true,
 			reason: "light sweep complete",
-			...result
+			scannedEntries: records.length,
+			newSignals,
+			deduped
 		};
-	} catch (error) {
-		const errorMessage = error instanceof Error ? error.message : String(error);
-		await writeLastRun(params.dreamingRoot, {
-			runId,
-			startedAt,
-			cfg,
-			ok: false,
-			reason: `light error: ${errorMessage}`,
-			startMs,
-			light: {
-				scannedEntries: 0,
-				newSignals: 0,
-				deduped: 0
-			},
-			errorMessage
-		}).catch(() => void 0);
+	} catch (err) {
+		const reason = err instanceof Error ? err.message : String(err);
+		appendMemoryTraceEvent({
+			phase: "dreaming_light",
+			providerId: "local",
+			request: { workspaceId: params.workspaceDir },
+			error: reason,
+			durationMs: Date.now() - started
+		});
 		return {
 			ok: false,
-			reason: errorMessage,
+			reason,
 			scannedEntries: 0,
 			newSignals: 0,
 			deduped: 0
 		};
 	}
 }
-async function scanRecentMemoryLines(workspaceDir, lookbackDays, now) {
-	const root = path.join(workspaceDir, "memory");
-	const cutoffMs = now.getTime() - lookbackDays * 24 * 60 * 60 * 1e3;
-	const files = await listMarkdownFiles(root);
-	const out = [];
-	for (const filePath of files) {
-		const stat = await fs.stat(filePath).catch(() => null);
-		if (!stat || stat.mtimeMs < cutoffMs) continue;
-		const content = await fs.readFile(filePath, "utf-8").catch(() => "");
-		const relativePath = normalizeMemoryPath(path.relative(workspaceDir, filePath));
-		let inFence = false;
-		for (const [index, rawLine] of content.split(/\r?\n/).entries()) {
-			if (/^\s*```/.test(rawLine)) {
-				inFence = !inFence;
-				continue;
-			}
-			if (inFence) continue;
-			const snippet = rawLine.replace(/^\s*[-*]\s+/, "").trim();
-			if (snippet.length < 8 || snippet.startsWith("#") || snippet.startsWith("<!--")) continue;
-			out.push({
-				path: relativePath,
-				startLine: index + 1,
-				endLine: index + 1,
-				snippet: snippet.slice(0, 360),
-				observedAt: stat.mtime.toISOString(),
-				observedAtMs: stat.mtimeMs
-			});
-		}
-	}
-	return out.sort((left, right) => right.observedAtMs - left.observedAtMs || left.path.localeCompare(right.path) || left.startLine - right.startLine);
-}
-async function listMarkdownFiles(root) {
-	const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
-	const out = [];
-	for (const entry of entries) {
-		const fullPath = path.join(root, entry.name);
-		if (entry.isDirectory()) out.push(...await listMarkdownFiles(fullPath));
-		else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) out.push(fullPath);
-	}
-	return out;
-}
-function similarity(left, right) {
-	const tokens = (value) => new Set(value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []);
-	const a = tokens(left);
-	const b = tokens(right);
-	if (a.size === 0 || b.size === 0) return 0;
-	let intersection = 0;
-	for (const token of a) if (b.has(token)) intersection += 1;
-	return intersection / (a.size + b.size - intersection);
-}
-function observedAtOrZero(entry) {
-	const parsed = entry.lastObservedAt ? Date.parse(entry.lastObservedAt) : 0;
-	return Number.isFinite(parsed) ? parsed : 0;
-}
-function newLightEntry(candidate) {
-	return {
-		key: buildEntryKey(candidate),
-		path: candidate.path,
-		startLine: candidate.startLine,
-		endLine: candidate.endLine,
-		snippet: candidate.snippet,
-		recallCount: 0,
-		sourceCount: 1,
-		groundedCount: 0,
-		lightHits: 1,
-		remHits: 0,
-		phaseHitCount: 2,
-		totalSignalCount: 2,
-		totalScore: 0,
-		maxScore: 0,
-		queryHashes: [],
-		recallDays: [],
-		firstRecalledAt: candidate.observedAt,
-		lastRecalledAt: candidate.observedAt,
-		lastObservedAt: candidate.observedAt
-	};
-}
-async function writeLastRun(dreamingRoot, params) {
-	const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
-	const durationMs = Math.max(0, Date.now() - params.startMs);
-	const lastRun = {
-		version: 2,
-		phase: "light",
-		runId: params.runId,
-		startedAt: params.startedAt,
-		finishedAt,
-		durationMs,
-		ok: params.ok,
-		reason: params.reason,
-		config: params.cfg,
-		light: params.light,
-		...params.errorMessage ? { errorMessage: params.errorMessage } : {}
-	};
-	const lastRunPath = path.join(dreamingRoot, DREAMING_DIR_RELATIVE, "last-run-light.json");
-	await fs.mkdir(path.dirname(lastRunPath), { recursive: true });
-	const tmp = `${lastRunPath}.${process.pid}.${Date.now()}.tmp`;
-	await fs.writeFile(tmp, `${JSON.stringify(lastRun, null, 2)}\n`, "utf-8");
-	await fs.rename(tmp, lastRunPath);
+function trace(workspaceId, reason, resultCount, selectedRecordIds, started) {
+	appendMemoryTraceEvent({
+		phase: "dreaming_light",
+		providerId: "local",
+		request: {
+			workspaceId,
+			reason
+		},
+		resultCount,
+		selectedRecordIds,
+		durationMs: Date.now() - started
+	});
 }
 //#endregion
 export { runLightSweep };

package/dist/src/agent/memory/dreaming/preview.d.ts

@@ -1,22 +1,16 @@
 import type { DreamingDeepConfig } from './config.js';
 type PreviewItem = {
-    key: string;
-    hash: string;
-    snippet: string;
-    path: string;
-    startLine: number;
-    endLine: number;
+    recordId: string;
+    content: string;
     score: number;
     avgScore: number;
     recallCount: number;
+    uniqueQueries: number;
     recencyDecay: number;
-    alreadyPromotedByKey: boolean;
-    alreadyPromotedByHash: boolean;
     skippedReason: string | null;
 };
 export declare function previewDreamingDeepPromotion(params: {
     workspaceDir: string;
-    dreamingRoot: string;
     config?: Partial<DreamingDeepConfig>;
     limit?: number;
     now?: Date;

package/dist/src/agent/memory/dreaming/preview.js

@@ -1,93 +1,50 @@
-import { MEMORY_MD_FILENAME } from "./constants.js";
-import { clamp01, compareCandidatesByScore, computeCandidateScore, extractPromotionMarkers, isContaminatedSnippet, isExpiredEntry, readFileLines, resolveDeepDefaults, sliceRange, snippetHash } from "./utils.js";
-import { loadDreamingStore } from "./short-term-store.js";
-import path from "node:path";
-import fs from "node:fs/promises";
+import { listMemoryRecords, listMemorySignals } from "../../../storage/sqlite/memory-records-repository.js";
+import "../../../storage/sqlite/index.js";
+import { resolveDeepDefaults } from "./utils.js";
 //#region src/agent/memory/dreaming/preview.ts
 async function previewDreamingDeepPromotion(params) {
 	const cfg = resolveDeepDefaults(params.config);
-	const memoryPath = path.join(params.dreamingRoot, MEMORY_MD_FILENAME);
+	const memoryPath = "sqlite://memory_records";
 	if (!cfg.enabled) return {
 		ok: true,
 		reason: "dreaming disabled",
 		items: [],
 		memoryPath
 	};
-	const { store } = await loadDreamingStore({ dreamingRoot: params.dreamingRoot });
-	const nowMs = (params.now ?? /* @__PURE__ */ new Date()).getTime();
-	const ranked = Object.values(store.entries ?? {}).filter((e) => {
-		if (!e || typeof e !== "object") return false;
-		if (e.promotedAt) return false;
-		if (!e.path || !e.path.startsWith("memory/")) return false;
-		if (e.recallCount < cfg.minRecallCount) return false;
-		if ((e.queryHashes?.length ?? 0) < cfg.minUniqueQueries) return false;
-		if (isExpiredEntry(e.lastRecalledAt, nowMs, cfg.maxAgeDays)) return false;
-		return clamp01(e.recallCount > 0 ? e.totalScore / e.recallCount : 0) >= cfg.minScore;
-	}).map((e) => {
-		const { avgScore, score, recencyDecay } = computeCandidateScore(e, nowMs, cfg.recencyHalfLifeDays);
-		return {
-			...e,
-			avgScore,
-			score,
-			recencyDecay
-		};
-	}).sort(compareCandidatesByScore);
-	const markers = extractPromotionMarkers(await fs.readFile(memoryPath, "utf-8").catch((err) => {
-		if (err?.code === "ENOENT") return "";
-		throw err;
-	}));
+	const now = params.now ?? /* @__PURE__ */ new Date();
+	const cutoff = now.getTime() - cfg.maxAgeDays * 864e5;
+	const signals = listMemorySignals({
+		workspaceId: params.workspaceDir,
+		limit: 500
+	});
 	const limit = Math.min(Math.max(params.limit ?? 20, 1), 50);
-	const out = [];
-	const scanCap = Math.min(ranked.length, Math.max(limit * 3, limit));
-	for (const candidate of ranked.slice(0, scanCap)) {
-		if (markers.keys.has(candidate.key)) {
-			out.push({
-				key: candidate.key,
-				hash: "",
-				snippet: "",
-				path: candidate.path,
-				startLine: candidate.startLine,
-				endLine: candidate.endLine,
-				score: candidate.score,
-				avgScore: candidate.avgScore,
-				recallCount: candidate.recallCount,
-				recencyDecay: candidate.recencyDecay,
-				alreadyPromotedByKey: true,
-				alreadyPromotedByHash: false,
-				skippedReason: "already promoted (key)"
-			});
-			continue;
-		}
-		const lines = await readFileLines(path.join(params.workspaceDir, candidate.path));
-		if (!lines) continue;
-		const startLine = Math.max(1, Math.floor(candidate.startLine));
-		const endLine = Math.max(startLine, Math.floor(candidate.endLine));
-		const snippet = sliceRange(lines, startLine, endLine);
-		if (!snippet) continue;
-		if (isContaminatedSnippet(snippet)) continue;
-		const hash = snippetHash(snippet);
-		const alreadyPromotedByHash = markers.hashes.has(hash);
-		out.push({
-			key: candidate.key,
-			hash,
-			snippet,
-			path: candidate.path,
-			startLine,
-			endLine,
-			score: candidate.score,
-			avgScore: candidate.avgScore,
-			recallCount: candidate.recallCount,
-			recencyDecay: candidate.recencyDecay,
-			alreadyPromotedByKey: false,
-			alreadyPromotedByHash,
-			skippedReason: alreadyPromotedByHash ? "already promoted (hash)" : null
-		});
-		if (out.filter((x) => !x.skippedReason).length >= limit) break;
-	}
 	return {
 		ok: true,
 		reason: "ok",
-		items: out,
+		items: listMemoryRecords({
+			providerId: "local",
+			workspaceId: params.workspaceDir,
+			status: "candidate",
+			limit: 500
+		}).map((record) => {
+			const recalls = signals.filter((signal) => signal.recordId === record.id && Date.parse(signal.createdAt) >= cutoff && (signal.source === "search_recall" || signal.source === "context_injection"));
+			const uniqueQueries = new Set(recalls.map((signal) => String(signal.metadata.query ?? "")).filter(Boolean));
+			const avgScore = recalls.length ? recalls.reduce((sum, signal) => sum + (signal.score ?? 0), 0) / recalls.length : 0;
+			const ageDays = Math.max(0, (now.getTime() - Date.parse(record.updatedAt)) / 864e5);
+			const recencyDecay = Math.pow(.5, ageDays / cfg.recencyHalfLifeDays);
+			const score = Math.max(0, Math.min(1, avgScore * recencyDecay));
+			const skippedReason = recalls.length < cfg.minRecallCount ? "insufficient recalls" : uniqueQueries.size < cfg.minUniqueQueries ? "insufficient query diversity" : score < cfg.minScore ? "score below threshold" : null;
+			return {
+				recordId: record.id,
+				content: record.content,
+				score,
+				avgScore,
+				recallCount: recalls.length,
+				uniqueQueries: uniqueQueries.size,
+				recencyDecay,
+				skippedReason
+			};
+		}).sort((left, right) => right.score - left.score).slice(0, limit),
 		memoryPath
 	};
 }

package/dist/src/agent/memory/dreaming/promotion-lifecycle.d.ts

@@ -1,19 +1,4 @@
 import type { MemoryRecord, MemorySensitivity } from '../types.js';
-export interface DreamingPromotionInput {
-    agentId: string;
-    workspaceId: string;
-    candidateKey: string;
-    content: string;
-    sourcePath: string;
-    lineStart: number;
-    lineEnd: number;
-    score: number;
-    recallCount: number;
-    observedAt: string;
-    sensitivity: MemorySensitivity;
-}
-/** Persist a promoted Dreaming insight as the authoritative structured record. */
-export declare function activateDreamingPromotion(input: DreamingPromotionInput): MemoryRecord;
 export interface RemInsightInput {
     agentId: string;
     workspaceId: string;
@@ -24,6 +9,7 @@ export interface RemInsightInput {
     observedAt: string;
     evidence: string[];
     sensitivity: MemorySensitivity;
+    status?: 'active' | 'candidate';
 }
 export declare function remPatternKey(memberKeys: string[]): string;
 /** Persist a REM cluster as a stable, recallable derived insight. */

package/dist/src/agent/memory/dreaming/promotion-lifecycle.js

@@ -2,42 +2,6 @@ import { upsertMemoryRecord } from "../../../storage/sqlite/memory-records-repos
 import "../../../storage/sqlite/index.js";
 import { createHash } from "node:crypto";
 //#region src/agent/memory/dreaming/promotion-lifecycle.ts
-/** Persist a promoted Dreaming insight as the authoritative structured record. */
-function activateDreamingPromotion(input) {
-	return upsertMemoryRecord({
-		id: `dreaming:${createHash("sha256").update(`${input.agentId}\0${input.candidateKey}`).digest("hex").slice(0, 32)}`,
-		providerId: "local",
-		kind: "project_context",
-		sourceAgentId: input.agentId,
-		workspaceId: input.workspaceId,
-		content: input.content,
-		source: {
-			provider: "dreaming",
-			path: input.sourcePath,
-			lineStart: input.lineStart,
-			lineEnd: input.lineEnd
-		},
-		confidence: input.score,
-		status: "active",
-		sensitivity: input.sensitivity,
-		canonicalKey: `dreaming:${input.candidateKey}`,
-		explicitness: "observed",
-		durability: "durable",
-		importance: input.score,
-		disclosurePolicy: "referenceable",
-		evidence: [{
-			relation: "derived_from",
-			sourceText: input.content,
-			observedAt: input.observedAt,
-			confidence: input.score
-		}],
-		tags: [
-			"dreaming",
-			"promoted",
-			`recalls:${input.recallCount}`
-		]
-	});
-}
 function remPatternKey(memberKeys) {
 	return createHash("sha256").update([...memberKeys].sort().join("\0")).digest("hex").slice(0, 24);
 }
@@ -52,12 +16,9 @@ function activateRemInsight(input) {
 		sourceAgentId: input.agentId,
 		workspaceId: input.workspaceId,
 		content,
-		source: {
-			provider: "dreaming",
-			path: "DREAMS.md"
-		},
+		source: { provider: "dreaming" },
 		confidence: input.strength,
-		status: "active",
+		status: input.status ?? "active",
 		sensitivity: input.sensitivity,
 		canonicalKey: `dreaming-rem:${patternKey}`,
 		explicitness: "inferred",
@@ -80,4 +41,4 @@ function activateRemInsight(input) {
 	});
 }
 //#endregion
-export { activateDreamingPromotion, activateRemInsight, remPatternKey };
+export { activateRemInsight, remPatternKey };

package/dist/src/agent/memory/dreaming/rem-patterns.d.ts

@@ -1,21 +1,14 @@
+import { type DreamingActiveMode } from '../../../storage/sqlite/index.js';
 import type { DreamingRemConfig } from './config.js';
-/**
- * REM phase: cross-session pattern discovery.
- *
- * Scans the short-term store for entries that share query hashes or
- * appear across multiple indexed memory sources, then clusters them to identify
- * recurring themes/patterns. Bumps `remHits` on touched entries and
- * optionally writes a pattern summary to DREAMS.md.
- *
- * Runs weekly; expensive but insightful.
- */
+type RemExecutionConfig = Pick<DreamingRemConfig, 'enabled' | 'lookbackDays' | 'limit' | 'minPatternStrength'>;
+/** Discover cross-session patterns from structured usage signals. */
 export declare function runRemPatterns(params: {
     agentId: string;
+    runId: string;
+    mode: DreamingActiveMode;
     workspaceDir: string;
-    dreamingRoot: string;
-    config?: Partial<DreamingRemConfig>;
+    config?: Partial<RemExecutionConfig>;
     sensitiveWritePolicy?: 'deny' | 'confirm' | 'allow';
-    promotionWritePolicy?: 'deny' | 'confirm' | 'allow';
     now?: Date;
 }): Promise<{
     ok: boolean;
@@ -23,3 +16,4 @@ export declare function runRemPatterns(params: {
     patternsDiscovered: number;
     entriesAnalyzed: number;
 }>;
+export {};