@prometheus-ai/memory 0.5.0

package/src/core/chat-normalize.ts

@@ -0,0 +1,160 @@
+type ExtractionRate = {
+	total: number;
+	survived: number;
+	dropped: number;
+	rate: number;
+	dropped_samples: string[];
+};
+
+const CONTRACTIONS: readonly [RegExp, string][] = [
+	[/\bu\b/g, "you"],
+	[/\bur\b/g, "your"],
+	[/\bu're\b/g, "you are"],
+	[/\br\b/g, "are"],
+	[/\by\b/g, "why"],
+	[/\bb4\b/g, "before"],
+	[/\bbc\b/g, "because"],
+	[/\bcuz\b/g, "because"],
+	[/\bgonna\b/g, "going to"],
+	[/\bwanna\b/g, "want to"],
+	[/\bgotta\b/g, "got to"],
+	[/\bkinda\b/g, "kind of"],
+	[/\bsorta\b/g, "sort of"],
+	[/\bdunno\b/g, "don't know"],
+	[/\blemme\b/g, "let me"],
+	[/\bgimme\b/g, "give me"],
+	[/\boutta\b/g, "out of"],
+	[/\bhafta\b/g, "have to"],
+	[/\bshoulda\b/g, "should have"],
+	[/\bwoulda\b/g, "would have"],
+	[/\bcoulda\b/g, "could have"],
+];
+
+const FILLER_WORDS: Readonly<Record<string, true>> = {
+	afaik: true,
+	brb: true,
+	fr: true,
+	fwiw: true,
+	idc: true,
+	idk: true,
+	iirc: true,
+	ikr: true,
+	imho: true,
+	imo: true,
+	irl: true,
+	istg: true,
+	lmao: true,
+	lmaoo: true,
+	lmfao: true,
+	lol: true,
+	ngl: true,
+	nvm: true,
+	omg: true,
+	omgg: true,
+	omggg: true,
+	rofl: true,
+	smh: true,
+	tbh: true,
+	tldr: true,
+	w: true,
+	wdym: true,
+	wtf: true,
+};
+
+const FRAGMENT_STARTERS: Readonly<Record<string, true>> = {
+	building: true,
+	checking: true,
+	coming: true,
+	deploying: true,
+	feeling: true,
+	fixing: true,
+	going: true,
+	hoping: true,
+	looking: true,
+	planning: true,
+	running: true,
+	testing: true,
+	thinking: true,
+	trying: true,
+	wondering: true,
+	working: true,
+};
+
+const EDGE_PUNCTUATION_RE = /^[.,!?;:'"]+|[.,!?;:'"]+$/g;
+const REPEATED_CHARS_RE = /(.)\1{2,}/g;
+function replaceNonAsciiRuns(value: string): string {
+	let normalized = "";
+	let inNonAsciiRun = false;
+	for (let index = 0; index < value.length; index++) {
+		const char = value[index];
+		if (char === undefined) continue;
+		if (char.charCodeAt(0) > 0x7f) {
+			if (!inNonAsciiRun) normalized += " ";
+			inNonAsciiRun = true;
+		} else {
+			normalized += char;
+			inNonAsciiRun = false;
+		}
+	}
+	return normalized;
+}
+
+export function normalizeChat(text: string, options: { add_implicit_subjects?: boolean } = {}): string | null {
+	const addImplicitSubjects = options.add_implicit_subjects ?? true;
+	if (text.trim().length === 0) return null;
+
+	let normalized = text.toLowerCase().trim();
+	for (const [pattern, replacement] of CONTRACTIONS) {
+		normalized = normalized.replace(pattern, replacement);
+	}
+
+	const meaningful = normalized
+		.split(/\s+/)
+		.filter(word => FILLER_WORDS[word.replace(EDGE_PUNCTUATION_RE, "")] !== true);
+	if (meaningful.length === 0) return null;
+
+	normalized = meaningful.join(" ");
+	normalized = normalized.replace(REPEATED_CHARS_RE, "$1");
+	normalized = replaceNonAsciiRuns(normalized);
+	normalized = normalized.split(/\s+/).filter(Boolean).join(" ");
+
+	const words = normalized.length === 0 ? [] : normalized.split(" ");
+	const wordCount = words.length;
+	if (wordCount < 2) {
+		if (wordCount === 1 && (words[0]?.length ?? 0) > 5) return normalized;
+		return null;
+	}
+
+	if (addImplicitSubjects && wordCount === 2) {
+		const firstWord = words[0] ?? "";
+		if (FRAGMENT_STARTERS[firstWord] === true) normalized = `i am ${normalized}`;
+	}
+
+	return normalized;
+}
+export function normalizeBatch(messages: string[]): (string | null)[] {
+	return messages.map(message => normalizeChat(message));
+}
+export function extractionRate(messages: string[]): ExtractionRate {
+	const normalized = normalizeBatch(messages);
+	let survived = 0;
+	const droppedSamples: string[] = [];
+
+	for (let i = 0; i < messages.length; i += 1) {
+		if (normalized[i] !== null) {
+			survived += 1;
+		} else if (droppedSamples.length < 5) {
+			const message = messages[i];
+			if (message !== undefined) droppedSamples.push(message);
+		}
+	}
+
+	const dropped = messages.length - survived;
+	return {
+		total: messages.length,
+		survived,
+		dropped,
+		rate: messages.length === 0 ? 0.0 : Math.round((survived / messages.length) * 1000) / 1000,
+		dropped_samples: droppedSamples,
+	};
+}

package/src/core/content-sanitizer.ts

@@ -0,0 +1,136 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const SIZE_HARD_CAP = 1_000_000;
+export const SIZE_BASE64_CHECK = 100_000;
+export const ENTROPY_THRESHOLD = 5.0;
+
+const DATA_URI_RE = /^data:(?<mime>[^;]+)?(?:;base64)?,(?<payload>.*)/i;
+const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+
+export interface BlobMetadata {
+	blob_ref?: string;
+	original_size?: number;
+	mime?: string;
+	extraction_reason?: "data_uri" | "size_cap" | "high_entropy";
+	entropy?: number;
+}
+
+export function blobRoot(env: NodeJS.ProcessEnv = process.env): string {
+	return env.PROMETHEUS_MEMORY_BLOB_DIR && env.PROMETHEUS_MEMORY_BLOB_DIR.length > 0
+		? env.PROMETHEUS_MEMORY_BLOB_DIR
+		: join(homedir(), ".prometheus", "memory", "blobs");
+}
+
+export function computeSha256(data: Uint8Array | string): string {
+	return createHash("sha256").update(data).digest("hex");
+}
+
+export function isDataUri(content: string): boolean {
+	return content.startsWith("data:");
+}
+
+export function parseDataUri(content: string): [mimeType: string, raw: Buffer] | null {
+	const match = DATA_URI_RE.exec(content);
+	if (match?.groups === undefined) return null;
+
+	const mimeType = match.groups.mime ?? "application/octet-stream";
+	const payload = match.groups.payload ?? "";
+	if (!isValidBase64(payload)) return null;
+
+	return [mimeType, Buffer.from(payload, "base64")];
+}
+
+export function shannonEntropy(text: string): number {
+	if (text.length === 0) return 0.0;
+
+	const counts = new Map<string, number>();
+	for (const char of text) counts.set(char, (counts.get(char) ?? 0) + 1);
+
+	let entropy = 0.0;
+	for (const count of counts.values()) {
+		const p = count / text.length;
+		entropy -= p * Math.log2(p);
+	}
+	return entropy;
+}
+
+export function looksLikeBase64Blob(content: string): boolean {
+	if (content.length < SIZE_BASE64_CHECK) return false;
+	return shannonEntropy(content) > ENTROPY_THRESHOLD;
+}
+
+export function storeBlob(rawBytes: Uint8Array): string {
+	const sha256 = computeSha256(rawBytes);
+	const blobDir = join(blobRoot(), sha256.slice(0, 2), sha256.slice(0, 4));
+	mkdirSync(blobDir, { recursive: true });
+	const blobPath = join(blobDir, sha256);
+	if (!existsSync(blobPath)) writeFileSync(blobPath, rawBytes);
+	return sha256;
+}
+
+export function sanitizeContent(content: string): [sanitizedContent: string, blobMetadata: BlobMetadata] {
+	const originalSize = Buffer.byteLength(content, "utf8");
+
+	if (isDataUri(content)) {
+		const parsed = parseDataUri(content);
+		if (parsed !== null) {
+			const [mimeType, rawBytes] = parsed;
+			const sha256 = storeBlob(rawBytes);
+			const blobRef = `blob://sha256/${sha256}`;
+			return [
+				`[Binary content extracted: ${mimeType}, ${rawBytes.length.toLocaleString("en-US")} bytes → ${blobRef}]`,
+				{
+					blob_ref: blobRef,
+					original_size: rawBytes.length,
+					mime: mimeType,
+					extraction_reason: "data_uri",
+				},
+			];
+		}
+	}
+
+	if (originalSize > SIZE_HARD_CAP) {
+		const rawBytes = Buffer.from(content, "utf8");
+		const sha256 = storeBlob(rawBytes);
+		const blobRef = `blob://sha256/${sha256}`;
+		return [
+			`[Large content extracted: ${originalSize.toLocaleString("en-US")} bytes → ${blobRef}]`,
+			{
+				blob_ref: blobRef,
+				original_size: originalSize,
+				extraction_reason: "size_cap",
+			},
+		];
+	}
+
+	if (originalSize > SIZE_BASE64_CHECK && looksLikeBase64Blob(content)) {
+		const rawBytes = Buffer.from(content, "utf8");
+		const sha256 = storeBlob(rawBytes);
+		const entropy = Math.round(shannonEntropy(content) * 100) / 100;
+		const blobRef = `blob://sha256/${sha256}`;
+		return [
+			`[Encoded content extracted: ${originalSize.toLocaleString("en-US")} bytes, entropy ${entropy.toFixed(1)} bits/char → ${blobRef}]`,
+			{
+				blob_ref: blobRef,
+				original_size: originalSize,
+				entropy,
+				extraction_reason: "high_entropy",
+			},
+		];
+	}
+
+	return [content, {}];
+}
+function isValidBase64(payload: string): boolean {
+	if (payload.length % 4 !== 0) return false;
+	if (!BASE64_RE.test(payload)) return false;
+	try {
+		Buffer.from(payload, "base64");
+		return true;
+	} catch {
+		return false;
+	}
+}

package/src/core/cost-log.ts

@@ -0,0 +1,103 @@
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+
+export const DEFAULT_LOG_DIR = join(homedir(), ".prometheus", "memory", "data");
+export const DEFAULT_LOG_DB = join(DEFAULT_LOG_DIR, "cost_log.db");
+
+export interface CostStats {
+	total_calls: number;
+	total_memories_injected: number;
+	total_tokens: number;
+	total_estimated_cost_usd: number;
+}
+
+type AggregateRow = {
+	calls: number | null;
+	total_memories: number | null;
+	total_tokens: number | null;
+	total_cost: number | null;
+};
+
+export function getConn(dbPath?: string): Database {
+	const path = dbPath ?? DEFAULT_LOG_DB;
+	mkdirSync(dirname(path), { recursive: true });
+	return new Database(path, { create: true, readwrite: true, strict: true });
+}
+
+export function initCostLog(dbPath?: string): void {
+	const conn = getConn(dbPath);
+	try {
+		conn.run(`
+			CREATE TABLE IF NOT EXISTS cost_entries (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				session_id TEXT,
+				memory_count INTEGER,
+				token_count INTEGER,
+				estimated_cost_usd REAL,
+				model TEXT DEFAULT 'default',
+				timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+			)
+		`);
+	} finally {
+		conn.close();
+	}
+}
+export function logCost(
+	sessionId: string,
+	memoryCount: number,
+	tokenCount: number,
+	estimatedCostUsd: number,
+	model = "default",
+	dbPath?: string,
+): void {
+	initCostLog(dbPath);
+	const conn = getConn(dbPath);
+	try {
+		conn
+			.query(`
+				INSERT INTO cost_entries (session_id, memory_count, token_count, estimated_cost_usd, model, timestamp)
+				VALUES (?, ?, ?, ?, ?, ?)
+			`)
+			.run(sessionId, memoryCount, tokenCount, estimatedCostUsd, model, localIsoTimestamp(new Date()));
+	} finally {
+		conn.close();
+	}
+}
+export function getCostStats(sessionId?: string, dbPath?: string): CostStats {
+	initCostLog(dbPath);
+	const conn = getConn(dbPath);
+	try {
+		const row = (
+			sessionId
+				? conn
+						.query(`
+						SELECT COUNT(*) as calls, SUM(memory_count) as total_memories,
+							SUM(token_count) as total_tokens, SUM(estimated_cost_usd) as total_cost
+						FROM cost_entries WHERE session_id = ?
+					`)
+						.get(sessionId)
+				: conn
+						.query(`
+						SELECT COUNT(*) as calls, SUM(memory_count) as total_memories,
+							SUM(token_count) as total_tokens, SUM(estimated_cost_usd) as total_cost
+						FROM cost_entries
+					`)
+						.get()
+		) as AggregateRow | null;
+
+		return {
+			total_calls: row?.calls ?? 0,
+			total_memories_injected: row?.total_memories ?? 0,
+			total_tokens: row?.total_tokens ?? 0,
+			total_estimated_cost_usd: Math.round((row?.total_cost ?? 0) * 1_000_000) / 1_000_000,
+		};
+	} finally {
+		conn.close();
+	}
+}
+function localIsoTimestamp(date: Date): string {
+	const offsetMs = date.getTimezoneOffset() * 60_000;
+	return new Date(date.getTime() - offsetMs).toISOString().replace("Z", "");
+}