@matthewfl/pi-contemplator 0.0.1

package/src/session-ledger/recall.ts

@@ -0,0 +1,258 @@
+import {
+	isObservationsDroppedEntry,
+	isObservationsRecordedEntry,
+	isReflectionsRecordedEntry,
+	isReviewResultEntry,
+	type Entry,
+	type Observation,
+	type Reflection,
+	type ReviewResult,
+} from "./types.js";
+
+const SOURCE_TYPES = new Set(["message", "custom_message", "branch_summary"]);
+
+export type { Entry, Observation, Reflection };
+
+type ObservationLedgerLocation = {
+	entryId: string;
+	entryIndex: number;
+	recordIndex: number;
+};
+
+type ReflectionLedgerLocation = {
+	entryId: string;
+	entryIndex: number;
+	recordIndex: number;
+};
+
+export type RecalledObservation = {
+	observation: Observation;
+	observationEntryId: string;
+	observationRecordIndex: number;
+	status: "active" | "dropped";
+	sourceEntryIds: string[];
+	sourceEntries: Entry[];
+	missingSourceEntryIds: string[];
+	nonSourceEntryIds: string[];
+};
+
+export type RecalledReflection = {
+	reflection: Reflection;
+	reflectionEntryId: string;
+	reflectionRecordIndex: number;
+};
+
+export type RecalledReviewResult = {
+	review: ReviewResult;
+	reviewEntryId: string;
+};
+
+export type RecallResult =
+	| {
+			status: "not_found";
+			memoryId: string;
+			kind: undefined;
+			reflections: [];
+			reviews: [];
+			observations: [];
+			sourceEntries: [];
+			missingSourceEntryIds: [];
+			nonSourceEntryIds: [];
+			missingSupportingObservationIds: [];
+			collision: false;
+			partial: false;
+	  }
+	| {
+			status: "found";
+			memoryId: string;
+			kind: "observation" | "reflection" | "review" | "mixed";
+			reflections: RecalledReflection[];
+			reviews: RecalledReviewResult[];
+			observations: RecalledObservation[];
+			sourceEntries: Entry[];
+			missingSourceEntryIds: string[];
+			nonSourceEntryIds: string[];
+			missingSupportingObservationIds: string[];
+			collision: boolean;
+			partial: boolean;
+	  };
+
+type IndexedObservation = ObservationLedgerLocation & { observation: Observation };
+type IndexedReflection = ReflectionLedgerLocation & { reflection: Reflection };
+type IndexedReviewResult = { review: ReviewResult; entryId: string };
+
+function isSourceEntry(entry: Entry): boolean {
+	return SOURCE_TYPES.has(entry.type);
+}
+
+function uniqueById(entries: Entry[]): Entry[] {
+	const seen = new Set<string>();
+	const result: Entry[] = [];
+	for (const entry of entries) {
+		if (seen.has(entry.id)) continue;
+		seen.add(entry.id);
+		result.push(entry);
+	}
+	return result;
+}
+
+function uniqueStrings(values: string[]): string[] {
+	return Array.from(new Set(values));
+}
+
+function indexLedger(entries: Entry[]): {
+	observations: IndexedObservation[];
+	reflections: IndexedReflection[];
+	reviews: IndexedReviewResult[];
+	droppedIds: Set<string>;
+} {
+	const observations: IndexedObservation[] = [];
+	const reflections: IndexedReflection[] = [];
+	const reviews: IndexedReviewResult[] = [];
+	const droppedIds = new Set<string>();
+
+	for (let entryIndex = 0; entryIndex < entries.length; entryIndex++) {
+		const entry = entries[entryIndex];
+		if (isObservationsRecordedEntry(entry)) {
+			entry.data.observations.forEach((observation, recordIndex) => {
+				observations.push({ observation, entryId: entry.id, entryIndex, recordIndex });
+			});
+			continue;
+		}
+		if (isReflectionsRecordedEntry(entry)) {
+			entry.data.reflections.forEach((reflection, recordIndex) => {
+				reflections.push({ reflection, entryId: entry.id, entryIndex, recordIndex });
+			});
+			continue;
+		}
+		if (isObservationsDroppedEntry(entry)) {
+			entry.data.observationIds.forEach((id) => droppedIds.add(id));
+			continue;
+		}
+		if (isReviewResultEntry(entry)) reviews.push({ review: entry.data.result, entryId: entry.id });
+	}
+
+	return { observations, reflections, reviews, droppedIds };
+}
+
+function resolveObservationSources(entries: Entry[], observation: Observation, location: ObservationLedgerLocation): RecalledObservation {
+	const sourceEntryIds = uniqueStrings(observation.sourceEntryIds);
+	const byId = new Map(entries.map((entry) => [entry.id, entry]));
+	const sourceEntries: Entry[] = [];
+	const missingSourceEntryIds: string[] = [];
+	const nonSourceEntryIds: string[] = [];
+
+	for (const sourceEntryId of sourceEntryIds) {
+		const sourceEntry = byId.get(sourceEntryId);
+		if (!sourceEntry) {
+			missingSourceEntryIds.push(sourceEntryId);
+			continue;
+		}
+		if (!isSourceEntry(sourceEntry)) {
+			nonSourceEntryIds.push(sourceEntryId);
+			continue;
+		}
+		sourceEntries.push(sourceEntry);
+	}
+
+	return {
+		observation,
+		observationEntryId: location.entryId,
+		observationRecordIndex: location.recordIndex,
+		status: "active",
+		sourceEntryIds,
+		sourceEntries,
+		missingSourceEntryIds,
+		nonSourceEntryIds,
+	};
+}
+
+function notFound(memoryId: string): RecallResult {
+	return {
+		status: "not_found",
+		memoryId,
+		kind: undefined,
+		reflections: [],
+		reviews: [],
+		observations: [],
+		sourceEntries: [],
+		missingSourceEntryIds: [],
+		nonSourceEntryIds: [],
+		missingSupportingObservationIds: [],
+		collision: false,
+		partial: false,
+	};
+}
+
+export function recallMemorySources(entries: Entry[], memoryId: string): RecallResult {
+	const { observations: indexedObservations, reflections: indexedReflections, reviews: indexedReviews, droppedIds } = indexLedger(entries);
+	const directObservationMatches = indexedObservations.filter(({ observation }) => observation.id === memoryId);
+	const reflectionMatches = indexedReflections.filter(({ reflection }) => reflection.id === memoryId);
+	const reviewMatches = indexedReviews.filter(({ review }) => review.id === memoryId);
+
+	if (directObservationMatches.length === 0 && reflectionMatches.length === 0 && reviewMatches.length === 0) return notFound(memoryId);
+
+	const observationsById = new Map<string, IndexedObservation>();
+	for (const indexed of indexedObservations) {
+		if (!observationsById.has(indexed.observation.id)) observationsById.set(indexed.observation.id, indexed);
+	}
+
+	const recalledByKey = new Map<string, RecalledObservation>();
+	const missingSupportingObservationIds: string[] = [];
+
+	function addObservation(indexed: IndexedObservation): void {
+		const key = `${indexed.entryId}:${indexed.recordIndex}`;
+		if (recalledByKey.has(key)) return;
+		const recalled = resolveObservationSources(entries, indexed.observation, indexed);
+		recalled.status = droppedIds.has(indexed.observation.id) ? "dropped" : "active";
+		recalledByKey.set(key, recalled);
+	}
+
+	for (const match of directObservationMatches) addObservation(match);
+
+	for (const { reflection } of reflectionMatches) {
+		for (const observationId of uniqueStrings(reflection.supportingObservationIds)) {
+			const indexed = observationsById.get(observationId);
+			if (!indexed) {
+				missingSupportingObservationIds.push(observationId);
+				continue;
+			}
+			addObservation(indexed);
+		}
+	}
+
+	const recalledObservations = Array.from(recalledByKey.values());
+	const recalledReflections: RecalledReflection[] = reflectionMatches.map(({ reflection, entryId, recordIndex }) => ({
+		reflection,
+		reflectionEntryId: entryId,
+		reflectionRecordIndex: recordIndex,
+	}));
+	const recalledReviews: RecalledReviewResult[] = reviewMatches.map(({ review, entryId }) => ({ review, reviewEntryId: entryId }));
+	const sourceEntries = uniqueById(recalledObservations.flatMap((match) => match.sourceEntries));
+	const missingSourceEntryIds = uniqueStrings(recalledObservations.flatMap((match) => match.missingSourceEntryIds));
+	const nonSourceEntryIds = uniqueStrings(recalledObservations.flatMap((match) => match.nonSourceEntryIds));
+	const uniqueMissingSupportingObservationIds = uniqueStrings(missingSupportingObservationIds);
+	const matchCount = directObservationMatches.length + reflectionMatches.length + reviewMatches.length;
+	const kinds = [directObservationMatches.length > 0, reflectionMatches.length > 0, reviewMatches.length > 0].filter(Boolean).length;
+
+	return {
+		status: "found",
+		memoryId,
+		kind: kinds > 1
+			? "mixed"
+			: reviewMatches.length > 0
+				? "review"
+				: reflectionMatches.length > 0
+					? "reflection"
+					: "observation",
+		reflections: recalledReflections,
+		reviews: recalledReviews,
+		observations: recalledObservations,
+		sourceEntries,
+		missingSourceEntryIds,
+		nonSourceEntryIds,
+		missingSupportingObservationIds: uniqueMissingSupportingObservationIds,
+		collision: matchCount > 1,
+		partial: missingSourceEntryIds.length > 0 || nonSourceEntryIds.length > 0 || uniqueMissingSupportingObservationIds.length > 0,
+	};
+}

package/src/session-ledger/render-summary.ts

@@ -0,0 +1,31 @@
+import type { Observation, Reflection } from "./types.js";
+
+const CONTEXT_USAGE_INSTRUCTIONS = `These are condensed memories from earlier in this session.
+
+- Reflections: stable, long-lived facts about the user, project, decisions, and constraints. New reflection lines may include ids in brackets.
+- Observations: timestamped events from the conversation history, in chronological order. Observation lines include ids in brackets.
+
+Treat these as past records. When entries conflict, the most recent observation reflects the latest known state. Work that prior observations describe as completed should not be redone unless the user explicitly asks to revisit it.
+
+When exact source context is needed for precision or traceability, use the recall tool with the relevant observation or reflection id. This is especially useful when a reflection materially affects a decision or is too compressed to continue confidently. Do not use recall as broad search or inject raw source unless it is needed.`;
+
+export function observationToSummaryLine(observation: Observation): string {
+	return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`;
+}
+
+export function reflectionToSummaryLine(reflection: Reflection): string {
+	return `[${reflection.id}] ${reflection.content}`;
+}
+
+export function renderSummary(reflections: Reflection[], observations: Observation[]): string {
+	if (reflections.length === 0 && observations.length === 0) return "";
+
+	const parts: string[] = [CONTEXT_USAGE_INSTRUCTIONS];
+	if (reflections.length > 0) {
+		parts.push(`## Reflections\n${reflections.map(reflectionToSummaryLine).join("\n")}`);
+	}
+	if (observations.length > 0) {
+		parts.push(`## Observations\n${observations.map(observationToSummaryLine).join("\n")}`);
+	}
+	return parts.join("\n\n");
+}

package/src/session-ledger/search.ts

@@ -0,0 +1,184 @@
+import {
+	isObservationsDroppedEntry,
+	isObservationsRecordedEntry,
+	isReflectionsRecordedEntry,
+	isReviewResultEntry,
+	type Entry,
+	type Relevance,
+	type ReviewOutcome,
+	type ReviewScope,
+} from "./types.js";
+
+export type MemorySearchResult = {
+	kind: "observation" | "reflection" | "review";
+	id: string;
+	content: string;
+	relevance?: Relevance;
+	timestamp?: string;
+	status?: "active" | "dropped";
+	scope?: ReviewScope;
+	outcome?: ReviewOutcome;
+	title?: string;
+	score: number;
+};
+
+export type MemorySearch = {
+	query: string;
+	results: MemorySearchResult[];
+	observationsSearched: number;
+	reflectionsSearched: number;
+	reviewsSearched: number;
+};
+
+type SearchCandidate = Omit<MemorySearchResult, "score"> & {
+	sourceIndex: number;
+};
+
+const RELEVANCE_BOOST: Record<Relevance, number> = {
+	low: 1,
+	medium: 2,
+	high: 3,
+	critical: 4,
+};
+
+function normalizeSearchText(value: string): string {
+	return value.normalize("NFKC").toLocaleLowerCase();
+}
+
+function terms(value: string): string[] {
+	return Array.from(
+		new Set(normalizeSearchText(value).match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_./:-]*/gu) ?? []),
+	);
+}
+
+function relevanceScore(
+	content: string,
+	query: string,
+	queryTerms: string[],
+): number {
+	const normalizedContent = normalizeSearchText(content);
+	const phrase = normalizeSearchText(query.trim());
+	const termMatches = queryTerms.reduce(
+		(total, term) => total + (normalizedContent.includes(term) ? 1 : 0),
+		0,
+	);
+	const phraseBoost =
+		phrase.length > 0 && normalizedContent.includes(phrase) ? 5 : 0;
+	if (termMatches === 0 && phraseBoost === 0) return 0;
+	return termMatches * 10 + phraseBoost;
+}
+
+function candidates(entries: Entry[]): {
+	items: SearchCandidate[];
+	observations: number;
+	reflections: number;
+	reviews: number;
+} {
+	const dropped = new Set<string>();
+	for (const entry of entries) {
+		if (isObservationsDroppedEntry(entry)) {
+			for (const id of entry.data.observationIds) dropped.add(id);
+		}
+	}
+
+	const items: SearchCandidate[] = [];
+	let observations = 0;
+	let reflections = 0;
+	let reviews = 0;
+	for (let sourceIndex = 0; sourceIndex < entries.length; sourceIndex++) {
+		const entry = entries[sourceIndex];
+		if (isObservationsRecordedEntry(entry)) {
+			observations += entry.data.observations.length;
+			for (const observation of entry.data.observations) {
+				items.push({
+					kind: "observation",
+					id: observation.id,
+					content: observation.content,
+					relevance: observation.relevance,
+					timestamp: observation.timestamp,
+					status: dropped.has(observation.id) ? "dropped" : "active",
+					sourceIndex,
+				});
+			}
+		}
+		if (isReflectionsRecordedEntry(entry)) {
+			reflections += entry.data.reflections.length;
+			for (const reflection of entry.data.reflections) {
+				items.push({
+					kind: "reflection",
+					id: reflection.id,
+					content: reflection.content,
+					sourceIndex,
+				});
+			}
+			continue;
+		}
+		if (isReviewResultEntry(entry)) {
+			const review = entry.data.result;
+			reviews++;
+			const content = review.outcome === "proposal"
+				? [review.scope, "proposal", review.title, review.summary, review.evidence, review.conceptualDesign].join("\n")
+				: [review.scope, "review concluded with no proposal", review.reason, review.evidenceReviewed, review.reconsiderIf ?? ""].join("\n");
+			items.push({
+				kind: "review",
+				id: review.id,
+				content,
+				scope: review.scope,
+				outcome: review.outcome,
+				title: review.outcome === "proposal" ? review.title : undefined,
+				sourceIndex,
+			});
+		}
+	}
+	return { items, observations, reflections, reviews };
+}
+
+export function searchMemories(
+	entries: Entry[],
+	query: string,
+	limit = 8,
+): MemorySearch {
+	const normalizedQuery = query.trim();
+	const queryTerms = terms(normalizedQuery);
+	const searched = candidates(entries);
+	const results = searched.items
+		.map((candidate): MemorySearchResult | undefined => {
+			const lexical = relevanceScore(
+				candidate.content,
+				normalizedQuery,
+				queryTerms,
+			);
+			if (lexical === 0) return undefined;
+			const relevanceBoost = candidate.relevance
+				? RELEVANCE_BOOST[candidate.relevance]
+				: 0;
+			const kindBoost = candidate.kind === "reflection" ? 3 : 0;
+			const recencyBoost = Math.min(
+				candidate.sourceIndex / Math.max(entries.length, 1),
+				1,
+			);
+			return {
+				kind: candidate.kind,
+				id: candidate.id,
+				content: candidate.content,
+				relevance: candidate.relevance,
+				timestamp: candidate.timestamp,
+				status: candidate.status,
+				scope: candidate.scope,
+				outcome: candidate.outcome,
+				title: candidate.title,
+				score: lexical + relevanceBoost + kindBoost + recencyBoost,
+			} satisfies MemorySearchResult;
+		})
+		.filter((result): result is MemorySearchResult => result !== undefined)
+		.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id))
+		.slice(0, Math.max(1, Math.min(20, Math.floor(limit))));
+
+	return {
+		query: normalizedQuery,
+		results,
+		observationsSearched: searched.observations,
+		reflectionsSearched: searched.reflections,
+		reviewsSearched: searched.reviews,
+	};
+}