memory-braid 0.6.0 → 0.7.0

package/src/memory-model.ts

@@ -0,0 +1,336 @@
+import { normalizeWhitespace } from "./chunking.js";
+import type { ExtractedEntity } from "./entities.js";
+import type {
+  MemoryBraidResult,
+  MemoryKind,
+  MemoryLayer,
+  MemoryOwner,
+  TaxonomyBuckets,
+} from "./types.js";
+
+const TOPIC_STOPWORDS = new Set([
+  "about",
+  "after",
+  "agent",
+  "always",
+  "before",
+  "from",
+  "have",
+  "into",
+  "just",
+  "keep",
+  "like",
+  "memory",
+  "never",
+  "note",
+  "only",
+  "remember",
+  "that",
+  "their",
+  "them",
+  "they",
+  "this",
+  "turn",
+  "user",
+  "using",
+  "what",
+  "when",
+  "will",
+  "with",
+]);
+
+export function asRecord(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return {};
+  }
+  return value as Record<string, unknown>;
+}
+
+export function asString(value: unknown): string | undefined {
+  if (typeof value !== "string") {
+    return undefined;
+  }
+  const trimmed = value.trim();
+  return trimmed || undefined;
+}
+
+export function normalizeMemoryKind(raw: unknown): MemoryKind | undefined {
+  return raw === "fact" ||
+    raw === "preference" ||
+    raw === "decision" ||
+    raw === "task" ||
+    raw === "heuristic" ||
+    raw === "lesson" ||
+    raw === "strategy" ||
+    raw === "other"
+    ? raw
+    : undefined;
+}
+
+export function normalizeMemoryOwner(raw: unknown): MemoryOwner | undefined {
+  return raw === "user" || raw === "agent" ? raw : undefined;
+}
+
+export function normalizeMemoryLayer(raw: unknown): MemoryLayer | undefined {
+  return raw === "episodic" || raw === "semantic" || raw === "procedural" ? raw : undefined;
+}
+
+export function emptyTaxonomy(): TaxonomyBuckets {
+  return {
+    people: [],
+    places: [],
+    organizations: [],
+    projects: [],
+    tools: [],
+    topics: [],
+  };
+}
+
+function slugify(value: string): string {
+  return value
+    .normalize("NFKD")
+    .replace(/\p{M}+/gu, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+}
+
+function pushBucket(target: string[], value: string): void {
+  const cleaned = normalizeWhitespace(value);
+  if (!cleaned) {
+    return;
+  }
+  const existing = new Set(target.map((entry) => slugify(entry)));
+  const key = slugify(cleaned);
+  if (!key || existing.has(key)) {
+    return;
+  }
+  target.push(cleaned);
+}
+
+function firstWords(text: string, count: number): string {
+  return text
+    .split(/\s+/)
+    .filter(Boolean)
+    .slice(0, count)
+    .join(" ");
+}
+
+function normalizeEntityRows(raw: unknown): ExtractedEntity[] {
+  if (!Array.isArray(raw)) {
+    return [];
+  }
+  const out: ExtractedEntity[] = [];
+  for (const value of raw) {
+    const row = asRecord(value);
+    const text = asString(row.text);
+    const type = asString(row.type);
+    const canonicalUri = asString(row.canonicalUri);
+    if (!text || !type || !canonicalUri) {
+      continue;
+    }
+    if (type !== "person" && type !== "organization" && type !== "location" && type !== "misc") {
+      continue;
+    }
+    out.push({
+      text,
+      type,
+      score:
+        typeof row.score === "number" && Number.isFinite(row.score)
+          ? Math.max(0, Math.min(1, row.score))
+          : 0,
+      canonicalUri,
+    });
+  }
+  return out;
+}
+
+function deriveToolCandidates(text: string): string[] {
+  const matches = [
+    ...text.matchAll(/`([^`]{2,40})`/g),
+    ...text.matchAll(/\b(?:use|using|with|tool|library|framework)\s+([A-Z][A-Za-z0-9._-]{1,40})/g),
+  ];
+  return matches.map((match) => normalizeWhitespace(match[1] ?? "")).filter(Boolean);
+}
+
+function deriveProjectCandidates(text: string): string[] {
+  const matches = [
+    ...text.matchAll(/\bproject\s+([A-Z][A-Za-z0-9._-]{1,50})/gi),
+    ...text.matchAll(/\b(?:repo|workspace)\s+([A-Z][A-Za-z0-9._-]{1,50})/gi),
+  ];
+  return matches.map((match) => normalizeWhitespace(match[1] ?? "")).filter(Boolean);
+}
+
+function deriveTopicCandidates(text: string): string[] {
+  const tokens = text.match(/[\p{L}\p{N}][\p{L}\p{N}-]{2,}/gu) ?? [];
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const token of tokens) {
+    const normalized = slugify(token);
+    if (!normalized || TOPIC_STOPWORDS.has(normalized) || seen.has(normalized)) {
+      continue;
+    }
+    seen.add(normalized);
+    out.push(token);
+    if (out.length >= 3) {
+      break;
+    }
+  }
+  return out;
+}
+
+export function buildTaxonomy(params: {
+  text: string;
+  entities?: unknown;
+  existingTaxonomy?: unknown;
+}): TaxonomyBuckets {
+  const taxonomy = normalizeTaxonomy(params.existingTaxonomy);
+  for (const entity of normalizeEntityRows(params.entities)) {
+    if (entity.type === "person") {
+      pushBucket(taxonomy.people, entity.text);
+    } else if (entity.type === "organization") {
+      pushBucket(taxonomy.organizations, entity.text);
+    } else if (entity.type === "location") {
+      pushBucket(taxonomy.places, entity.text);
+    }
+  }
+
+  for (const candidate of deriveToolCandidates(params.text)) {
+    pushBucket(taxonomy.tools, candidate);
+  }
+  for (const candidate of deriveProjectCandidates(params.text)) {
+    pushBucket(taxonomy.projects, candidate);
+  }
+  for (const candidate of deriveTopicCandidates(params.text)) {
+    pushBucket(taxonomy.topics, candidate);
+  }
+
+  return taxonomy;
+}
+
+export function normalizeTaxonomy(raw: unknown): TaxonomyBuckets {
+  const source = asRecord(raw);
+  const out = emptyTaxonomy();
+  const keys = Object.keys(out) as Array<keyof TaxonomyBuckets>;
+  for (const key of keys) {
+    const values = Array.isArray(source[key]) ? source[key] : [];
+    for (const value of values) {
+      if (typeof value === "string") {
+        pushBucket(out[key], value);
+      }
+    }
+  }
+  return out;
+}
+
+export function taxonomyTerms(taxonomy: TaxonomyBuckets): string[] {
+  return [
+    ...taxonomy.people,
+    ...taxonomy.places,
+    ...taxonomy.organizations,
+    ...taxonomy.projects,
+    ...taxonomy.tools,
+    ...taxonomy.topics,
+  ];
+}
+
+export function taxonomyOverlap(left: TaxonomyBuckets, right: TaxonomyBuckets): number {
+  const leftTerms = new Set(taxonomyTerms(left).map(slugify));
+  const rightTerms = new Set(taxonomyTerms(right).map(slugify));
+  if (leftTerms.size === 0 || rightTerms.size === 0) {
+    return 0;
+  }
+  let shared = 0;
+  for (const term of leftTerms) {
+    if (rightTerms.has(term)) {
+      shared += 1;
+    }
+  }
+  return shared / Math.max(leftTerms.size, rightTerms.size);
+}
+
+export function primaryTaxonomyAnchor(taxonomy: TaxonomyBuckets): string | undefined {
+  return (
+    taxonomy.people[0] ??
+    taxonomy.organizations[0] ??
+    taxonomy.projects[0] ??
+    taxonomy.tools[0] ??
+    taxonomy.topics[0] ??
+    taxonomy.places[0]
+  );
+}
+
+export function formatTaxonomySummary(taxonomy: TaxonomyBuckets): string {
+  const lines: string[] = [];
+  const ordered: Array<keyof TaxonomyBuckets> = [
+    "people",
+    "places",
+    "organizations",
+    "projects",
+    "tools",
+    "topics",
+  ];
+  for (const key of ordered) {
+    if (taxonomy[key].length > 0) {
+      lines.push(`${key}=${taxonomy[key].join(", ")}`);
+    }
+  }
+  return lines.join(" | ");
+}
+
+export function inferMemoryLayer(result: MemoryBraidResult): MemoryLayer {
+  const metadata = asRecord(result.metadata);
+  const explicit = normalizeMemoryLayer(metadata.memoryLayer);
+  if (explicit) {
+    return explicit;
+  }
+  const sourceType = asString(metadata.sourceType);
+  if (sourceType === "capture") {
+    return "episodic";
+  }
+  if (sourceType === "agent_learning") {
+    return "procedural";
+  }
+  if (sourceType === "compendium") {
+    return "semantic";
+  }
+  const owner = normalizeMemoryOwner(metadata.memoryOwner);
+  if (owner === "agent") {
+    return "procedural";
+  }
+  return "episodic";
+}
+
+export function summarizeSnippet(text: string, maxChars = 140): string {
+  const normalized = normalizeWhitespace(text);
+  if (normalized.length <= maxChars) {
+    return normalized;
+  }
+  return `${normalized.slice(0, maxChars - 1).trimEnd()}…`;
+}
+
+export function stripCapturePreamble(text: string): string {
+  const normalized = normalizeWhitespace(text);
+  return normalized.replace(/^(?:remember that|note that|we discussed that)\s+/i, "");
+}
+
+export function summarizeClusterText(texts: string[], kind?: MemoryKind): string {
+  const latest = stripCapturePreamble(texts[texts.length - 1] ?? "");
+  const base = latest || stripCapturePreamble(texts[0] ?? "");
+  if (!base) {
+    return "";
+  }
+  if (kind === "preference") {
+    return `Preference: ${firstWords(base, 24)}`;
+  }
+  if (kind === "decision") {
+    return `Decision: ${firstWords(base, 24)}`;
+  }
+  if (kind === "fact") {
+    return `Fact: ${firstWords(base, 24)}`;
+  }
+  if (kind === "task") {
+    return `Recurring task context: ${firstWords(base, 24)}`;
+  }
+  return firstWords(base, 28);
+}

package/src/memory-selection.ts

@@ -0,0 +1,257 @@
+import { normalizeWhitespace } from "./chunking.js";
+import {
+  isLikelyTranscriptLikeText,
+  isLikelyTurnRecap,
+} from "./capture.js";
+import {
+  primaryTaxonomyAnchor,
+  taxonomyTerms,
+} from "./memory-model.js";
+import type {
+  MemoryBraidConfig,
+} from "./config.js";
+import type {
+  MemoryKind,
+  MemorySelectionDecision,
+  TaxonomyBuckets,
+} from "./types.js";
+
+type SelectionResult = {
+  decision: MemorySelectionDecision;
+  score: number;
+  reasons: string[];
+};
+
+function clampScore(value: number): number {
+  return Math.max(0, Math.min(1, value));
+}
+
+function pushReason(reasons: string[], condition: boolean, reason: string): void {
+  if (condition) {
+    reasons.push(reason);
+  }
+}
+
+function stableSignal(text: string): boolean {
+  return /\b(?:prefer|timezone|name is|works at|work at|team|organization|project|repo|workspace|we decided|decision|we will|we use|deploy|stack|tooling)\b/i.test(
+    text,
+  );
+}
+
+function explicitRememberSignal(text: string): boolean {
+  return /^(?:remember|note)\b/i.test(text);
+}
+
+function volatileSignal(text: string): boolean {
+  return /\b(?:today|tomorrow|yesterday|later today|this afternoon|tonight|this week|next week|this session|this chat|just now|one-off)\b/i.test(
+    text,
+  );
+}
+
+function recurringTaskSignal(text: string): boolean {
+  return /\b(?:every|weekly|monthly|each|recurring|routine|regularly)\b/i.test(text);
+}
+
+function firstPersonOwnershipSignal(text: string): boolean {
+  return /\b(?:my|i prefer|i like|i use|we decided|our|we use)\b/i.test(text);
+}
+
+function thresholdForKind(cfg: MemoryBraidConfig, kind: MemoryKind): number {
+  if (kind === "preference" || kind === "decision") {
+    return cfg.capture.selection.minPreferenceDecisionScore;
+  }
+  if (kind === "fact") {
+    return cfg.capture.selection.minFactScore;
+  }
+  if (kind === "task") {
+    return cfg.capture.selection.minTaskScore;
+  }
+  return cfg.capture.selection.minOtherScore;
+}
+
+export function scoreObservedMemory(params: {
+  text: string;
+  kind: MemoryKind;
+  extractionScore: number;
+  taxonomy: TaxonomyBuckets;
+  source: "heuristic" | "ml";
+  cfg: MemoryBraidConfig;
+}): SelectionResult {
+  const text = normalizeWhitespace(params.text);
+  const reasons: string[] = [];
+  if (!text || isLikelyTranscriptLikeText(text) || isLikelyTurnRecap(text)) {
+    return {
+      decision: "ignore",
+      score: 0,
+      reasons: ["invalid_or_recap"],
+    };
+  }
+
+  let score = clampScore(params.extractionScore) * 0.45;
+  const taxonomyCount = taxonomyTerms(params.taxonomy).length;
+  const hasAnchor = Boolean(primaryTaxonomyAnchor(params.taxonomy));
+
+  if (params.kind === "preference") {
+    score += 0.22;
+    reasons.push("kind:preference");
+  } else if (params.kind === "decision") {
+    score += 0.2;
+    reasons.push("kind:decision");
+  } else if (params.kind === "fact") {
+    score += 0.14;
+    reasons.push("kind:fact");
+  } else if (params.kind === "task") {
+    score += 0.04;
+    reasons.push("kind:task");
+  }
+
+  pushReason(reasons, explicitRememberSignal(text), "explicit_remember");
+  if (explicitRememberSignal(text)) {
+    score += 0.06;
+  }
+  pushReason(reasons, stableSignal(text), "stable_signal");
+  if (stableSignal(text)) {
+    score += 0.12;
+  }
+  pushReason(reasons, firstPersonOwnershipSignal(text), "first_person");
+  if (firstPersonOwnershipSignal(text)) {
+    score += 0.08;
+  }
+  pushReason(reasons, hasAnchor, "taxonomy_anchor");
+  if (hasAnchor) {
+    score += 0.08;
+  }
+  if (taxonomyCount >= 2) {
+    score += 0.04;
+    reasons.push("taxonomy_rich");
+  }
+  if (params.source === "ml") {
+    reasons.push("ml_extracted");
+  }
+  pushReason(reasons, volatileSignal(text), "volatile_signal");
+  if (volatileSignal(text)) {
+    score -= 0.35;
+  }
+  if (params.kind === "task" && !recurringTaskSignal(text)) {
+    score -= 0.2;
+    reasons.push("one_off_task_penalty");
+  }
+  if (params.kind === "other") {
+    score -= 0.18;
+    reasons.push("kind:other_penalty");
+  }
+
+  const finalScore = clampScore(score);
+  return {
+    decision: finalScore >= thresholdForKind(params.cfg, params.kind) ? "episodic" : "ignore",
+    score: finalScore,
+    reasons,
+  };
+}
+
+export function scoreProceduralMemory(params: {
+  text: string;
+  confidence?: number;
+  captureIntent: "explicit_tool" | "self_reflection";
+  cfg: MemoryBraidConfig;
+}): SelectionResult {
+  const text = normalizeWhitespace(params.text);
+  const reasons: string[] = [];
+  if (!text || isLikelyTranscriptLikeText(text) || isLikelyTurnRecap(text)) {
+    return {
+      decision: "ignore",
+      score: 0,
+      reasons: ["invalid_or_recap"],
+    };
+  }
+
+  let score = clampScore(params.confidence ?? 0.65) * 0.4;
+  if (/\b(?:always|never|prefer|avoid|use|keep|store|limit|filter|dedupe|search|persist|only|when|if|strategy|approach|plan)\b/i.test(text)) {
+    score += 0.3;
+    reasons.push("reusable_procedure");
+  }
+  if (params.captureIntent === "explicit_tool") {
+    score += 0.12;
+    reasons.push("explicit_tool");
+  } else {
+    score += 0.05;
+    reasons.push("self_reflection");
+  }
+  if (text.length >= 32 && text.length <= 220) {
+    score += 0.08;
+    reasons.push("compact_atomic");
+  }
+  if (volatileSignal(text)) {
+    score -= 0.35;
+    reasons.push("volatile_signal");
+  }
+
+  const finalScore = clampScore(score);
+  return {
+    decision: finalScore >= params.cfg.capture.selection.minProceduralScore ? "procedural" : "ignore",
+    score: finalScore,
+    reasons,
+  };
+}
+
+export function scoreSemanticPromotion(params: {
+  kind: MemoryKind;
+  supportCount: number;
+  recallSupport: number;
+  taxonomy: TaxonomyBuckets;
+  firstSeenAt: number;
+  lastSeenAt: number;
+  sessionKeys: Set<string>;
+  text: string;
+  cfg: MemoryBraidConfig;
+}): SelectionResult {
+  const reasons: string[] = [];
+  let score = 0;
+  score += Math.min(0.4, Math.max(0, params.supportCount - 1) * 0.18);
+  if (params.supportCount > 1) {
+    reasons.push("repeated_support");
+  }
+  score += Math.min(0.18, params.recallSupport * 0.06);
+  if (params.recallSupport > 0) {
+    reasons.push("recall_reinforced");
+  }
+  if (params.sessionKeys.size > 1) {
+    score += 0.14;
+    reasons.push("cross_session");
+  }
+  const ageDays = Math.max(0, (params.lastSeenAt - params.firstSeenAt) / (24 * 60 * 60 * 1000));
+  if (ageDays >= 1) {
+    score += Math.min(0.12, ageDays / 14);
+    reasons.push("survived_over_time");
+  }
+  if (params.kind === "preference" || params.kind === "decision" || params.kind === "fact") {
+    score += 0.1;
+    reasons.push(`kind:${params.kind}`);
+  } else if (params.kind === "task" || params.kind === "other") {
+    score -= 0.12;
+    reasons.push(`kind:${params.kind}_penalty`);
+  }
+  if (primaryTaxonomyAnchor(params.taxonomy)) {
+    score += 0.08;
+    reasons.push("taxonomy_anchor");
+  }
+  if (taxonomyTerms(params.taxonomy).length >= 2) {
+    score += 0.04;
+    reasons.push("taxonomy_rich");
+  }
+  if (volatileSignal(params.text) && params.kind !== "preference" && params.kind !== "decision") {
+    score -= 0.18;
+    reasons.push("volatile_signal");
+  }
+
+  const finalScore = clampScore(score);
+  return {
+    decision: finalScore >= params.cfg.consolidation.minSelectionScore ? "semantic" : "ignore",
+    score: finalScore,
+    reasons,
+  };
+}
+
+export function summarizeSelection(result: SelectionResult): string {
+  return `${result.decision} score=${result.score.toFixed(2)} reasons=${result.reasons.join(",") || "n/a"}`;
+}

package/src/state.ts

@@ -2,6 +2,7 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import type {
   CaptureDedupeState,
+  ConsolidationState,
   LifecycleState,
   PluginStatsState,
   RemediationState,
@@ -47,6 +48,16 @@ const DEFAULT_STATS: PluginStatsState = {
     agentLearningAutoRejected: 0,
     agentLearningInjected: 0,
     agentLearningRecallHits: 0,
+    selectionSkipped: 0,
+    agentLearningRejectedSelection: 0,
+    consolidationRuns: 0,
+    consolidationCandidates: 0,
+    clustersFormed: 0,
+    semanticCreated: 0,
+    semanticUpdated: 0,
+    episodicMarkedConsolidated: 0,
+    contradictionsDetected: 0,
+    supersededMarked: 0,
   },
 };
 
@@ -55,12 +66,19 @@ const DEFAULT_REMEDIATION: RemediationState = {
   quarantined: {},
 };
 
+const DEFAULT_CONSOLIDATION: ConsolidationState = {
+  version: 1,
+  newEpisodicSinceLastRun: 0,
+  semanticByCompendiumKey: {},
+};
+
 export type StatePaths = {
   rootDir: string;
   captureDedupeFile: string;
   lifecycleFile: string;
   statsFile: string;
   remediationFile: string;
+  consolidationFile: string;
   stateLockFile: string;
 };
 
@@ -72,6 +90,7 @@ export function createStatePaths(stateDir: string): StatePaths {
     lifecycleFile: path.join(rootDir, "lifecycle.v1.json"),
     statsFile: path.join(rootDir, "stats.v1.json"),
     remediationFile: path.join(rootDir, "remediation.v1.json"),
+    consolidationFile: path.join(rootDir, "consolidation.v1.json"),
     stateLockFile: path.join(rootDir, "state.v1.lock"),
   };
 }
@@ -160,6 +179,27 @@ export async function writeRemediationState(
   await writeJsonFile(paths.remediationFile, state);
 }
 
+export async function readConsolidationState(paths: StatePaths): Promise<ConsolidationState> {
+  const value = await readJsonFile(paths.consolidationFile, DEFAULT_CONSOLIDATION);
+  return {
+    version: 1,
+    lastConsolidationAt: value.lastConsolidationAt,
+    lastConsolidationReason: value.lastConsolidationReason,
+    newEpisodicSinceLastRun:
+      typeof value.newEpisodicSinceLastRun === "number" && Number.isFinite(value.newEpisodicSinceLastRun)
+        ? Math.max(0, Math.round(value.newEpisodicSinceLastRun))
+        : 0,
+    semanticByCompendiumKey: { ...(value.semanticByCompendiumKey ?? {}) },
+  };
+}
+
+export async function writeConsolidationState(
+  paths: StatePaths,
+  state: ConsolidationState,
+): Promise<void> {
+  await writeJsonFile(paths.consolidationFile, state);
+}
+
 export async function withStateLock<T>(
   lockFilePath: string,
   fn: () => Promise<T>,