nemoris 0.1.0

package/src/memory/retrieval-planner.js

@@ -0,0 +1,66 @@
+const STOPWORDS = new Set([
+  "the",
+  "and",
+  "with",
+  "that",
+  "this",
+  "from",
+  "into",
+  "your",
+  "have",
+  "will",
+  "would",
+  "should",
+  "could",
+  "through",
+  "using",
+  "use",
+  "for",
+  "job",
+  "type",
+  "bounded",
+  "runtime",
+  "overhead",
+  "execute",
+  "keep",
+  "only",
+  "what",
+  "deserves"
+]);
+
+const TASK_HINTS = {
+  heartbeat: ["heartbeat", "memory", "status", "check", "triage"],
+  workspace_health: ["workspace", "health", "status", "calendar", "issues", "weather", "blockers", "deltas"],
+  memory_rollup: ["memory", "summary", "notes", "rollup", "handoff", "inbox", "projects", "backlog", "update"]
+};
+
+function tokenize(text) {
+  return String(text || "")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/i)
+    .filter(Boolean)
+    .filter((token) => token.length > 2 && !STOPWORDS.has(token));
+}
+
+function unique(tokens) {
+  return [...new Set(tokens)];
+}
+
+export function planRetrievalQueries({ objective, taskType, purpose, identity, reportGuidance = null }) {
+  const objectiveTokens = tokenize(objective);
+  const purposeTokens = tokenize(purpose || identity?.purpose || "").slice(0, 12);
+  const taskTokens = TASK_HINTS[taskType] || tokenize(String(taskType || "").replaceAll("_", " "));
+  const guidanceTokens = tokenize(
+    [reportGuidance?.focus || [], reportGuidance?.qualityChecks || []]
+      .flat()
+      .join(" ")
+  ).slice(0, 12);
+
+  const fileQuery = unique([...objectiveTokens, ...taskTokens, ...purposeTokens, ...guidanceTokens]).slice(0, 22).join(" ");
+  const qmdQuery = unique([...taskTokens, ...guidanceTokens, ...purposeTokens, ...objectiveTokens]).slice(0, 10).join(" ");
+
+  return {
+    fileQuery: fileQuery || objective,
+    qmdQuery: qmdQuery || fileQuery || objective
+  };
+}

package/src/memory/scoring.js

@@ -0,0 +1,145 @@
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+const TYPE_WEIGHT = {
+  fact: 1,
+  summary: 0.9,
+  event: 0.55,
+  scratchpad: 0.75
+};
+
+const SYNONYMS = {
+  heartbeat: ["check", "status", "pulse", "monitor"],
+  memory: ["notes", "context", "recall", "summary"],
+  workspace: ["vault", "project", "repo", "files"],
+  health: ["status", "stability", "signal", "check"],
+  purpose: ["mission", "role", "goal"],
+  soul: ["identity", "voice", "values"],
+  cron: ["schedule", "job", "automation"]
+};
+
+export const DEFAULT_RETRIEVAL_BLEND = {
+  lexicalWeight: 0.36,
+  embeddingWeight: 0.3,
+  recencyWeight: 0.14,
+  salienceWeight: 0.14,
+  typeWeight: 0.06,
+  semanticRescueBonus: 0.06,
+  shadowSnapshotPenalty: 0.12
+};
+
+function tokenize(text) {
+  return String(text || "")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/i)
+    .filter(Boolean);
+}
+
+function expandTokens(tokens) {
+  const expanded = new Set(tokens);
+  for (const token of tokens) {
+    const synonyms = SYNONYMS[token];
+    if (!synonyms) continue;
+    for (const synonym of synonyms) expanded.add(synonym);
+  }
+  return [...expanded];
+}
+
+export function computeLexicalScore(query, text) {
+  const queryTokens = new Set(expandTokens(tokenize(query)));
+  if (queryTokens.size === 0) return 0;
+  const textTokens = tokenize(text);
+  if (textTokens.length === 0) return 0;
+  let hits = 0;
+  for (const token of textTokens) {
+    if (queryTokens.has(token)) hits += 1;
+  }
+  return Math.min(1, hits / Math.max(queryTokens.size, 1));
+}
+
+export function computeRecencyScore(timestamp, now = Date.now(), halfLifeDays = 14) {
+  if (!timestamp) return 0.2;
+  const ageDays = Math.max(0, now - new Date(timestamp).getTime()) / DAY_MS;
+  return Math.pow(0.5, ageDays / halfLifeDays);
+}
+
+export function scoreMemory(item, query, now = Date.now(), blend = {}) {
+  const weights = {
+    ...DEFAULT_RETRIEVAL_BLEND,
+    ...blend
+  };
+  const text = [item.title, item.content, item.summary, item.reason, item.category]
+    .filter(Boolean)
+    .join(" ");
+  const lexicalScore = Number(item.lexicalScore ?? computeLexicalScore(query, text));
+  const embeddingSimilarity = Number(item.embeddingSimilarity ?? 0);
+  const recencyScore = computeRecencyScore(item.timestamp, now);
+  const salienceScore = Math.max(0, Math.min(1, Number(item.salience ?? 0.5)));
+  const typeScore = TYPE_WEIGHT[item.type] ?? 0.5;
+  const shadowSnapshotPenalty =
+    item.sourceKind === "shadow_snapshot" || String(item.title || "").startsWith("shadow snapshot")
+      ? weights.shadowSnapshotPenalty
+      : 0;
+  const semanticRescueBonus =
+    lexicalScore < 0.12 && embeddingSimilarity >= 0.35 && item.embeddingFreshness === "fresh"
+      ? weights.semanticRescueBonus
+      : 0;
+
+  const blendedScore =
+    lexicalScore * weights.lexicalWeight +
+    embeddingSimilarity * weights.embeddingWeight +
+    recencyScore * weights.recencyWeight +
+    salienceScore * weights.salienceWeight +
+    typeScore * weights.typeWeight +
+    semanticRescueBonus -
+    shadowSnapshotPenalty;
+
+  return {
+    score: blendedScore,
+    lexicalScore,
+    embeddingSimilarity,
+    recencyScore,
+    salienceScore,
+    typeScore,
+    semanticRescueBonus,
+    shadowSnapshotPenalty
+  };
+}
+
+// Compatibility helper retained for tests and older call sites.
+export function computeDenseSimilarity(query, text, dimensions = 64) {
+  const queryVector = vectorize(expandTokens(tokenize(query)), dimensions);
+  const textVector = vectorize(expandTokens(tokenize(text)), dimensions);
+  return cosineSimilarity(queryVector, textVector);
+}
+
+function vectorize(tokens, dimensions) {
+  const vector = new Array(dimensions).fill(0);
+  for (const token of tokens) {
+    const hash = stableHash(token);
+    const index = Math.abs(hash) % dimensions;
+    const sign = hash % 2 === 0 ? 1 : -1;
+    vector[index] += sign * (1 + token.length / 10);
+  }
+  return vector;
+}
+
+function stableHash(text) {
+  let hash = 0;
+  for (let i = 0; i < text.length; i += 1) {
+    hash = (hash * 31 + text.charCodeAt(i)) | 0;
+  }
+  return hash;
+}
+
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/src/memory/simhash.js

@@ -0,0 +1,78 @@
+const STOP_WORDS = new Set([
+  "a", "an", "the", "is", "was", "are", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "shall",
+  "should", "may", "might", "can", "could", "must", "ought",
+  "i", "me", "my", "we", "our", "you", "your", "he", "him", "his",
+  "she", "her", "it", "its", "they", "them", "their",
+  "this", "that", "these", "those",
+  "in", "on", "at", "to", "for", "of", "with", "by", "from",
+  "and", "but", "or", "not", "no", "nor", "so", "yet",
+  "if", "then", "else", "when", "where", "how", "what", "which", "who",
+  "all", "each", "every", "both", "few", "more", "most", "some", "any",
+  "just", "very", "also", "too", "only"
+]);
+
+// FNV-1a 64-bit hash
+const FNV_OFFSET = 0xcbf29ce484222325n;
+const FNV_PRIME = 0x100000001b3n;
+const MASK_64 = (1n << 64n) - 1n;
+
+function fnv1a64(str) {
+  let hash = FNV_OFFSET;
+  for (let i = 0; i < str.length; i++) {
+    hash ^= BigInt(str.charCodeAt(i));
+    hash = (hash * FNV_PRIME) & MASK_64;
+  }
+  return hash;
+}
+
+function tokenize(text) {
+  return String(text || "")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((token) => token.length > 0 && !STOP_WORDS.has(token));
+}
+
+export function computeSimhash(text) {
+  const tokens = tokenize(text);
+  if (tokens.length === 0) return 0n;
+
+  const weights = new Int32Array(64);
+
+  for (const token of tokens) {
+    const hash = fnv1a64(token);
+    for (let i = 0; i < 64; i++) {
+      if ((hash >> BigInt(i)) & 1n) {
+        weights[i] += 1;
+      } else {
+        weights[i] -= 1;
+      }
+    }
+  }
+
+  let result = 0n;
+  for (let i = 0; i < 64; i++) {
+    if (weights[i] > 0) {
+      result |= (1n << BigInt(i));
+    }
+  }
+  return result;
+}
+
+export function hammingDistance(a, b) {
+  let xor = a ^ b;
+  let count = 0;
+  while (xor > 0n) {
+    count += Number(xor & 1n);
+    xor >>= 1n;
+  }
+  return count;
+}
+
+export function simhashToHex(hash) {
+  return hash.toString(16).padStart(16, "0");
+}
+
+export function hexToSimhash(hex) {
+  return BigInt(`0x${hex}`);
+}