@ngockhoale/ukit 1.1.6

package/src/core/memory/retrieval.js

@@ -0,0 +1,476 @@
+import { compactContextBlock, recordCompaction } from '../compact/index.js';
+import { readCompactPressureState } from '../compact/threshold.js';
+import {
+  buildCompactMachineKey,
+  readPromptCacheEntry,
+  writePromptCacheEntry,
+} from '../token/index.js';
+import { listMemoryItems } from './store.js';
+
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
+  'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
+  'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
+]);
+
+function normalize(text) {
+  return String(text ?? '')
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .split(/\s+/)
+    .filter((token) => token && !STOPWORDS.has(token) && token.length > 1);
+}
+
+function compactPhraseList(values, { limit = 1, fallback = 'n/a' } = {}) {
+  const normalized = [...new Set(
+    (values ?? [])
+      .map((value) => String(value ?? '').trim())
+      .filter(Boolean),
+  )];
+
+  if (normalized.length === 0) {
+    return fallback;
+  }
+
+  const shown = normalized.slice(0, limit);
+  const remaining = normalized.length - shown.length;
+  return remaining > 0
+    ? `${shown.join('; ')} (+${remaining} more)`
+    : shown.join('; ');
+}
+
+function buildSegments(item) {
+  const content = item.content ?? {};
+
+  if (item.type === 'user') {
+    return [
+      { text: item.summary, weight: 2 },
+      { text: JSON.stringify(content.preferences ?? {}), weight: 1 },
+      { text: (content.rules ?? []).join(' '), weight: 2 },
+    ];
+  }
+
+  if (item.type === 'project') {
+    return [
+      { text: item.summary, weight: 2 },
+      { text: content.name, weight: 2 },
+      { text: content.architecture, weight: 3 },
+      { text: (content.techStack ?? []).join(' '), weight: 2 },
+      { text: (content.activeRules ?? []).join(' '), weight: 2 },
+      { text: (content.conventions ?? []).join(' '), weight: 1 },
+      { text: (content.decisions ?? []).map((decision) => `${decision.what} ${decision.why}`).join(' '), weight: 7 },
+    ];
+  }
+
+  return [
+    { text: item.summary, weight: 2 },
+    { text: content.taskDescription, weight: 4 },
+    { text: content.outcome, weight: 1 },
+    { text: (content.keyActions ?? []).join(' '), weight: 3 },
+    { text: (content.nextSteps ?? []).join(' '), weight: 2 },
+    { text: (content.filesChanged ?? []).join(' '), weight: 1 },
+    { text: (content.decisionsMade ?? []).map((decision) => `${decision.what} ${decision.why}`).join(' '), weight: 4 },
+  ];
+}
+
+function getTimestamp(item) {
+  return item.content?.updatedAt
+    ?? item.content?.endedAt
+    ?? item.content?.startedAt
+    ?? 0;
+}
+
+function summarizeForSearch(item) {
+  const content = item.content ?? {};
+
+  if (item.type === 'project') {
+    const decisions = (content.decisions ?? []).map((decision) => decision.what).slice(0, 3).join('; ');
+    return decisions ? `${item.summary} — ${decisions}` : item.summary;
+  }
+
+  if (item.type === 'session') {
+    const actions = (content.keyActions ?? []).slice(0, 2).join('; ');
+    return actions ? `${item.summary} — ${actions}` : item.summary;
+  }
+
+  return item.summary;
+}
+
+function scoreItem(item, queryTokens) {
+  if (queryTokens.length === 0) {
+    return 1;
+  }
+
+  let score = 0;
+  const segments = buildSegments(item);
+
+  for (const segment of segments) {
+    const segmentTokens = new Set(tokenize(segment.text));
+    for (const token of queryTokens) {
+      if (segmentTokens.has(token)) {
+        score += segment.weight;
+      }
+    }
+  }
+
+  if (score === 0) {
+    return 0;
+  }
+
+  const recencyBonus = getTimestamp(item) > 0 ? Math.min(1, getTimestamp(item) / Date.now()) : 0;
+  return score + recencyBonus;
+}
+
+function withinScope(item, scope, projectId) {
+  if (scope !== 'all' && item.type !== scope) {
+    return false;
+  }
+
+  if (!projectId) {
+    return true;
+  }
+
+  if (item.type === 'project') {
+    return item.content?.id === projectId;
+  }
+
+  if (item.type === 'session') {
+    return item.content?.projectId === projectId;
+  }
+
+  return true;
+}
+
+function buildSearchResult(item, relevanceScore) {
+  return {
+    id: item.id,
+    summary: summarizeForSearch(item),
+    relevanceScore,
+    source: item.type,
+    timestamp: getTimestamp(item),
+    sourcePath: item.sourcePath,
+  };
+}
+
+export async function search(query, scope = 'all', limit = 10, { projectRoot, projectId } = {}) {
+  const items = await listMemoryItems(projectRoot);
+  const queryTokens = tokenize(query);
+
+  return items
+    .filter((item) => withinScope(item, scope, projectId))
+    .map((item) => ({
+      item,
+      score: scoreItem(item, queryTokens),
+    }))
+    .filter((entry) => entry.score > 0)
+    .sort((left, right) => (
+      right.score - left.score
+      || getTimestamp(right.item) - getTimestamp(left.item)
+      || right.item.summary.localeCompare(left.item.summary)
+    ))
+    .slice(0, limit)
+    .map((entry) => buildSearchResult(entry.item, entry.score));
+}
+
+function findRelatedItemIds(targetItem, allItems) {
+  if (targetItem.type === 'session') {
+    const projectId = targetItem.content?.projectId;
+    return allItems
+      .filter((item) => item.id !== targetItem.id)
+      .filter((item) => (
+        (item.type === 'project' && item.content?.id === projectId)
+        || (item.type === 'session' && item.content?.projectId === projectId)
+      ))
+      .map((item) => item.id)
+      .slice(0, 5);
+  }
+
+  if (targetItem.type === 'project') {
+    const projectId = targetItem.content?.id;
+    return allItems
+      .filter((item) => item.id !== targetItem.id)
+      .filter((item) => item.type === 'session' && item.content?.projectId === projectId)
+      .map((item) => item.id)
+      .slice(0, 5);
+  }
+
+  return allItems
+    .filter((item) => item.id !== targetItem.id && item.type !== 'user')
+    .map((item) => item.id)
+    .slice(0, 5);
+}
+
+export async function expand(id, { projectRoot } = {}) {
+  const items = await listMemoryItems(projectRoot);
+  const target = items.find((item) => item.id === id);
+  if (!target) {
+    return { id, fullContent: '', relatedItems: [] };
+  }
+
+  return {
+    id: target.id,
+    fullContent: JSON.stringify(target.content, null, 2),
+    relatedItems: findRelatedItemIds(target, items),
+  };
+}
+
+function buildDetailedInjectionSnippet(item) {
+  const content = item.content ?? {};
+  if (item.type === 'project') {
+    const decisions = compactPhraseList(
+      (content.decisions ?? []).map((decision) => `${decision.what} (${decision.why ?? 'no reason'})`),
+      { limit: 2 },
+    );
+    const rules = compactPhraseList(content.activeRules ?? [], { limit: 2 });
+    const stack = (content.techStack ?? []).slice(0, 3).join(', ');
+    return `${content.name ?? content.id ?? 'Project'} — architecture: ${content.architecture ?? 'n/a'} — stack: ${stack || 'n/a'} — decisions: ${decisions || 'n/a'}${rules ? ` — rules: ${rules}` : ''}`;
+  }
+
+  if (item.type === 'session') {
+    const actions = compactPhraseList(content.keyActions ?? [], { limit: 2 });
+    const nextSteps = compactPhraseList(content.nextSteps ?? [], { limit: 1 });
+    const files = (content.filesChanged ?? []).slice(0, 3).join(', ');
+    return `${content.taskDescription ?? item.id} — outcome: ${content.outcome ?? 'unknown'}${actions ? ` — actions: ${actions}` : ''}${files ? ` — files: ${files}` : ''}${nextSteps ? ` — next: ${nextSteps}` : ''}`;
+  }
+
+  return `User memory — rules: ${(content.rules ?? []).slice(0, 3).join('; ') || 'n/a'} — preferences: ${JSON.stringify(content.preferences ?? {})}`;
+}
+
+function buildInjectionSnippet(item) {
+  const content = item.content ?? {};
+  if (item.type === 'project') {
+    const decisions = compactPhraseList((content.decisions ?? []).map((decision) => decision.what), { limit: 1 });
+    const rules = compactPhraseList(content.activeRules ?? [], { limit: 1 });
+    return `project=${content.name ?? content.id ?? 'Project'}: decisions=${decisions || 'n/a'}${rules ? `: rules=${rules}` : ''}`;
+  }
+
+  if (item.type === 'session') {
+    const actions = compactPhraseList(content.keyActions ?? [], { limit: 1 });
+    const nextSteps = compactPhraseList(content.nextSteps ?? [], { limit: 1 });
+    return `session=${content.taskDescription ?? item.id}: outcome=${content.outcome ?? 'unknown'}${actions ? `: actions=${actions}` : ''}${nextSteps ? `: next=${nextSteps}` : ''}`;
+  }
+
+  return `user-rules=${(content.rules ?? []).slice(0, 3).join('; ') || 'n/a'}`;
+}
+
+function normalizePressurePhase(phase) {
+  const normalized = String(phase ?? '').trim().toLowerCase();
+  return ['monitor', 'soft', 'hard'].includes(normalized) ? normalized : 'monitor';
+}
+
+function resolveRecallPressurePlan({
+  maxTokens = 1000,
+  limit = 5,
+  pressureState = null,
+} = {}) {
+  const phase = normalizePressurePhase(pressureState?.phase);
+  if (phase === 'hard') {
+    return {
+      phase,
+      maxTokens: Math.min(maxTokens, Math.max(96, Math.round(maxTokens * 0.72))),
+      limit: Math.max(1, Math.min(limit, 3)),
+      includeUserMemory: false,
+    };
+  }
+
+  if (phase === 'soft') {
+    return {
+      phase,
+      maxTokens: Math.min(maxTokens, Math.max(112, Math.round(maxTokens * 0.88))),
+      limit: Math.max(1, Math.min(limit, 4)),
+      includeUserMemory: true,
+    };
+  }
+
+  return {
+    phase,
+    maxTokens,
+    limit: Math.max(1, limit),
+    includeUserMemory: true,
+  };
+}
+
+function isProjectAnchor(item, projectId) {
+  return Boolean(item && item.type === 'project' && (!projectId || item.content?.id === projectId));
+}
+
+function isSessionAnchor(item, projectId) {
+  return Boolean(item && item.type === 'session' && (!projectId || item.content?.projectId === projectId));
+}
+
+function dedupeItemsById(items) {
+  const seen = new Set();
+  const unique = [];
+  for (const item of items ?? []) {
+    if (!item?.id || seen.has(item.id)) {
+      continue;
+    }
+    seen.add(item.id);
+    unique.push(item);
+  }
+  return unique;
+}
+
+function selectContextItems({
+  items = [],
+  rankedItems = [],
+  projectId,
+  limit = 5,
+  includeUserMemory = true,
+  allowProjectFallback = false,
+} = {}) {
+  const itemById = new Map(items.map((item) => [item.id, item]));
+  const rankedSelection = rankedItems
+    .map((entry) => itemById.get(entry.id))
+    .filter(Boolean)
+    .filter((item) => includeUserMemory || item.type !== 'user');
+  if (rankedSelection.length === 0 && !allowProjectFallback) {
+    return [];
+  }
+  const projectAnchor = projectId
+    ? items.find((item) => item.type === 'project' && item.content?.id === projectId)
+    : rankedSelection.find((item) => item.type === 'project');
+  const prioritized = dedupeItemsById([
+    rankedSelection.length > 0 || allowProjectFallback ? projectAnchor : null,
+    ...rankedSelection.filter((item) => isSessionAnchor(item, projectId)),
+    ...rankedSelection.filter((item) => isProjectAnchor(item, projectId) && item.id !== projectAnchor?.id),
+    ...rankedSelection.filter((item) => item.type !== 'session' && item.type !== 'project'),
+  ]);
+
+  return prioritized.slice(0, Math.max(1, limit));
+}
+
+function buildContextRequestKey({
+  currentTask,
+  projectId,
+  maxTokens,
+  limit,
+  inputCompression,
+  selectedItems,
+  pressurePhase,
+}) {
+  return buildCompactMachineKey('memory-recall-v1', {
+    currentTask: normalize(currentTask),
+    projectId: projectId ?? null,
+    maxTokens,
+    limit,
+    inputCompression,
+    pressurePhase,
+    items: selectedItems.map((item) => ({
+      id: item.id,
+      timestamp: getTimestamp(item),
+    })),
+  });
+}
+
+export async function getContextInjection(
+  currentTask,
+  projectId,
+  {
+    projectRoot,
+    maxTokens = 1000,
+    limit = 5,
+    promptCache = false,
+    inputCompression = true,
+  } = {},
+) {
+  const items = await listMemoryItems(projectRoot);
+  const pressureState = await readCompactPressureState(projectRoot);
+  const recallPlan = resolveRecallPressurePlan({
+    maxTokens,
+    limit,
+    pressureState,
+  });
+  const rankedItems = await search(currentTask, 'all', Math.max(limit * 3, 8), { projectRoot, projectId });
+  const selectedItems = selectContextItems({
+    items,
+    rankedItems,
+    projectId,
+    limit: recallPlan.limit,
+    includeUserMemory: recallPlan.includeUserMemory,
+    allowProjectFallback: !String(currentTask ?? '').trim(),
+  });
+
+  if (selectedItems.length === 0) {
+    return '';
+  }
+
+  const requestKey = buildContextRequestKey({
+    currentTask,
+    projectId,
+    maxTokens: recallPlan.maxTokens,
+    limit: recallPlan.limit,
+    inputCompression,
+    selectedItems,
+    pressurePhase: recallPlan.phase,
+  });
+
+  if (promptCache) {
+    const cached = await readPromptCacheEntry(projectRoot, requestKey, { touch: true });
+    if (cached?.content) {
+      return cached.content;
+    }
+  }
+
+  const rawBodyLines = selectedItems.map((item) => `- [${item.type}] ${buildDetailedInjectionSnippet(item)}`);
+  const bodyLines = selectedItems.map((item) => `- ${buildInjectionSnippet(item)}`);
+  const rawLines = ['## Previous Context', ...rawBodyLines];
+  const rawText = rawLines.join('\n');
+  const conciseText = ['## Previous Context', ...bodyLines].join('\n');
+  const compacted = inputCompression
+    ? compactContextBlock(bodyLines, {
+        header: '## Previous Context',
+        maxTokens: recallPlan.maxTokens,
+        maxLines: Math.max(4, recallPlan.limit + 2),
+        anchorLines: bodyLines.slice(0, 2),
+        maxAnchors: 2,
+      })
+    : {
+        text: conciseText,
+        tokensBefore: 0,
+        tokensAfter: 0,
+        savedTokens: 0,
+      };
+
+  const output = compacted.text || conciseText;
+
+  if (promptCache) {
+    await writePromptCacheEntry(projectRoot, {
+      requestKey,
+      kind: 'memory-recall',
+      updatedAt: Date.now(),
+      tokensBefore: compacted.tokensBefore,
+      tokensAfter: compacted.tokensAfter,
+      savedTokens: compacted.savedTokens,
+      content: output,
+      metadata: {
+        projectId,
+        limit: recallPlan.limit,
+        pressurePhase: recallPlan.phase,
+        selectedIds: selectedItems.map((item) => item.id),
+      },
+    });
+  }
+
+  if (inputCompression) {
+    await recordCompaction(projectRoot, {
+      source: 'memory-recall',
+      beforeText: rawText,
+      afterText: output,
+      metadata: {
+        projectId,
+        limit: recallPlan.limit,
+        pressurePhase: recallPlan.phase,
+        selectedIds: selectedItems.map((item) => item.id),
+      },
+    });
+  }
+
+  return output;
+}

package/src/core/memory/store.js

@@ -0,0 +1,202 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { buildRuntimePaths } from '../runtimePaths.js';
+import { readJsonIfExists, writeJson } from '../fileOps.js';
+
+function defaultUserMemory() {
+  return {
+    preferences: {},
+    rules: [],
+    updatedAt: Date.now(),
+  };
+}
+
+async function readDirectoryJsonItems(dirPath) {
+  let entries = [];
+  try {
+    entries = await fs.readdir(dirPath, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+
+  const items = [];
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.endsWith('.json')) {
+      continue;
+    }
+
+    const fullPath = path.join(dirPath, entry.name);
+    const content = await readJsonIfExists(fullPath);
+    if (content) {
+      items.push({
+        fileName: entry.name,
+        filePath: fullPath,
+        content,
+      });
+    }
+  }
+
+  return items.sort((left, right) => left.fileName.localeCompare(right.fileName));
+}
+
+function summarizeUserMemory(userMemory) {
+  const preferenceCount = Object.keys(userMemory.preferences ?? {}).length;
+  const ruleCount = Array.isArray(userMemory.rules) ? userMemory.rules.length : 0;
+  return `${preferenceCount} preferences, ${ruleCount} rules`;
+}
+
+function summarizeProjectMemory(projectMemory) {
+  const decisionCount = Array.isArray(projectMemory.decisions) ? projectMemory.decisions.length : 0;
+  const techStack = Array.isArray(projectMemory.techStack) ? projectMemory.techStack.join(', ') : '';
+  return `${projectMemory.name ?? projectMemory.id ?? 'project'} — ${decisionCount} decisions${techStack ? ` — ${techStack}` : ''}`;
+}
+
+function summarizeSessionMemory(sessionMemory) {
+  const task = sessionMemory.taskDescription ?? sessionMemory.id ?? 'session';
+  const outcome = sessionMemory.outcome ?? 'unknown';
+  return `${task} — ${outcome}`;
+}
+
+function createSearchableText(type, content) {
+  if (type === 'user') {
+    return [
+      JSON.stringify(content.preferences ?? {}),
+      ...(content.rules ?? []),
+    ].join('\n');
+  }
+
+  if (type === 'project') {
+    return [
+      content.name,
+      content.architecture,
+      ...(content.techStack ?? []),
+      ...(content.activeRules ?? []),
+      ...(content.conventions ?? []),
+      ...((content.decisions ?? []).flatMap((decision) => [decision.what, decision.why])),
+    ].filter(Boolean).join('\n');
+  }
+
+  return [
+    content.taskDescription,
+    content.outcome,
+    ...(content.keyActions ?? []),
+    ...(content.nextSteps ?? []),
+    ...(content.filesChanged ?? []),
+  ].filter(Boolean).join('\n');
+}
+
+function computeRelevance(searchableText, query) {
+  if (!query) {
+    return 1;
+  }
+
+  const haystack = searchableText.toLowerCase();
+  const needle = query.toLowerCase();
+  if (!haystack.includes(needle)) {
+    return 0;
+  }
+
+  const parts = needle.split(/\s+/).filter(Boolean);
+  let score = 1;
+  for (const part of parts) {
+    if (haystack.includes(part)) {
+      score += 1;
+    }
+  }
+
+  return score;
+}
+
+export async function exportMemory(projectRoot) {
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  const user = (await readJsonIfExists(runtimePaths.userMemoryPath)) ?? defaultUserMemory();
+  const projects = (await readDirectoryJsonItems(runtimePaths.projectsDir)).map((item) => item.content);
+  const sessions = (await readDirectoryJsonItems(runtimePaths.sessionsDir)).map((item) => item.content);
+
+  return { user, projects, sessions };
+}
+
+export async function listMemoryItems(projectRoot) {
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  const userMemory = (await readJsonIfExists(runtimePaths.userMemoryPath)) ?? defaultUserMemory();
+  const projectMemories = await readDirectoryJsonItems(runtimePaths.projectsDir);
+  const sessionMemories = await readDirectoryJsonItems(runtimePaths.sessionsDir);
+
+  return [
+    {
+      id: 'user:user',
+      type: 'user',
+      summary: summarizeUserMemory(userMemory),
+      searchableText: createSearchableText('user', userMemory),
+      sourcePath: runtimePaths.userMemoryPath,
+      content: userMemory,
+    },
+    ...projectMemories.map((item) => ({
+      id: `project:${item.content.id ?? item.fileName.replace(/\.json$/, '')}`,
+      type: 'project',
+      summary: summarizeProjectMemory(item.content),
+      searchableText: createSearchableText('project', item.content),
+      sourcePath: item.filePath,
+      content: item.content,
+    })),
+    ...sessionMemories.map((item) => ({
+      id: `session:${item.content.id ?? item.fileName.replace(/\.json$/, '')}`,
+      type: 'session',
+      summary: summarizeSessionMemory(item.content),
+      searchableText: createSearchableText('session', item.content),
+      sourcePath: item.filePath,
+      content: item.content,
+    })),
+  ];
+}
+
+export async function searchMemoryItems(projectRoot, query, { limit = 10 } = {}) {
+  const items = await listMemoryItems(projectRoot);
+  return items
+    .map((item) => ({
+      ...item,
+      relevanceScore: computeRelevance(`${item.summary}\n${item.searchableText}`, query),
+    }))
+    .filter((item) => item.relevanceScore > 0)
+    .sort((left, right) => right.relevanceScore - left.relevanceScore)
+    .slice(0, limit);
+}
+
+export async function forgetMemoryItem(projectRoot, memoryId) {
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  if (memoryId === 'user:user' || memoryId === 'user') {
+    await writeJson(runtimePaths.userMemoryPath, defaultUserMemory());
+    return { removed: true, type: 'user', path: runtimePaths.userMemoryPath };
+  }
+
+  const [type, rawId] = String(memoryId).split(':');
+  if (!type || !rawId) {
+    return { removed: false, type: null, path: null };
+  }
+
+  const baseDir = type === 'project'
+    ? runtimePaths.projectsDir
+    : type === 'session'
+      ? runtimePaths.sessionsDir
+      : null;
+  if (!baseDir) {
+    return { removed: false, type: null, path: null };
+  }
+
+  const targetPath = path.join(baseDir, `${rawId}.json`);
+  try {
+    await fs.unlink(targetPath);
+    return { removed: true, type, path: targetPath };
+  } catch {
+    return { removed: false, type, path: targetPath };
+  }
+}
+
+export async function countMemoryItems(projectRoot) {
+  const exported = await exportMemory(projectRoot);
+  return {
+    projectCount: exported.projects.length,
+    sessionCount: exported.sessions.length,
+    userCount: Object.keys(exported.user.preferences ?? {}).length + (exported.user.rules ?? []).length,
+  };
+}