@yemi33/minions 0.1.2369 → 0.1.2371

package/engine/memory-retrieval.js

@@ -0,0 +1,165 @@
+// engine/memory-retrieval.js — deterministic task-aware memory selection.
+
+const memoryStore = require('./memory-store');
+
+const DEFAULT_TOP_K = 8;
+const DEFAULT_MAX_BYTES = 8 * 1024;
+const DEFAULT_CANDIDATES = 50;
+
+function buildTaskMemoryQuery(context = {}) {
+  const references = Array.isArray(context.references)
+    ? context.references.map(ref => {
+      if (typeof ref === 'string') return ref;
+      if (!ref || typeof ref !== 'object') return '';
+      return [ref.path, ref.url, ref.title, ref.type].filter(Boolean).join(' ');
+    }).join(' ')
+    : String(context.references || '');
+  return [
+    context.title,
+    context.description,
+    context.type,
+    context.failureClass,
+    context.sourcePlan,
+    references,
+    context.prTitle,
+    context.prBranch,
+  ].filter(Boolean).join('\n').trim();
+}
+
+function _trustBoost(trust) {
+  if (trust === 'human') return 12;
+  if (trust === 'system') return 8;
+  if (trust === 'agent') return 3;
+  return 0;
+}
+
+function _scopeBoost(record, context) {
+  if (record.scopeType === 'project' && record.scopeKey === context.project) return 10;
+  if (record.scopeType === 'agent' && record.scopeKey === context.agent) return 7;
+  return 2;
+}
+
+function _pathBoost(record, query) {
+  const source = String(record.sourcePath || '').toLowerCase();
+  if (!source) return 0;
+  const pathTokens = String(query || '').toLowerCase().match(/[a-z0-9_.-]+[\\/][a-z0-9_./\\-]+/g) || [];
+  return pathTokens.some(token => source.includes(token.replace(/\\/g, '/'))) ? 15 : 0;
+}
+
+function _recencyBoost(updatedAt, now) {
+  const ageDays = Math.max(0, (now - Number(updatedAt || 0)) / 86400000);
+  if (ageDays <= 7) return 4;
+  if (ageDays <= 30) return 2;
+  if (ageDays <= 90) return 1;
+  return 0;
+}
+
+function rankMemoryCandidates(records, context = {}, query = '') {
+  const now = Date.now();
+  return records.map((record, index) => ({
+    ...record,
+    score:
+      ((records.length - index) * 10) +
+      _scopeBoost(record, context) +
+      _trustBoost(record.trust) +
+      _pathBoost(record, query) +
+      (_recencyBoost(record.updatedAt, now)) +
+      (Number(record.importance || 0) * 5) +
+      (Number(record.confidence || 0) * 3),
+  })).sort((a, b) =>
+    b.score - a.score ||
+    Number(b.updatedAt || 0) - Number(a.updatedAt || 0) ||
+    String(a.id).localeCompare(String(b.id))
+  );
+}
+
+function dedupeMemoryCandidates(records) {
+  const seen = new Set();
+  return records.filter(record => {
+    const key = record.contentHash || `${record.sourceType}\0${record.sourceRef}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function _truncateUtf8(text, maxBytes) {
+  const input = String(text || '');
+  if (Buffer.byteLength(input, 'utf8') <= maxBytes) return input;
+  let low = 0;
+  let high = input.length;
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (Buffer.byteLength(input.slice(0, mid), 'utf8') <= maxBytes) low = mid;
+    else high = mid - 1;
+  }
+  return input.slice(0, low);
+}
+
+function _formatRecord(record, bodyBytes) {
+  const provenance = [
+    `id=${record.id}`,
+    `type=${record.memoryType}`,
+    `scope=${record.scopeType}${record.scopeKey ? `:${record.scopeKey}` : ''}`,
+    `date=${new Date(record.validFrom || record.createdAt).toISOString().slice(0, 10)}`,
+    `source=${record.sourcePath || record.sourceRef}`,
+  ].join(' | ');
+  const body = _truncateUtf8(record.body, bodyBytes);
+  return `### ${record.title}\n_${provenance}_\n\n${body}${body.length < record.body.length ? '\n\n_[excerpt truncated]_' : ''}`;
+}
+
+function packMemoryResults(records, options = {}) {
+  const topK = Math.max(1, Math.min(30, Number(options.topK) || DEFAULT_TOP_K));
+  const maxBytes = Math.max(1024, Math.min(64 * 1024, Number(options.maxBytes) || DEFAULT_MAX_BYTES));
+  const selected = [];
+  const blocks = [];
+  let used = 0;
+  for (const record of records.slice(0, topK)) {
+    const remaining = maxBytes - used;
+    if (remaining < 256) break;
+    const block = _formatRecord(record, Math.max(128, remaining - 256));
+    const separator = blocks.length ? '\n\n---\n\n' : '';
+    const bytes = Buffer.byteLength(separator + block, 'utf8');
+    if (bytes > remaining) continue;
+    blocks.push(block);
+    selected.push(record);
+    used += bytes;
+  }
+  return { text: blocks.join('\n\n---\n\n'), selected, bytes: used };
+}
+
+function retrieveRelevantMemory(context = {}, options = {}) {
+  const started = process.hrtime.bigint();
+  const query = buildTaskMemoryQuery(context);
+  if (!query) {
+    return { query: '', candidates: [], selected: [], text: '', bytes: 0, durationMs: 0 };
+  }
+  const candidates = memoryStore.searchMemoryRecords(query, {
+    project: context.project,
+    agent: context.agent,
+    limit: Number(options.candidateLimit) || DEFAULT_CANDIDATES,
+  });
+  const ranked = dedupeMemoryCandidates(rankMemoryCandidates(candidates, context, query));
+  const packed = packMemoryResults(ranked, options);
+  const durationMs = Number(process.hrtime.bigint() - started) / 1e6;
+  return {
+    query,
+    candidates: ranked,
+    selected: packed.selected,
+    text: packed.text,
+    bytes: packed.bytes,
+    durationMs,
+  };
+}
+
+module.exports = {
+  DEFAULT_TOP_K,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_CANDIDATES,
+  buildTaskMemoryQuery,
+  rankMemoryCandidates,
+  dedupeMemoryCandidates,
+  packMemoryResults,
+  retrieveRelevantMemory,
+  _truncateUtf8,
+};

package/engine/memory-store.js

@@ -0,0 +1,367 @@
+// engine/memory-store.js — SQL-backed long-term memory records and retrieval.
+
+const crypto = require('crypto');
+const { getDb, withTransaction, prepareCached } = require('./db');
+const { emitStateEvent } = require('./db-events');
+
+const MEMORY_TYPES = new Set(['semantic', 'episodic', 'procedural']);
+const SCOPE_TYPES = new Set(['global', 'project', 'agent']);
+const STATUSES = new Set(['active', 'superseded', 'retracted', 'expired']);
+const TRUST_LEVELS = new Set(['human', 'system', 'agent', 'external']);
+const MAX_RETRIEVAL_RUNS = 5000;
+const QUERY_STOP_WORDS = new Set([
+  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from', 'in', 'is',
+  'it', 'of', 'on', 'or', 'that', 'the', 'this', 'to', 'with',
+  'add', 'change', 'code', 'file', 'fix', 'implement', 'issue', 'project',
+  'task', 'test', 'update', 'work', 'item', 'agent',
+]);
+
+function _clamp(value, fallback) {
+  const n = Number(value);
+  return Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : fallback;
+}
+
+function _json(value, fallback) {
+  try { return JSON.stringify(value == null ? fallback : value); }
+  catch { return JSON.stringify(fallback); }
+}
+
+function _parseJson(value, fallback) {
+  try { return JSON.parse(value); } catch { return fallback; }
+}
+
+function _hash(value) {
+  return crypto.createHash('sha256').update(String(value || '')).digest('hex');
+}
+
+function _idFor(record, contentHash) {
+  if (record.id) return String(record.id);
+  return `M-${_hash(`${record.sourceType}\0${record.sourceRef}\0${contentHash}`).slice(0, 24)}`;
+}
+
+function _normalize(record) {
+  if (!record || typeof record !== 'object') throw new TypeError('memory record is required');
+  const memoryType = MEMORY_TYPES.has(record.memoryType) ? record.memoryType : 'semantic';
+  const scopeType = SCOPE_TYPES.has(record.scopeType) ? record.scopeType : 'global';
+  const scopeKey = scopeType === 'global' ? '' : String(record.scopeKey || '').trim();
+  if (scopeType !== 'global' && !scopeKey) throw new TypeError(`memory scopeKey required for ${scopeType} scope`);
+  const title = String(record.title || '').trim();
+  const body = String(record.body || '').trim();
+  if (!title) throw new TypeError('memory title is required');
+  if (!body) throw new TypeError('memory body is required');
+  const sourceType = String(record.sourceType || '').trim();
+  const sourceRef = String(record.sourceRef || '').trim();
+  if (!sourceType || !sourceRef) throw new TypeError('memory sourceType and sourceRef are required');
+  const contentHash = _hash(body);
+  const now = Date.now();
+  return {
+    id: _idFor(record, contentHash),
+    memoryType,
+    scopeType,
+    scopeKey,
+    title,
+    body,
+    tags: Array.isArray(record.tags) ? record.tags.map(String).filter(Boolean) : [],
+    sourceType,
+    sourcePath: record.sourcePath ? String(record.sourcePath) : null,
+    sourceRef,
+    trust: TRUST_LEVELS.has(record.trust) ? record.trust : 'agent',
+    importance: _clamp(record.importance, 0.5),
+    confidence: _clamp(record.confidence, 0.5),
+    status: STATUSES.has(record.status) ? record.status : 'active',
+    validFrom: Number.isFinite(Number(record.validFrom)) ? Number(record.validFrom) : now,
+    validTo: Number.isFinite(Number(record.validTo)) ? Number(record.validTo) : null,
+    supersedesId: record.supersedesId ? String(record.supersedesId) : null,
+    supersededBy: record.supersededBy ? String(record.supersededBy) : null,
+    contentHash,
+    metadata: record.metadata && typeof record.metadata === 'object' ? record.metadata : {},
+    createdAt: Number.isFinite(Number(record.createdAt)) ? Number(record.createdAt) : now,
+    updatedAt: now,
+  };
+}
+
+function _row(row) {
+  if (!row) return null;
+  return {
+    id: row.id,
+    memoryType: row.memory_type,
+    scopeType: row.scope_type,
+    scopeKey: row.scope_key,
+    title: row.title,
+    body: row.body,
+    tags: _parseJson(row.tags, []),
+    sourceType: row.source_type,
+    sourcePath: row.source_path,
+    sourceRef: row.source_ref,
+    trust: row.trust,
+    importance: row.importance,
+    confidence: row.confidence,
+    status: row.status,
+    validFrom: row.valid_from,
+    validTo: row.valid_to,
+    supersedesId: row.supersedes_id,
+    supersededBy: row.superseded_by,
+    contentHash: row.content_hash,
+    metadata: _parseJson(row.metadata, {}),
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    ftsRank: row.fts_rank,
+  };
+}
+
+function upsertMemoryRecord(input) {
+  const record = _normalize(input);
+  const db = getDb();
+  withTransaction(db, () => {
+    if (!record.supersedesId) {
+      const prior = db.prepare(`
+        SELECT id FROM memory_records
+        WHERE source_type=? AND source_ref=? AND status='active' AND content_hash<>?
+        ORDER BY updated_at DESC LIMIT 1
+      `).get(record.sourceType, record.sourceRef, record.contentHash);
+      if (prior) record.supersedesId = prior.id;
+    }
+    if (record.supersedesId) {
+      db.prepare(`
+        UPDATE memory_records
+        SET status='superseded', valid_to=?, superseded_by=?, updated_at=?
+        WHERE id=? AND status='active'
+      `).run(record.validFrom, record.id, record.updatedAt, record.supersedesId);
+    }
+    db.prepare(`
+      INSERT INTO memory_records (
+        id, memory_type, scope_type, scope_key, title, body, tags,
+        source_type, source_path, source_ref, trust, importance, confidence,
+        status, valid_from, valid_to, supersedes_id, superseded_by,
+        content_hash, metadata, created_at, updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        memory_type=excluded.memory_type,
+        scope_type=excluded.scope_type,
+        scope_key=excluded.scope_key,
+        title=excluded.title,
+        body=excluded.body,
+        tags=excluded.tags,
+        source_path=excluded.source_path,
+        trust=excluded.trust,
+        importance=excluded.importance,
+        confidence=excluded.confidence,
+        status=excluded.status,
+        valid_from=excluded.valid_from,
+        valid_to=excluded.valid_to,
+        supersedes_id=excluded.supersedes_id,
+        superseded_by=excluded.superseded_by,
+        metadata=excluded.metadata,
+        updated_at=excluded.updated_at
+    `).run(
+      record.id, record.memoryType, record.scopeType, record.scopeKey,
+      record.title, record.body, _json(record.tags, []),
+      record.sourceType, record.sourcePath, record.sourceRef, record.trust,
+      record.importance, record.confidence, record.status, record.validFrom,
+      record.validTo, record.supersedesId, record.supersededBy,
+      record.contentHash, _json(record.metadata, {}), record.createdAt, record.updatedAt,
+    );
+  });
+  emitStateEvent('memory_records', { id: record.id, status: record.status });
+  return getMemoryRecord(record.id);
+}
+
+function getMemoryRecord(id) {
+  const db = getDb();
+  return _row(prepareCached(db, 'SELECT * FROM memory_records WHERE id=?').get(String(id)));
+}
+
+function updateMemoryRecordState(id, action) {
+  const recordId = String(id || '').trim();
+  if (!recordId) throw new TypeError('memory record id is required');
+  if (!['pin', 'retract', 'restore'].includes(action)) {
+    throw new TypeError(`unsupported memory action: ${action}`);
+  }
+  if (!getMemoryRecord(recordId)) return null;
+
+  const db = getDb();
+  const now = Date.now();
+  if (action === 'pin') {
+    db.prepare(`
+      UPDATE memory_records
+      SET trust='human', importance=1, confidence=MAX(confidence, 0.9),
+          status='active', valid_to=NULL, superseded_by=NULL, updated_at=?
+      WHERE id=?
+    `).run(now, recordId);
+  } else if (action === 'restore') {
+    db.prepare(`
+      UPDATE memory_records
+      SET status='active', valid_to=NULL, superseded_by=NULL, updated_at=?
+      WHERE id=?
+    `).run(now, recordId);
+  } else {
+    db.prepare(`
+      UPDATE memory_records
+      SET status='retracted', valid_to=?, updated_at=?
+      WHERE id=?
+    `).run(now, now, recordId);
+  }
+  emitStateEvent('memory_records', { id: recordId, action });
+  return getMemoryRecord(recordId);
+}
+
+function listMemoryRecords(filters = {}) {
+  const where = [];
+  const args = [];
+  for (const [column, value] of [
+    ['status', filters.status],
+    ['memory_type', filters.memoryType],
+    ['scope_type', filters.scopeType],
+    ['scope_key', filters.scopeKey],
+  ]) {
+    if (value == null || value === '') continue;
+    where.push(`${column}=?`);
+    args.push(String(value));
+  }
+  const limit = Math.max(1, Math.min(500, Number(filters.limit) || 100));
+  const sql = `SELECT * FROM memory_records${where.length ? ` WHERE ${where.join(' AND ')}` : ''} ORDER BY updated_at DESC LIMIT ?`;
+  return prepareCached(getDb(), sql).all(...args, limit).map(_row);
+}
+
+function _matchQuery(query) {
+  const tokens = String(query || '')
+    .toLowerCase()
+    .match(/[a-z0-9][a-z0-9_.-]{1,63}/g);
+  if (!tokens) return '';
+  const unique = [...new Set(tokens)]
+    .filter(token => !QUERY_STOP_WORDS.has(token))
+    .slice(0, 32);
+  return unique.map(token => `"${token.replace(/"/g, '""')}"`).join(' OR ');
+}
+
+function searchMemoryRecords(query, options = {}) {
+  const match = _matchQuery(query);
+  if (!match) return [];
+  const where = [
+    'memory_records_fts MATCH ?',
+    "(m.memory_type<>'procedural' OR m.trust IN ('human','system'))",
+  ];
+  const args = [match];
+  const statuses = Array.isArray(options.statuses)
+    ? options.statuses.filter(status => STATUSES.has(status))
+    : ['active'];
+  if (!statuses.length) return [];
+  where.push(`m.status IN (${statuses.map(() => '?').join(',')})`);
+  args.push(...statuses);
+  if (!options.allScopes) {
+    const scopes = ["m.scope_type='global'"];
+    if (options.project) {
+      scopes.push("(m.scope_type='project' AND m.scope_key=?)");
+      args.push(String(options.project));
+    }
+    if (options.agent) {
+      scopes.push("(m.scope_type='agent' AND m.scope_key=?)");
+      args.push(String(options.agent));
+    }
+    where.push(`(${scopes.join(' OR ')})`);
+  }
+  if (Array.isArray(options.memoryTypes) && options.memoryTypes.length) {
+    const types = options.memoryTypes.filter(t => MEMORY_TYPES.has(t));
+    if (types.length) {
+      where.push(`m.memory_type IN (${types.map(() => '?').join(',')})`);
+      args.push(...types);
+    }
+  }
+  const limit = Math.max(1, Math.min(200, Number(options.limit) || 50));
+  args.push(limit);
+  const sql = `
+    SELECT m.*, bm25(memory_records_fts, 5.0, 1.0, 2.0, 3.0) AS fts_rank
+    FROM memory_records_fts
+    JOIN memory_records m ON m.rowid=memory_records_fts.rowid
+    WHERE ${where.join(' AND ')}
+    ORDER BY fts_rank
+    LIMIT ?
+  `;
+  return prepareCached(getDb(), sql).all(...args).map(_row);
+}
+
+function transitionMemoryRecord(id, status, options = {}) {
+  if (!STATUSES.has(status) || status === 'active') throw new TypeError(`invalid memory status: ${status}`);
+  const now = Date.now();
+  const result = getDb().prepare(`
+    UPDATE memory_records
+    SET status=?, valid_to=?, superseded_by=COALESCE(?, superseded_by), updated_at=?
+    WHERE id=?
+  `).run(status, Number(options.validTo) || now, options.supersededBy || null, now, String(id));
+  if (result.changes) emitStateEvent('memory_records', { id: String(id), status });
+  return result.changes > 0;
+}
+
+function supersedeMatchingRecords(options = {}) {
+  const texts = Array.isArray(options.oldTexts) ? options.oldTexts.map(String).filter(Boolean) : [];
+  if (!texts.length || !options.supersededBy) return 0;
+  const db = getDb();
+  const now = Date.now();
+  let changed = 0;
+  withTransaction(db, () => {
+    for (const text of texts) {
+      const rows = db.prepare(`
+        SELECT id FROM memory_records
+        WHERE status='active' AND scope_type=? AND scope_key=? AND instr(body, ?) > 0
+      `).all(options.scopeType || 'global', options.scopeKey || '', text);
+      for (const row of rows) {
+        const result = db.prepare(`
+          UPDATE memory_records
+          SET status='superseded', valid_to=?, superseded_by=?, updated_at=?
+          WHERE id=? AND status='active'
+        `).run(now, String(options.supersededBy), now, row.id);
+        changed += Number(result.changes) || 0;
+      }
+    }
+  });
+  if (changed) emitStateEvent('memory_records', { supersededBy: String(options.supersededBy), changed });
+  return changed;
+}
+
+function recordRetrievalRun(run) {
+  const db = getDb();
+  withTransaction(db, () => {
+    db.prepare(`
+      INSERT INTO memory_retrieval_runs (
+        work_item_id, project, agent, query, candidate_count, selected_count,
+        selected_bytes, duration_ms, shadow, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      run.workItemId || null, run.project || null, run.agent || null,
+      String(run.query || ''), Number(run.candidateCount) || 0,
+      Number(run.selectedCount) || 0, Number(run.selectedBytes) || 0,
+      Number(run.durationMs) || 0, run.shadow ? 1 : 0, Date.now(),
+    );
+    db.prepare(`
+      DELETE FROM memory_retrieval_runs
+      WHERE id NOT IN (SELECT id FROM memory_retrieval_runs ORDER BY id DESC LIMIT ?)
+    `).run(MAX_RETRIEVAL_RUNS);
+  });
+}
+
+function getRetrievalStats() {
+  return prepareCached(getDb(), `
+    SELECT COUNT(*) AS runs,
+           COALESCE(AVG(duration_ms), 0) AS avg_duration_ms,
+           COALESCE(AVG(selected_count), 0) AS avg_selected_count,
+           COALESCE(AVG(selected_bytes), 0) AS avg_selected_bytes,
+           COALESCE(MAX(duration_ms), 0) AS max_duration_ms
+    FROM memory_retrieval_runs
+  `).get();
+}
+
+module.exports = {
+  MEMORY_TYPES,
+  SCOPE_TYPES,
+  STATUSES,
+  upsertMemoryRecord,
+  getMemoryRecord,
+  updateMemoryRecordState,
+  listMemoryRecords,
+  searchMemoryRecords,
+  transitionMemoryRecord,
+  supersedeMatchingRecords,
+  recordRetrievalRun,
+  getRetrievalStats,
+  _matchQuery,
+};