myaiforone 1.0.0

package/src/logger.ts

@@ -0,0 +1,51 @@
+import { appendFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+
+export type LogLevel = "debug" | "info" | "warn" | "error";
+
+const LEVELS: Record<LogLevel, number> = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+};
+
+let currentLevel: LogLevel = "info";
+let logFilePath: string | null = null;
+
+export function configureLogger(level: LogLevel, filePath?: string): void {
+  currentLevel = level;
+  if (filePath) {
+    logFilePath = filePath;
+    mkdirSync(dirname(filePath), { recursive: true });
+  }
+}
+
+function write(level: LogLevel, msg: string, ...args: unknown[]): void {
+  if (LEVELS[level] < LEVELS[currentLevel]) return;
+
+  const ts = new Date().toISOString();
+  const prefix = `[${ts}] [${level.toUpperCase()}]`;
+  const line = args.length > 0 ? `${prefix} ${msg} ${args.map(a => JSON.stringify(a)).join(" ")}` : `${prefix} ${msg}`;
+
+  if (level === "error") {
+    console.error(line);
+  } else {
+    console.log(line);
+  }
+
+  if (logFilePath) {
+    try {
+      appendFileSync(logFilePath, line + "\n");
+    } catch {
+      // silently ignore file write errors
+    }
+  }
+}
+
+export const log = {
+  debug: (msg: string, ...args: unknown[]) => write("debug", msg, ...args),
+  info: (msg: string, ...args: unknown[]) => write("info", msg, ...args),
+  warn: (msg: string, ...args: unknown[]) => write("warn", msg, ...args),
+  error: (msg: string, ...args: unknown[]) => write("error", msg, ...args),
+};

package/src/memory/daily.ts

@@ -0,0 +1,101 @@
+/**
+ * Daily memory — append-only daily log files.
+ * After each exchange, a summary is appended to memory/daily/YYYY-MM-DD.md.
+ * At session start, today + yesterday are loaded for continuity.
+ */
+
+import { readFileSync, appendFileSync, existsSync, mkdirSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+
+function dateStr(date: Date): string {
+  return date.toISOString().split("T")[0];
+}
+
+function timeStr(date: Date): string {
+  return date.toTimeString().split(" ")[0].slice(0, 5);
+}
+
+export function getDailyDir(memoryDir: string): string {
+  const dir = join(memoryDir, "daily");
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+/**
+ * Append an exchange summary to today's daily log.
+ */
+export function appendDailyEntry(
+  memoryDir: string,
+  sender: string,
+  userMessage: string,
+  agentResponse: string,
+): void {
+  const dir = getDailyDir(memoryDir);
+  const today = dateStr(new Date());
+  const time = timeStr(new Date());
+  const filePath = join(dir, `${today}.md`);
+
+  // Truncate long messages for the daily log
+  const userSnippet = userMessage.slice(0, 200).replace(/\n/g, " ");
+  const agentSnippet = agentResponse.slice(0, 300).replace(/\n/g, " ");
+
+  const entry = `\n### ${time}\n- **User:** ${userSnippet}\n- **Agent:** ${agentSnippet}\n`;
+
+  // Add header if new file
+  if (!existsSync(filePath)) {
+    appendFileSync(filePath, `# Daily Log — ${today}\n`);
+  }
+
+  appendFileSync(filePath, entry);
+}
+
+/**
+ * Load today's and yesterday's daily logs for context injection.
+ */
+export function loadRecentDaily(memoryDir: string): string {
+  const dir = getDailyDir(memoryDir);
+
+  const today = new Date();
+  const yesterday = new Date(today);
+  yesterday.setDate(yesterday.getDate() - 1);
+
+  const files = [
+    { date: dateStr(yesterday), label: "Yesterday" },
+    { date: dateStr(today), label: "Today" },
+  ];
+
+  const parts: string[] = [];
+
+  for (const { date, label } of files) {
+    const filePath = join(dir, `${date}.md`);
+    if (existsSync(filePath)) {
+      try {
+        const content = readFileSync(filePath, "utf-8").trim();
+        if (content) {
+          parts.push(`[${label}'s Activity — ${date}]\n${content}\n[/${label}]`);
+        }
+      } catch { /* ignore */ }
+    }
+  }
+
+  return parts.join("\n\n");
+}
+
+/**
+ * Get all daily file paths for indexing.
+ */
+export function listDailyFiles(memoryDir: string): Array<{ path: string; date: string }> {
+  const dir = getDailyDir(memoryDir);
+
+  try {
+    return readdirSync(dir)
+      .filter((f: string) => f.endsWith(".md"))
+      .map((f: string) => ({
+        path: join(dir, f),
+        date: f.replace(".md", ""),
+      }))
+      .sort((a: any, b: any) => a.date.localeCompare(b.date));
+  } catch {
+    return [];
+  }
+}

package/src/memory/embeddings.ts

@@ -0,0 +1,185 @@
+/**
+ * Embedding providers — auto-selects best available:
+ * 1. OpenAI text-embedding-3-small (if OPENAI_API_KEY set)
+ * 2. TF-IDF local (zero dependencies, always works)
+ */
+
+import { log } from "../logger.js";
+
+export interface EmbeddingProvider {
+  name: string;
+  dimensions: number;
+  embed(text: string): Promise<number[]>;
+  embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+// ─── TF-IDF Local Provider (zero deps) ──────────────────────────────
+
+const STOP_WORDS = new Set([
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "shall", "can", "need", "dare", "ought",
+  "used", "to", "of", "in", "for", "on", "with", "at", "by", "from",
+  "as", "into", "through", "during", "before", "after", "above", "below",
+  "between", "out", "off", "over", "under", "again", "further", "then",
+  "once", "here", "there", "when", "where", "why", "how", "all", "each",
+  "every", "both", "few", "more", "most", "other", "some", "such", "no",
+  "nor", "not", "only", "own", "same", "so", "than", "too", "very",
+  "just", "because", "but", "and", "or", "if", "while", "that", "this",
+  "it", "its", "i", "me", "my", "we", "our", "you", "your", "he", "him",
+  "his", "she", "her", "they", "them", "their", "what", "which", "who",
+]);
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, " ")
+    .split(/\s+/)
+    .filter(t => t.length > 1 && !STOP_WORDS.has(t));
+}
+
+// Global vocabulary built from all indexed texts
+const globalVocab = new Map<string, number>(); // word -> index
+let vocabSize = 0;
+
+export function buildVocab(texts: string[]): void {
+  for (const text of texts) {
+    for (const token of tokenize(text)) {
+      if (!globalVocab.has(token)) {
+        globalVocab.set(token, vocabSize++);
+      }
+    }
+  }
+}
+
+function tfidfVector(text: string): number[] {
+  const tokens = tokenize(text);
+  if (tokens.length === 0) return new Array(Math.max(vocabSize, 1)).fill(0);
+
+  // Term frequency
+  const tf = new Map<string, number>();
+  for (const t of tokens) {
+    tf.set(t, (tf.get(t) || 0) + 1);
+  }
+
+  // Build sparse vector
+  const vec = new Array(Math.max(vocabSize, 1)).fill(0);
+  for (const [word, count] of tf) {
+    const idx = globalVocab.get(word);
+    if (idx !== undefined) {
+      vec[idx] = count / tokens.length; // normalized TF
+    }
+  }
+
+  // L2 normalize
+  const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
+  if (norm > 0) {
+    for (let i = 0; i < vec.length; i++) vec[i] /= norm;
+  }
+
+  return vec;
+}
+
+class TFIDFProvider implements EmbeddingProvider {
+  name = "tfidf-local";
+  dimensions = 0; // dynamic, grows with vocab
+
+  async embed(text: string): Promise<number[]> {
+    buildVocab([text]);
+    this.dimensions = vocabSize;
+    return tfidfVector(text);
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    buildVocab(texts);
+    this.dimensions = vocabSize;
+    return texts.map(t => tfidfVector(t));
+  }
+}
+
+// ─── OpenAI Provider ─────────────────────────────────────────────────
+
+class OpenAIProvider implements EmbeddingProvider {
+  name = "openai-text-embedding-3-small";
+  dimensions = 1536;
+  private apiKey: string;
+
+  constructor(apiKey: string) {
+    this.apiKey = apiKey;
+  }
+
+  async embed(text: string): Promise<number[]> {
+    const results = await this.embedBatch([text]);
+    return results[0];
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    try {
+      const resp = await fetch("https://api.openai.com/v1/embeddings", {
+        method: "POST",
+        headers: {
+          "Authorization": `Bearer ${this.apiKey}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          model: "text-embedding-3-small",
+          input: texts,
+        }),
+      });
+
+      if (!resp.ok) {
+        log.warn(`OpenAI embeddings failed (${resp.status}), falling back to TF-IDF`);
+        const fallback = new TFIDFProvider();
+        return fallback.embedBatch(texts);
+      }
+
+      const data = await resp.json() as {
+        data: Array<{ embedding: number[] }>;
+      };
+
+      return data.data.map(d => d.embedding);
+    } catch (err) {
+      log.warn(`OpenAI embeddings error: ${err}, falling back to TF-IDF`);
+      const fallback = new TFIDFProvider();
+      return fallback.embedBatch(texts);
+    }
+  }
+}
+
+// ─── Provider selection ──────────────────────────────────────────────
+
+let cachedProvider: EmbeddingProvider | null = null;
+
+export function getEmbeddingProvider(): EmbeddingProvider {
+  if (cachedProvider) return cachedProvider;
+
+  const openaiKey = process.env.OPENAI_API_KEY;
+  if (openaiKey) {
+    cachedProvider = new OpenAIProvider(openaiKey);
+    log.info(`Memory embeddings: OpenAI (text-embedding-3-small)`);
+  } else {
+    cachedProvider = new TFIDFProvider();
+    log.info(`Memory embeddings: TF-IDF local (set OPENAI_API_KEY for better quality)`);
+  }
+
+  return cachedProvider;
+}
+
+// ─── Cosine similarity ───────────────────────────────────────────────
+
+export function cosineSimilarity(a: number[], b: number[]): number {
+  // Handle mismatched dimensions (TF-IDF vocab can grow)
+  const len = Math.min(a.length, b.length);
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  // Include remaining dimensions in norms
+  for (let i = len; i < a.length; i++) normA += a[i] * a[i];
+  for (let i = len; i < b.length; i++) normB += b[i] * b[i];
+
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}

package/src/memory/index.ts

@@ -0,0 +1,218 @@
+/**
+ * MemoryManager — orchestrates advanced memory for agents.
+ * - Semantic search over stored memories
+ * - Daily memory logging
+ * - Auto-compaction (prompts agent to save before context grows too large)
+ * - Indexes context.md, daily logs, and conversation exchanges
+ */
+
+import { randomUUID } from "node:crypto";
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { getEmbeddingProvider, buildVocab, type EmbeddingProvider } from "./embeddings.js";
+import { createStore, type VectorStore, type MemoryChunk } from "./store.js";
+import { hybridSearch, formatSearchResults, type SearchResult } from "./search.js";
+import { appendDailyEntry, loadRecentDaily, listDailyFiles } from "./daily.js";
+import { log } from "../logger.js";
+
+export interface MemoryManager {
+  /** Search memories relevant to a query */
+  search(query: string, topK?: number): Promise<SearchResult[]>;
+
+  /** Format search results for prompt injection */
+  searchFormatted(query: string, topK?: number): Promise<string>;
+
+  /** Index a new text chunk into memory */
+  index(text: string, source: string, metadata?: Record<string, string>): Promise<void>;
+
+  /** Index a conversation exchange */
+  indexExchange(userMessage: string, agentResponse: string, sender: string): Promise<void>;
+
+  /** Load recent daily context */
+  loadDailyContext(): string;
+
+  /** Re-index all memory sources (context.md + daily files) */
+  reindex(): Promise<void>;
+
+  /** Get auto-compaction prompt if needed */
+  getCompactionPrompt(messageCount: number): string | null;
+
+  /** Get memory stats */
+  stats(): Promise<{ chunks: number; store: string; embeddings: string }>;
+}
+
+// ─── Chunk text into ~400 token pieces ───────────────────────────────
+
+function chunkText(text: string, maxChars: number = 1200): string[] {
+  const paragraphs = text.split(/\n\n+/);
+  const chunks: string[] = [];
+  let current = "";
+
+  for (const para of paragraphs) {
+    if (current.length + para.length > maxChars && current.length > 0) {
+      chunks.push(current.trim());
+      current = "";
+    }
+    current += para + "\n\n";
+  }
+
+  if (current.trim()) {
+    chunks.push(current.trim());
+  }
+
+  return chunks.length > 0 ? chunks : [text];
+}
+
+// ─── Auto-compaction thresholds ──────────────────────────────────────
+
+const COMPACTION_WARN_MESSAGES = 20;   // suggest compaction after 20 messages
+const COMPACTION_FORCE_MESSAGES = 40;  // strongly suggest after 40
+
+// ─── Create MemoryManager ────────────────────────────────────────────
+
+export async function createMemoryManager(memoryDir: string): Promise<MemoryManager> {
+  const embedder = getEmbeddingProvider();
+  const store = await createStore(memoryDir);
+  const contextPath = join(memoryDir, "context.md");
+
+  // Initial index if store is empty
+  const count = await store.count();
+  if (count === 0) {
+    log.info(`Memory: empty store, indexing existing files...`);
+    await indexExistingFiles(memoryDir, contextPath, embedder, store);
+  }
+
+  return {
+    async search(query: string, topK = 5): Promise<SearchResult[]> {
+      const queryVector = await embedder.embed(query);
+      const allChunks = await store.getAll();
+      return hybridSearch(queryVector, query, allChunks, { topK });
+    },
+
+    async searchFormatted(query: string, topK = 5): Promise<string> {
+      const queryVector = await embedder.embed(query);
+      const allChunks = await store.getAll();
+      const results = hybridSearch(queryVector, query, allChunks, { topK });
+      return formatSearchResults(results);
+    },
+
+    async index(text: string, source: string, metadata?: Record<string, string>): Promise<void> {
+      const chunks = chunkText(text);
+      const vectors = await embedder.embedBatch(chunks);
+
+      const memoryChunks: MemoryChunk[] = chunks.map((text, i) => ({
+        id: `${source}-${randomUUID().slice(0, 8)}`,
+        text,
+        vector: vectors[i],
+        source,
+        timestamp: new Date().toISOString(),
+        metadata,
+      }));
+
+      await store.addBatch(memoryChunks);
+    },
+
+    async indexExchange(userMessage: string, agentResponse: string, sender: string): Promise<void> {
+      // Log to daily file
+      appendDailyEntry(memoryDir, sender, userMessage, agentResponse);
+
+      // Index the exchange as a memory chunk
+      const summary = `User (${sender}): ${userMessage.slice(0, 200)}\nAgent: ${agentResponse.slice(0, 400)}`;
+      const vector = await embedder.embed(summary);
+
+      await store.add({
+        id: `conv-${Date.now()}`,
+        text: summary,
+        vector,
+        source: "conversation",
+        timestamp: new Date().toISOString(),
+        metadata: { sender },
+      });
+    },
+
+    loadDailyContext(): string {
+      return loadRecentDaily(memoryDir);
+    },
+
+    async reindex(): Promise<void> {
+      await store.clear();
+      await indexExistingFiles(memoryDir, contextPath, embedder, store);
+      log.info(`Memory: reindexed`);
+    },
+
+    getCompactionPrompt(messageCount: number): string | null {
+      if (messageCount >= COMPACTION_FORCE_MESSAGES) {
+        return `[SYSTEM: Your conversation has reached ${messageCount} messages. Before the context gets compressed, save any important decisions, facts, or context to ${contextPath} using the Write tool. Focus on: key decisions made, user preferences learned, project state, and anything you'd need to know if starting fresh. Merge with existing content — don't overwrite.]`;
+      }
+      if (messageCount >= COMPACTION_WARN_MESSAGES) {
+        return `[SYSTEM: You're at ${messageCount} messages. Consider using /opcompact to save important context soon, or I'll remind you again at ${COMPACTION_FORCE_MESSAGES} messages.]`;
+      }
+      return null;
+    },
+
+    async stats() {
+      return {
+        chunks: await store.count(),
+        store: store.name,
+        embeddings: embedder.name,
+      };
+    },
+  };
+}
+
+// ─── Index existing files into the store ─────────────────────────────
+
+async function indexExistingFiles(
+  memoryDir: string,
+  contextPath: string,
+  embedder: EmbeddingProvider,
+  store: VectorStore,
+): Promise<void> {
+  const allTexts: Array<{ text: string; source: string; timestamp: string }> = [];
+
+  // Index context.md
+  if (existsSync(contextPath)) {
+    try {
+      const content = readFileSync(contextPath, "utf-8").trim();
+      if (content) {
+        const chunks = chunkText(content);
+        for (const chunk of chunks) {
+          allTexts.push({ text: chunk, source: "context", timestamp: new Date().toISOString() });
+        }
+      }
+    } catch { /* ignore */ }
+  }
+
+  // Index daily files
+  const dailyFiles = listDailyFiles(memoryDir);
+  for (const { path, date } of dailyFiles) {
+    try {
+      const content = readFileSync(path, "utf-8").trim();
+      if (content) {
+        const chunks = chunkText(content);
+        for (const chunk of chunks) {
+          allTexts.push({ text: chunk, source: "daily", timestamp: `${date}T00:00:00.000Z` });
+        }
+      }
+    } catch { /* ignore */ }
+  }
+
+  if (allTexts.length === 0) return;
+
+  // Build vocab for TF-IDF (if using local embeddings)
+  buildVocab(allTexts.map(t => t.text));
+
+  // Embed all at once
+  const vectors = await embedder.embedBatch(allTexts.map(t => t.text));
+
+  const memoryChunks: MemoryChunk[] = allTexts.map((t, i) => ({
+    id: `${t.source}-${randomUUID().slice(0, 8)}`,
+    text: t.text,
+    vector: vectors[i],
+    source: t.source,
+    timestamp: t.timestamp,
+  }));
+
+  await store.addBatch(memoryChunks);
+  log.info(`Memory: indexed ${memoryChunks.length} chunks from ${dailyFiles.length} daily files + context.md`);
+}

package/src/memory/search.ts

@@ -0,0 +1,124 @@
+/**
+ * Hybrid semantic search: cosine similarity + BM25 keyword + temporal decay.
+ */
+
+import { cosineSimilarity } from "./embeddings.js";
+import type { MemoryChunk } from "./store.js";
+
+export interface SearchResult {
+  chunk: MemoryChunk;
+  score: number;         // combined score 0-1
+  vectorScore: number;   // cosine similarity
+  keywordScore: number;  // BM25-like score
+  decayFactor: number;   // temporal decay multiplier
+}
+
+// ─── BM25 keyword scoring ────────────────────────────────────────────
+
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+
+function tokenize(text: string): string[] {
+  return text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(t => t.length > 1);
+}
+
+function bm25Score(query: string, document: string, avgDocLen: number): number {
+  const queryTokens = tokenize(query);
+  const docTokens = tokenize(document);
+  if (queryTokens.length === 0 || docTokens.length === 0) return 0;
+
+  const docLen = docTokens.length;
+  const tf = new Map<string, number>();
+  for (const t of docTokens) {
+    tf.set(t, (tf.get(t) || 0) + 1);
+  }
+
+  let score = 0;
+  for (const qt of queryTokens) {
+    const freq = tf.get(qt) || 0;
+    if (freq === 0) continue;
+    const numerator = freq * (BM25_K1 + 1);
+    const denominator = freq + BM25_K1 * (1 - BM25_B + BM25_B * (docLen / avgDocLen));
+    score += numerator / denominator;
+  }
+
+  // Normalize to 0-1 range (approximate)
+  return Math.min(score / queryTokens.length, 1);
+}
+
+// ─── Temporal decay ──────────────────────────────────────────────────
+
+const HALF_LIFE_DAYS = 30;
+const DECAY_LAMBDA = Math.LN2 / HALF_LIFE_DAYS;
+
+function temporalDecay(timestamp: string, source: string): number {
+  // context.md and manual entries never decay
+  if (source === "context" || source === "manual") return 1.0;
+
+  const ageMs = Date.now() - new Date(timestamp).getTime();
+  const ageDays = ageMs / (1000 * 60 * 60 * 24);
+  return Math.exp(-DECAY_LAMBDA * ageDays);
+}
+
+// ─── Hybrid search ───────────────────────────────────────────────────
+
+export interface SearchOptions {
+  topK?: number;           // max results (default 5)
+  vectorWeight?: number;   // weight for vector score (default 0.6)
+  keywordWeight?: number;  // weight for keyword score (default 0.3)
+  decayWeight?: number;    // weight for temporal decay (default 0.1)
+  minScore?: number;       // minimum combined score (default 0.1)
+}
+
+export function hybridSearch(
+  queryVector: number[],
+  queryText: string,
+  chunks: MemoryChunk[],
+  options: SearchOptions = {},
+): SearchResult[] {
+  const {
+    topK = 5,
+    vectorWeight = 0.6,
+    keywordWeight = 0.3,
+    decayWeight = 0.1,
+    minScore = 0.1,
+  } = options;
+
+  if (chunks.length === 0) return [];
+
+  // Average document length for BM25
+  const avgDocLen = chunks.reduce((sum, c) => sum + tokenize(c.text).length, 0) / chunks.length;
+
+  const results: SearchResult[] = chunks.map(chunk => {
+    const vectorScore = cosineSimilarity(queryVector, chunk.vector);
+    const keywordScore = bm25Score(queryText, chunk.text, avgDocLen);
+    const decayFactor = temporalDecay(chunk.timestamp, chunk.source);
+
+    const score =
+      vectorWeight * vectorScore +
+      keywordWeight * keywordScore +
+      decayWeight * decayFactor;
+
+    return { chunk, score, vectorScore, keywordScore, decayFactor };
+  });
+
+  return results
+    .filter(r => r.score >= minScore)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, topK);
+}
+
+// ─── Format search results for prompt injection ──────────────────────
+
+export function formatSearchResults(results: SearchResult[]): string {
+  if (results.length === 0) return "";
+
+  const lines = ["[Relevant Memories]"];
+  for (const r of results) {
+    const date = r.chunk.timestamp.split("T")[0];
+    const source = r.chunk.source;
+    lines.push(`- [${date}/${source}] ${r.chunk.text}`);
+  }
+  lines.push("[/Relevant Memories]");
+  return lines.join("\n");
+}