@hasna/mementos 0.8.0 → 0.9.0

package/dist/server/index.js

@@ -490,7 +490,30 @@ var init_database = __esm(() => {
   CREATE INDEX IF NOT EXISTS idx_webhook_hooks_type ON webhook_hooks(type);
   CREATE INDEX IF NOT EXISTS idx_webhook_hooks_enabled ON webhook_hooks(enabled);
   INSERT OR IGNORE INTO _migrations (id) VALUES (10);
-  `
+  `,
+    `
+CREATE TABLE IF NOT EXISTS session_memory_jobs (
+  id TEXT PRIMARY KEY,
+  session_id TEXT NOT NULL,
+  agent_id TEXT,
+  project_id TEXT,
+  source TEXT NOT NULL DEFAULT 'manual' CHECK(source IN ('claude-code','codex','manual','open-sessions')),
+  status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','processing','completed','failed')),
+  transcript TEXT NOT NULL,
+  chunk_count INTEGER NOT NULL DEFAULT 0,
+  memories_extracted INTEGER NOT NULL DEFAULT 0,
+  error TEXT,
+  metadata TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  started_at TEXT,
+  completed_at TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_session_memory_jobs_status ON session_memory_jobs(status);
+CREATE INDEX IF NOT EXISTS idx_session_memory_jobs_agent ON session_memory_jobs(agent_id);
+CREATE INDEX IF NOT EXISTS idx_session_memory_jobs_project ON session_memory_jobs(project_id);
+CREATE INDEX IF NOT EXISTS idx_session_memory_jobs_session ON session_memory_jobs(session_id);
+INSERT OR IGNORE INTO _migrations (id) VALUES (13);
+`
   ];
 });
 
@@ -4499,6 +4522,457 @@ function getSynthesisStatus(runId, projectId, db) {
 
 // src/server/index.ts
 init_synthesis();
+
+// src/db/session-jobs.ts
+init_database();
+function parseJobRow(row) {
+  return {
+    id: row["id"],
+    session_id: row["session_id"],
+    agent_id: row["agent_id"] || null,
+    project_id: row["project_id"] || null,
+    source: row["source"],
+    status: row["status"],
+    transcript: row["transcript"],
+    chunk_count: row["chunk_count"],
+    memories_extracted: row["memories_extracted"],
+    error: row["error"] || null,
+    metadata: JSON.parse(row["metadata"] || "{}"),
+    created_at: row["created_at"],
+    started_at: row["started_at"] || null,
+    completed_at: row["completed_at"] || null
+  };
+}
+function createSessionJob(input, db) {
+  const d = db || getDatabase();
+  const id = uuid();
+  const timestamp = now();
+  const source = input.source ?? "manual";
+  const metadata = JSON.stringify(input.metadata ?? {});
+  d.run(`INSERT INTO session_memory_jobs
+      (id, session_id, agent_id, project_id, source, status, transcript, chunk_count, memories_extracted, metadata, created_at)
+     VALUES (?, ?, ?, ?, ?, 'pending', ?, 0, 0, ?, ?)`, [
+    id,
+    input.session_id,
+    input.agent_id ?? null,
+    input.project_id ?? null,
+    source,
+    input.transcript,
+    metadata,
+    timestamp
+  ]);
+  return getSessionJob(id, d);
+}
+function getSessionJob(id, db) {
+  const d = db || getDatabase();
+  const row = d.query("SELECT * FROM session_memory_jobs WHERE id = ?").get(id);
+  if (!row)
+    return null;
+  return parseJobRow(row);
+}
+function listSessionJobs(filter, db) {
+  const d = db || getDatabase();
+  const conditions = [];
+  const params = [];
+  if (filter?.agent_id) {
+    conditions.push("agent_id = ?");
+    params.push(filter.agent_id);
+  }
+  if (filter?.project_id) {
+    conditions.push("project_id = ?");
+    params.push(filter.project_id);
+  }
+  if (filter?.status) {
+    conditions.push("status = ?");
+    params.push(filter.status);
+  }
+  if (filter?.session_id) {
+    conditions.push("session_id = ?");
+    params.push(filter.session_id);
+  }
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  const limit = filter?.limit ?? 20;
+  const offset = filter?.offset ?? 0;
+  const rows = d.query(`SELECT * FROM session_memory_jobs ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`).all(...params, limit, offset);
+  return rows.map(parseJobRow);
+}
+function updateSessionJob(id, updates, db) {
+  const d = db || getDatabase();
+  const setClauses = [];
+  const params = [];
+  if (updates.status !== undefined) {
+    setClauses.push("status = ?");
+    params.push(updates.status);
+  }
+  if (updates.chunk_count !== undefined) {
+    setClauses.push("chunk_count = ?");
+    params.push(updates.chunk_count);
+  }
+  if (updates.memories_extracted !== undefined) {
+    setClauses.push("memories_extracted = ?");
+    params.push(updates.memories_extracted);
+  }
+  if ("error" in updates) {
+    setClauses.push("error = ?");
+    params.push(updates.error ?? null);
+  }
+  if ("started_at" in updates) {
+    setClauses.push("started_at = ?");
+    params.push(updates.started_at ?? null);
+  }
+  if ("completed_at" in updates) {
+    setClauses.push("completed_at = ?");
+    params.push(updates.completed_at ?? null);
+  }
+  if (setClauses.length === 0)
+    return getSessionJob(id, d);
+  params.push(id);
+  d.run(`UPDATE session_memory_jobs SET ${setClauses.join(", ")} WHERE id = ?`, params);
+  return getSessionJob(id, d);
+}
+function getNextPendingJob(db) {
+  const d = db || getDatabase();
+  const row = d.query("SELECT * FROM session_memory_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1").get();
+  if (!row)
+    return null;
+  return parseJobRow(row);
+}
+
+// src/lib/session-queue.ts
+init_database();
+
+// src/lib/session-processor.ts
+init_memories();
+init_registry();
+var SESSION_EXTRACTION_USER_TEMPLATE = (chunk, sessionId) => `Extract memories from this session chunk (session: ${sessionId}):
+
+${chunk}
+
+Return JSON array: [{"key": "...", "value": "...", "category": "knowledge|fact|preference|history", "importance": 1-10, "tags": [...]}]`;
+function chunkTranscript(transcript, chunkSize = 2000, overlap = 200) {
+  if (!transcript || transcript.length === 0)
+    return [];
+  if (transcript.length <= chunkSize)
+    return [transcript];
+  const chunks = [];
+  let start = 0;
+  while (start < transcript.length) {
+    const end = Math.min(start + chunkSize, transcript.length);
+    chunks.push(transcript.slice(start, end));
+    if (end === transcript.length)
+      break;
+    start += chunkSize - overlap;
+  }
+  return chunks;
+}
+async function extractMemoriesFromChunk(chunk, context, db) {
+  const provider = providerRegistry.getAvailable();
+  if (!provider)
+    return 0;
+  try {
+    const extracted = await provider.extractMemories(SESSION_EXTRACTION_USER_TEMPLATE(chunk, context.sessionId), {
+      sessionId: context.sessionId,
+      agentId: context.agentId,
+      projectId: context.projectId
+    });
+    let savedCount = 0;
+    const sourceTag = context.source ? `source:${context.source}` : "source:manual";
+    for (const memory of extracted) {
+      if (!memory.content || !memory.content.trim())
+        continue;
+      try {
+        createMemory({
+          key: memory.content.slice(0, 120).replace(/\s+/g, "-").toLowerCase(),
+          value: memory.content,
+          category: memory.category,
+          scope: memory.suggestedScope ?? "shared",
+          importance: memory.importance,
+          tags: [
+            ...memory.tags,
+            "session-extracted",
+            sourceTag,
+            `session:${context.sessionId}`
+          ],
+          source: "auto",
+          agent_id: context.agentId,
+          project_id: context.projectId,
+          session_id: context.sessionId,
+          metadata: {
+            auto_extracted: true,
+            session_source: context.source ?? "manual",
+            extracted_at: new Date().toISOString(),
+            reasoning: memory.reasoning
+          }
+        }, "merge", db);
+        savedCount++;
+      } catch {}
+    }
+    return savedCount;
+  } catch {
+    try {
+      const fallbacks = providerRegistry.getFallbacks();
+      for (const fallback of fallbacks) {
+        try {
+          const extracted = await fallback.extractMemories(SESSION_EXTRACTION_USER_TEMPLATE(chunk, context.sessionId), {
+            sessionId: context.sessionId,
+            agentId: context.agentId,
+            projectId: context.projectId
+          });
+          let savedCount = 0;
+          const sourceTag = context.source ? `source:${context.source}` : "source:manual";
+          for (const memory of extracted) {
+            if (!memory.content || !memory.content.trim())
+              continue;
+            try {
+              createMemory({
+                key: memory.content.slice(0, 120).replace(/\s+/g, "-").toLowerCase(),
+                value: memory.content,
+                category: memory.category,
+                scope: memory.suggestedScope ?? "shared",
+                importance: memory.importance,
+                tags: [
+                  ...memory.tags,
+                  "session-extracted",
+                  sourceTag,
+                  `session:${context.sessionId}`
+                ],
+                source: "auto",
+                agent_id: context.agentId,
+                project_id: context.projectId,
+                session_id: context.sessionId,
+                metadata: {
+                  auto_extracted: true,
+                  session_source: context.source ?? "manual",
+                  extracted_at: new Date().toISOString(),
+                  reasoning: memory.reasoning
+                }
+              }, "merge", db);
+              savedCount++;
+            } catch {}
+          }
+          if (savedCount > 0)
+            return savedCount;
+        } catch {
+          continue;
+        }
+      }
+    } catch {}
+    return 0;
+  }
+}
+async function processSessionJob(jobId, db) {
+  const result = {
+    jobId,
+    chunksProcessed: 0,
+    memoriesExtracted: 0,
+    errors: []
+  };
+  let job;
+  try {
+    job = getSessionJob(jobId, db);
+    if (!job) {
+      result.errors.push(`Job not found: ${jobId}`);
+      return result;
+    }
+  } catch (e) {
+    result.errors.push(`Failed to fetch job: ${String(e)}`);
+    return result;
+  }
+  try {
+    updateSessionJob(jobId, { status: "processing", started_at: new Date().toISOString() }, db);
+  } catch (e) {
+    result.errors.push(`Failed to mark job as processing: ${String(e)}`);
+    return result;
+  }
+  const chunks = chunkTranscript(job.transcript);
+  try {
+    updateSessionJob(jobId, { chunk_count: chunks.length }, db);
+  } catch {}
+  let totalMemories = 0;
+  for (let i = 0;i < chunks.length; i++) {
+    const chunk = chunks[i];
+    try {
+      const count = await extractMemoriesFromChunk(chunk, {
+        sessionId: job.session_id,
+        agentId: job.agent_id ?? undefined,
+        projectId: job.project_id ?? undefined,
+        source: job.source
+      }, db);
+      totalMemories += count;
+      result.chunksProcessed++;
+    } catch (e) {
+      result.errors.push(`Chunk ${i} failed: ${String(e)}`);
+    }
+  }
+  result.memoriesExtracted = totalMemories;
+  try {
+    if (result.errors.length > 0 && result.chunksProcessed === 0) {
+      updateSessionJob(jobId, {
+        status: "failed",
+        error: result.errors.join("; "),
+        completed_at: new Date().toISOString(),
+        memories_extracted: totalMemories,
+        chunk_count: chunks.length
+      }, db);
+    } else {
+      updateSessionJob(jobId, {
+        status: "completed",
+        completed_at: new Date().toISOString(),
+        memories_extracted: totalMemories,
+        chunk_count: chunks.length
+      }, db);
+    }
+  } catch (e) {
+    result.errors.push(`Failed to update job status: ${String(e)}`);
+  }
+  return result;
+}
+
+// src/lib/session-queue.ts
+var _pendingQueue = new Set;
+var _isProcessing = false;
+var _workerStarted = false;
+function enqueueSessionJob(jobId) {
+  _pendingQueue.add(jobId);
+  if (!_isProcessing) {
+    _processNext();
+  }
+}
+function getSessionQueueStats() {
+  try {
+    const db = getDatabase();
+    const rows = db.query("SELECT status, COUNT(*) as count FROM session_memory_jobs GROUP BY status").all();
+    const stats = {
+      pending: 0,
+      processing: 0,
+      completed: 0,
+      failed: 0
+    };
+    for (const row of rows) {
+      if (row.status === "pending")
+        stats.pending = row.count;
+      else if (row.status === "processing")
+        stats.processing = row.count;
+      else if (row.status === "completed")
+        stats.completed = row.count;
+      else if (row.status === "failed")
+        stats.failed = row.count;
+    }
+    return stats;
+  } catch {
+    return {
+      pending: _pendingQueue.size,
+      processing: _isProcessing ? 1 : 0,
+      completed: 0,
+      failed: 0
+    };
+  }
+}
+function startSessionQueueWorker() {
+  if (_workerStarted)
+    return;
+  _workerStarted = true;
+  setInterval(() => {
+    _processNext();
+  }, 5000);
+}
+async function _processNext() {
+  if (_isProcessing)
+    return;
+  let jobId;
+  if (_pendingQueue.size > 0) {
+    jobId = [..._pendingQueue][0];
+    _pendingQueue.delete(jobId);
+  } else {
+    try {
+      const job = getNextPendingJob();
+      if (job)
+        jobId = job.id;
+    } catch {
+      return;
+    }
+  }
+  if (!jobId)
+    return;
+  _isProcessing = true;
+  try {
+    await processSessionJob(jobId);
+  } catch {} finally {
+    _isProcessing = false;
+    if (_pendingQueue.size > 0) {
+      _processNext();
+    }
+  }
+}
+
+// src/lib/session-auto-resolve.ts
+function autoResolveAgentProject(metadata, db) {
+  let agentId = null;
+  let projectId = null;
+  let confidence = "none";
+  const methods = [];
+  if (metadata.agentName) {
+    try {
+      const agent = getAgent(metadata.agentName, db);
+      if (agent) {
+        agentId = agent.id;
+        confidence = "high";
+        methods.push(`agent-by-name:${metadata.agentName}`);
+        if (agent.active_project_id && !projectId) {
+          projectId = agent.active_project_id;
+          methods.push("project-from-agent-active");
+        }
+      }
+    } catch {}
+  }
+  if (metadata.workingDir && !projectId) {
+    try {
+      const project = getProject(metadata.workingDir, db);
+      if (project) {
+        projectId = project.id;
+        if (confidence !== "high")
+          confidence = "high";
+        methods.push(`project-by-path:${metadata.workingDir}`);
+      } else {
+        const allProjects = listProjects(db);
+        for (const p of allProjects) {
+          if (p.path && metadata.workingDir.startsWith(p.path)) {
+            projectId = p.id;
+            if (confidence !== "high")
+              confidence = "high";
+            methods.push(`project-by-path-prefix:${p.path}`);
+            break;
+          }
+        }
+      }
+    } catch {}
+  }
+  if (metadata.gitRemote && !projectId) {
+    try {
+      const repoName = metadata.gitRemote.replace(/\.git$/, "").split(/[/:]/).filter(Boolean).pop();
+      if (repoName) {
+        const project = getProject(repoName, db);
+        if (project) {
+          projectId = project.id;
+          if (confidence === "none")
+            confidence = "low";
+          methods.push(`project-by-git-remote:${repoName}`);
+        }
+      }
+    } catch {}
+  }
+  if (methods.length === 0) {
+    confidence = "none";
+  }
+  return {
+    agentId,
+    projectId,
+    confidence,
+    method: methods.join(", ") || "none"
+  };
+}
+
+// src/server/index.ts
 var DEFAULT_PORT = 19428;
 function parsePort() {
   const envPort = process.env["PORT"];
@@ -5600,8 +6074,51 @@ addRoute("POST", "/api/synthesis/rollback/:run_id", async (_req, _url, params) =
   const result = await rollbackSynthesis(params["run_id"]);
   return json(result);
 });
+addRoute("POST", "/api/sessions/ingest", async (req) => {
+  const body = await readJson(req) ?? {};
+  const { transcript, session_id, agent_id, project_id, source, metadata } = body;
+  if (!transcript || typeof transcript !== "string")
+    return errorResponse("transcript is required", 400);
+  if (!session_id || typeof session_id !== "string")
+    return errorResponse("session_id is required", 400);
+  let resolvedAgentId = agent_id;
+  let resolvedProjectId = project_id;
+  if (!resolvedAgentId || !resolvedProjectId) {
+    const resolved = autoResolveAgentProject(metadata ?? {});
+    if (!resolvedAgentId && resolved.agentId)
+      resolvedAgentId = resolved.agentId;
+    if (!resolvedProjectId && resolved.projectId)
+      resolvedProjectId = resolved.projectId;
+  }
+  const job = createSessionJob({
+    session_id,
+    transcript,
+    source: source ?? "manual",
+    agent_id: resolvedAgentId,
+    project_id: resolvedProjectId,
+    metadata: metadata ?? {}
+  });
+  enqueueSessionJob(job.id);
+  return json({ job_id: job.id, status: "queued", message: "Session queued for memory extraction" }, 202);
+});
+addRoute("GET", "/api/sessions/jobs", (_req, url) => {
+  const agentId = url.searchParams.get("agent_id") ?? undefined;
+  const projectId = url.searchParams.get("project_id") ?? undefined;
+  const status = url.searchParams.get("status") ?? undefined;
+  const limit = url.searchParams.get("limit") ? parseInt(url.searchParams.get("limit")) : 20;
+  const jobs = listSessionJobs({ agent_id: agentId, project_id: projectId, status, limit });
+  return json({ jobs, count: jobs.length });
+});
+addRoute("GET", "/api/sessions/jobs/:id", (_req, _url, params) => {
+  const job = getSessionJob(params["id"]);
+  if (!job)
+    return errorResponse("Session job not found", 404);
+  return json(job);
+});
+addRoute("GET", "/api/sessions/queue/stats", () => json(getSessionQueueStats()));
 function startServer(port) {
   loadWebhooksFromDb();
+  startSessionQueueWorker();
   const hostname = process.env["MEMENTOS_HOST"] ?? "127.0.0.1";
   Bun.serve({
     port,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/mementos",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Universal memory system for AI agents - CLI + MCP server + library API",
   "type": "module",
   "main": "dist/index.js",