onbuzz 5.0.2 → 5.3.0

package/src/core/toolResultOffload.js

@@ -0,0 +1,152 @@
+/**
+ * Generic large-tool-output offload.
+ *
+ * The web tool already externalizes large fetched pages into the per-agent
+ * DocumentStore (see webTool._maybeOffload) so a 200 KB article never enters
+ * the conversation. But EVERY other tool — terminal, filesystem, office,
+ * desktop/foundry, etc. — still dumps its raw result verbatim into the
+ * `[Tool Results — …]` feedback message (agentScheduler.processAgentQueues →
+ * formatToolResult). A single `cat` of a big file or a chatty build log can
+ * add tens of thousands of tokens that then ride along in context every turn
+ * and force premature compaction.
+ *
+ * This module applies the SAME externalize-and-handle treatment generically,
+ * at the one chokepoint all tool results pass through. For any completed result
+ * whose serialized body exceeds a threshold, it persists the full body to the
+ * agent's DocumentStore and replaces the body with a compact preview + a
+ * recall handle. The agent can recall passages semantically or read exact
+ * lines via code-map — without the full body surviving in the conversation.
+ *
+ * Web results that the web tool already externalized are skipped (no double
+ * offload). Failed/short results are left untouched.
+ */
+
+import { structuralPreview } from './structuralPreview.js';
+
+/** Default body size (chars) above which a tool result is externalized. */
+export const DEFAULT_TOOL_OUTPUT_OFFLOAD_THRESHOLD = 6000;
+/** How many leading chars of the body to keep inline as a preview. */
+export const DEFAULT_TOOL_OUTPUT_PREVIEW_CHARS = 1500;
+
+/**
+ * Reproduce the large-content branches of agentScheduler.formatToolResult so
+ * the threshold reflects the actual text cost the body would add to context.
+ * Only `completed` results carry potentially-large bodies; failed/other
+ * statuses render to short strings and are never worth externalizing.
+ *
+ * @param {object} toolResult
+ * @returns {string|null} the serialized body, or null if not offload-eligible.
+ */
+export function serializeToolResultBody(toolResult) {
+  if (!toolResult || toolResult.status !== 'completed') return null;
+  const r = toolResult.result;
+  if (r === undefined || r === null) return null;
+  if (typeof r === 'object') {
+    try {
+      return JSON.stringify(r, null, 2);
+    } catch {
+      return null; // circular / non-serializable → leave it to formatToolResult
+    }
+  }
+  return String(r);
+}
+
+/**
+ * Externalize a single large tool result in place. No-op (returns unchanged)
+ * when there's no document memory / agent context, the body is below
+ * threshold, the result was already externalized by its producing tool, or
+ * persistence fails. Mutates `toolResult.result` to a compact handle when it
+ * does externalize.
+ *
+ * @param {object} args
+ * @param {object} args.documentMemory  DocumentMemory instance (remember()).
+ * @param {string} args.agentId
+ * @param {object} args.toolResult       Mutated in place when externalized.
+ * @param {number} [args.threshold]
+ * @param {number} [args.previewChars]
+ * @returns {Promise<{externalized: boolean, toolResult: object}>}
+ */
+export async function maybeOffloadToolResult({
+  documentMemory,
+  agentId,
+  toolResult,
+  threshold = DEFAULT_TOOL_OUTPUT_OFFLOAD_THRESHOLD,
+  previewChars = DEFAULT_TOOL_OUTPUT_PREVIEW_CHARS,
+} = {}) {
+  if (!documentMemory || !agentId || !toolResult) {
+    return { externalized: false, toolResult };
+  }
+
+  // Skip results a producing tool already externalized (e.g. webTool sets
+  // contentExternalized; our own handle sets toolOutputExternalized).
+  const existing = toolResult.result;
+  if (
+    existing &&
+    typeof existing === 'object' &&
+    (existing.contentExternalized || existing.toolOutputExternalized)
+  ) {
+    return { externalized: false, toolResult };
+  }
+
+  const body = serializeToolResultBody(toolResult);
+  if (!body || body.length < threshold) {
+    return { externalized: false, toolResult };
+  }
+  const original = toolResult.result;
+
+  const toolId = toolResult.toolId || 'tool';
+  const source = `tool://${toolId}/${toolResult.id || 'result'}`;
+
+  let handle;
+  try {
+    handle = await documentMemory.remember(agentId, {
+      source,
+      title: `${toolId} output`,
+      content: body,
+    });
+  } catch {
+    // Best-effort: a persistence failure must never drop the result — keep it
+    // inline so the agent still sees its output this turn.
+    return { externalized: false, toolResult };
+  }
+
+  // Build the inline preview. For STRUCTURED results, a schema-agnostic
+  // structural preview (caps arrays/strings/keys, preserves the skeleton) is
+  // far more representative than the head-of-JSON slice — it shows every field
+  // and a sample of items for ANY tool/shape. Scalar/string results keep the
+  // leading-chars slice. The full body is offloaded either way (lossless).
+  let preview;
+  let resultsTruncated;
+  if (original && typeof original === 'object') {
+    const sp = structuralPreview(original);
+    preview = sp.preview;
+    resultsTruncated = sp.truncated;
+  } else {
+    preview = body.slice(0, previewChars);
+    resultsTruncated = body.length > previewChars;
+  }
+  const previewChars2 = typeof preview === 'string' ? preview.length : JSON.stringify(preview).length;
+
+  toolResult.result = {
+    toolOutputExternalized: true,
+    tool: toolId,
+    preview,
+    resultsTruncated,
+    omittedChars: Math.max(0, body.length - previewChars2),
+    document: {
+      docId: handle.docId,
+      lines: handle.lines,
+      bytes: handle.bytes,
+      chunkCount: handle.chunkCount,
+      indexed: handle.indexed,
+      embeddingsEnabled: handle.embeddingsEnabled,
+    },
+    recallHint: handle.embeddingsEnabled
+      ? `Full ${toolId} output saved as document ${handle.docId} (${handle.lines} lines). Recall relevant passages semantically over your saved documents, or read exact lines with code-map read-range on the saved file.`
+      : `Full ${toolId} output saved as document ${handle.docId} (${handle.lines} lines); semantic recall is OFF (set an embeddings provider in Settings to enable search). You can still read it with code-map read-range.`,
+  };
+
+  return { externalized: true, toolResult };
+}
+
+export default { serializeToolResultBody, maybeOffloadToolResult };

package/src/index.js

@@ -50,8 +50,6 @@ import WebTool from './tools/webTool.js';
 import VisualEditorTool from './tools/visualEditorTool.js';
 import PdfTool from './tools/pdfTool.js';
 import HelpTool from './tools/helpTool.js';
-import DocxTool from './tools/docxTool.js';
-import ExcelTool from './tools/excelTool.js';
 import OfficeTool from './tools/officeTool.js';
 import MemoryTool from './tools/memoryTool.js';
 import ComposioTool from './tools/composioTool.js';
@@ -65,6 +63,8 @@ import { DesktopTool } from './tools/desktop/DesktopTool.js';
 import ScheduleService from './services/scheduleService.js';
 import AsyncToolManager from './tools/asyncToolManager.js';
 import WebServer from './interfaces/webServer.js';
+import { wireMemoryActivityBroadcast } from './interfaces/memoryStatusRoutes.js';
+import { wireMonitorBroadcast } from './interfaces/monitorRoutes.js';
 
 import {
   SYSTEM_VERSION,
@@ -353,6 +353,22 @@ class LoxiaApplication {
       provider: this.embeddingService.config.provider,
       enabled: this.embeddingService.isEnabled,
     });
+    // Pre-load the local embedding model in the background so the user never
+    // has to click "pre-load model". Fire-and-forget — does not block startup.
+    this.embeddingService.warmupInBackground();
+
+    // Give the shared MemoryService a memory indexer so EVERY memory write
+    // (agent tool, Settings/UI add, etc.) keeps the vector surface in sync, and
+    // it can reconcile any pre-existing un-indexed memories on first load.
+    try {
+      const { getMemoryService } = await import('./services/memoryService.js');
+      const { MemoryIndexer } = await import('./services/embeddings/indexers/memoryIndexer.js');
+      getMemoryService(this.logger).setMemoryIndexer(
+        new MemoryIndexer({ embeddingService: this.embeddingService, logger: this.logger }),
+      );
+    } catch (err) {
+      this.logger.warn('[Loxia] Failed to wire memory indexer', { error: err?.message });
+    }
     
     // Initialize Model Routing Services
     this.benchmarkService = new BenchmarkService(this.config, this.logger);
@@ -502,16 +518,11 @@ class LoxiaApplication {
     // Register Help Tool (two-layer tool description system)
     await this.toolsRegistry.registerTool(HelpTool);
 
-    // Register Document (DOCX) Tool — legacy; superseded by OfficeTool.
-    // Kept for one release cycle so existing agent configs that reference
-    // toolId "doc" continue to work.
-    await this.toolsRegistry.registerTool(DocxTool);
-
-    // Register Spreadsheet (Excel) Tool — legacy; superseded by OfficeTool.
-    await this.toolsRegistry.registerTool(ExcelTool);
-
     // Register Office Tool — consolidated doc/sheet/pres surface
-    // (Word + Excel + PowerPoint create/read).
+    // (Word + Excel + PowerPoint create/read). Supersedes the former
+    // standalone `doc` (DOCX) and `spreadsheet` (Excel) tools, now removed;
+    // old agent configs referencing those toolIds are migrated to `office`
+    // by agentPool.sanitizeCapabilities.
     await this.toolsRegistry.registerTool(OfficeTool);
 
     // Register Memory Tool
@@ -779,6 +790,21 @@ class LoxiaApplication {
       // read config + telemetry, hot-swap providers, etc.
       webServer.embeddingService = this.embeddingService;
 
+      // Rebroadcast live per-turn auto-recall telemetry to web-ui clients
+      // (the memory-path Health/Activity strip). Best-effort; piggybacks the
+      // existing websocket and emits only data auto-recall already computed.
+      wireMemoryActivityBroadcast({
+        embeddingService: this.embeddingService,
+        broadcast: (message) => webServer.broadcastToSession(null, message),
+        logger: this.logger,
+      });
+
+      // Live-push new warn/error log entries to the Monitor App.
+      wireMonitorBroadcast({
+        logger: this.logger,
+        broadcast: (message) => webServer.broadcastToSession(null, message),
+      });
+
       // Set Credential Vault reference in Web Server
       webServer.setCredentialVault(this.credentialVault);
 

package/src/interfaces/__tests__/attachmentRoutes.test.js

@@ -0,0 +1,118 @@
+import { jest, describe, it, expect, beforeEach, afterEach } from '@jest/globals';
+import express from 'express';
+import { createServer } from 'http';
+import { registerAttachmentRoutes } from '../attachmentRoutes.js';
+
+async function post(url, body) {
+  const resp = await globalThis.fetch(url, {
+    method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body || {}),
+  });
+  const ct = resp.headers.get('content-type') || '';
+  return { status: resp.status, body: ct.includes('json') ? await resp.json() : await resp.text() };
+}
+
+function startApp(getIngestionService) {
+  const app = express();
+  // NOTE: deliberately NO global express.json — the route mounts its own parser.
+  registerAttachmentRoutes(app, { getIngestionService, logger: { error: jest.fn() } });
+  const server = createServer(app);
+  return new Promise((r) => server.listen(0, () => r({ server, baseUrl: `http://localhost:${server.address().port}` })));
+}
+
+let ingestion;
+beforeEach(() => {
+  ingestion = {
+    ingestUpload: jest.fn().mockResolvedValue({ fileId: 'f1', fileName: 'pic.png' }),
+    ingestServerPath: jest.fn().mockResolvedValue({ fileId: 'f2', fileName: 'doc.pdf' }),
+  };
+});
+
+describe('POST /api/agents/:id/attachments', () => {
+  let server, baseUrl;
+  beforeEach(async () => { ({ server, baseUrl } = await startApp(() => ingestion)); });
+  afterEach(() => server?.close());
+
+  it('routes a base64 upload to ingestUpload', async () => {
+    const r = await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'pic.png', dataBase64: 'AAAA', mode: 'content' });
+    expect(r.status).toBe(200);
+    expect(r.body).toEqual({ success: true, attachment: { fileId: 'f1', fileName: 'pic.png' } });
+    expect(ingestion.ingestUpload).toHaveBeenCalledWith(expect.objectContaining({ agentId: 'a1', fileName: 'pic.png', dataBase64: 'AAAA', mode: 'content' }));
+    expect(ingestion.ingestServerPath).not.toHaveBeenCalled();
+  });
+
+  it('routes a server filePath to ingestServerPath', async () => {
+    const r = await post(`${baseUrl}/api/agents/a1/attachments`, { filePath: '/srv/doc.pdf', fileName: 'doc.pdf' });
+    expect(r.status).toBe(200);
+    expect(r.body.attachment.fileId).toBe('f2');
+    expect(ingestion.ingestServerPath).toHaveBeenCalledWith(expect.objectContaining({ agentId: 'a1', filePath: '/srv/doc.pdf', fileName: 'doc.pdf' }));
+  });
+
+  it('forwards scope + conversationId to ingestion', async () => {
+    await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'x', dataBase64: 'AA', scope: 'conversation', conversationId: 'c1' });
+    expect(ingestion.ingestUpload).toHaveBeenCalledWith(expect.objectContaining({ scope: 'conversation', conversationId: 'c1' }));
+  });
+
+  it('prefers dataBase64 over filePath when both are present', async () => {
+    await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'x', dataBase64: 'AA', filePath: '/srv/x' });
+    expect(ingestion.ingestUpload).toHaveBeenCalledTimes(1);
+    expect(ingestion.ingestServerPath).not.toHaveBeenCalled();
+  });
+
+  it('400s when neither source is provided', async () => {
+    const r = await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'x' });
+    expect(r.status).toBe(400);
+    expect(r.body.error).toMatch(/Provide dataBase64.*or filePath/);
+  });
+
+  it('400s with the error message when ingestion throws (validation)', async () => {
+    ingestion.ingestUpload.mockRejectedValueOnce(new Error('Executable files (.exe) are not allowed'));
+    const r = await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'm.exe', dataBase64: 'AA' });
+    expect(r.status).toBe(400);
+    expect(r.body).toMatchObject({ success: false, error: /not allowed/ });
+  });
+});
+
+describe('service-absent + defensive', () => {
+  it('503 when the ingestion service is not wired yet (boot ordering)', async () => {
+    const { server, baseUrl } = await startApp(() => null);
+    try {
+      const r = await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'x', dataBase64: 'AA' });
+      expect(r.status).toBe(503);
+    } finally { server.close(); }
+  });
+
+  it('503 when no thunk is supplied at all', async () => {
+    const { server, baseUrl } = await startApp(undefined);
+    try {
+      expect((await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'x', dataBase64: 'AA' })).status).toBe(503);
+    } finally { server.close(); }
+  });
+
+  it('no-ops when app is null (with and without an options object)', () => {
+    expect(() => registerAttachmentRoutes(null, {})).not.toThrow();
+    expect(() => registerAttachmentRoutes(undefined)).not.toThrow(); // default {} options arm
+  });
+
+  it('handles a request whose body the JSON parser left unset (non-JSON content-type)', async () => {
+    const { server, baseUrl } = await startApp(() => ingestion);
+    try {
+      const resp = await globalThis.fetch(`${baseUrl}/api/agents/a1/attachments`, {
+        method: 'POST', headers: { 'content-type': 'text/plain' }, body: 'not json',
+      });
+      expect(resp.status).toBe(400); // req.body || {} → no source → 400, no crash
+    } finally { server.close(); }
+  });
+
+  it('works with no logger supplied (uses the no-op fallback) on the error path', async () => {
+    const ing = { ingestUpload: jest.fn().mockRejectedValue(new Error('boom')), ingestServerPath: jest.fn() };
+    const app = express();
+    registerAttachmentRoutes(app, { getIngestionService: () => ing }); // no logger
+    const server = createServer(app);
+    await new Promise((r) => server.listen(0, r));
+    const baseUrl = `http://localhost:${server.address().port}`;
+    try {
+      const r = await post(`${baseUrl}/api/agents/a1/attachments`, { fileName: 'x', dataBase64: 'AA' });
+      expect(r.status).toBe(400); // error path ran through the no-op logger without throwing
+    } finally { server.close(); }
+  });
+});

package/src/interfaces/__tests__/memoryStatusRoutes.test.js

@@ -0,0 +1,161 @@
+/**
+ * Tests for memoryStatusRoutes — the live per-agent memory-path view.
+ *
+ * Pure DI: the route registrar is exercised by capturing the handler off a
+ * fake `app.get` and invoking it with mock req/res; the broadcast wiring is
+ * exercised with a real EventEmitter + a capturing broadcast fn.
+ */
+import { describe, test, expect, jest } from '@jest/globals';
+import { EventEmitter } from 'events';
+import { registerMemoryStatusRoute, wireMemoryActivityBroadcast } from '../memoryStatusRoutes.js';
+
+/** Capture the single GET handler the registrar installs. */
+function mountRoute(deps) {
+  let handler = null;
+  const app = { get: jest.fn((path, h) => { if (path.endsWith('/memory-status')) handler = h; }) };
+  registerMemoryStatusRoute(app, deps);
+  return { app, invoke: (agentId) => handler({ params: { agentId } }, makeRes()) };
+}
+
+function makeRes() {
+  const res = {
+    statusCode: 200,
+    body: null,
+    status(code) { this.statusCode = code; return this; },
+    json(payload) { this.body = payload; return this; },
+  };
+  return res;
+}
+
+describe('registerMemoryStatusRoute', () => {
+  test('registers GET /api/agents/:agentId/memory-status', () => {
+    const { app } = mountRoute({ getEmbeddingService: () => null });
+    expect(app.get).toHaveBeenCalledWith('/api/agents/:agentId/memory-status', expect.any(Function));
+  });
+
+  test('returns disabled shape when no embedding service attached', async () => {
+    const res = makeRes();
+    let handler;
+    const app = { get: (_p, h) => { handler = h; } };
+    registerMemoryStatusRoute(app, { getEmbeddingService: () => null });
+    await handler({ params: { agentId: 'a1' } }, res);
+    expect(res.body).toEqual({ embeddings: { enabled: false }, surfaces: {}, lastRecall: null });
+  });
+
+  test('returns disabled shape when service lacks getMemoryStatus', async () => {
+    const res = makeRes();
+    let handler;
+    registerMemoryStatusRoute({ get: (_p, h) => { handler = h; } }, { getEmbeddingService: () => ({}) });
+    await handler({ params: { agentId: 'a1' } }, res);
+    expect(res.body.embeddings.enabled).toBe(false);
+  });
+
+  test('delegates to embeddingService.getMemoryStatus with the agentId', async () => {
+    const status = { embeddings: { enabled: true }, surfaces: { memory: 3 }, lastRecall: null };
+    const svc = { getMemoryStatus: jest.fn(async () => status) };
+    const res = makeRes();
+    let handler;
+    registerMemoryStatusRoute({ get: (_p, h) => { handler = h; } }, { getEmbeddingService: () => svc });
+    await handler({ params: { agentId: 'a7' } }, res);
+    expect(svc.getMemoryStatus).toHaveBeenCalledWith('a7');
+    expect(res.body).toBe(status);
+  });
+
+  test('500 + logs when getMemoryStatus throws', async () => {
+    const logger = { error: jest.fn() };
+    const svc = { getMemoryStatus: async () => { throw new Error('kaboom'); } };
+    const res = makeRes();
+    let handler;
+    registerMemoryStatusRoute({ get: (_p, h) => { handler = h; } }, { getEmbeddingService: () => svc, logger });
+    await handler({ params: { agentId: 'a1' } }, res);
+    expect(res.statusCode).toBe(500);
+    expect(res.body).toEqual({ error: 'kaboom' });
+    expect(logger.error).toHaveBeenCalled();
+  });
+
+  test('non-function getEmbeddingService is tolerated (disabled shape)', async () => {
+    const res = makeRes();
+    let handler;
+    registerMemoryStatusRoute({ get: (_p, h) => { handler = h; } }, {});
+    await handler({ params: { agentId: 'a1' } }, res);
+    expect(res.body.embeddings.enabled).toBe(false);
+  });
+
+  test('registrar with no deps arg does not throw (disabled shape)', async () => {
+    const res = makeRes();
+    let handler;
+    registerMemoryStatusRoute({ get: (_p, h) => { handler = h; } }); // no deps
+    await handler({ params: { agentId: 'a1' } }, res);
+    expect(res.body.embeddings.enabled).toBe(false);
+  });
+});
+
+describe('wireMemoryActivityBroadcast', () => {
+  const flush = () => new Promise((r) => setImmediate(r));
+
+  test('does NOT broadcast synchronously (deferred off the emit stack)', () => {
+    const embeddingService = { memoryActivity: new EventEmitter() };
+    const sent = [];
+    wireMemoryActivityBroadcast({ embeddingService, broadcast: (m) => sent.push(m) });
+    embeddingService.memoryActivity.emit('recall', { agentId: 'a1', injected: 2 });
+    expect(sent).toHaveLength(0); // not yet — emit returned immediately
+  });
+
+  test('rebroadcasts recall entries as memory:recall (deferred)', async () => {
+    const embeddingService = { memoryActivity: new EventEmitter() };
+    const sent = [];
+    const off = wireMemoryActivityBroadcast({ embeddingService, broadcast: (m) => sent.push(m) });
+
+    embeddingService.memoryActivity.emit('recall', { agentId: 'a1', injected: 2 });
+    await flush();
+    expect(sent).toEqual([{ type: 'memory:recall', data: { agentId: 'a1', injected: 2 } }]);
+
+    off(); // unsubscribe stops further delivery
+    embeddingService.memoryActivity.emit('recall', { agentId: 'a1', injected: 9 });
+    await flush();
+    expect(sent).toHaveLength(1);
+  });
+
+  test('no-op (returns a safe fn) when there is no emitter', () => {
+    const off = wireMemoryActivityBroadcast({ embeddingService: {}, broadcast: () => {} });
+    expect(typeof off).toBe('function');
+    expect(() => off()).not.toThrow();
+  });
+
+  test('no-op when broadcast is not a function', () => {
+    const embeddingService = { memoryActivity: new EventEmitter() };
+    const off = wireMemoryActivityBroadcast({ embeddingService, broadcast: null });
+    embeddingService.memoryActivity.emit('recall', { agentId: 'a1' }); // must not throw
+    expect(typeof off).toBe('function');
+  });
+
+  test('a throwing broadcast is swallowed and logged (deferred)', async () => {
+    const embeddingService = { memoryActivity: new EventEmitter() };
+    const logger = { warn: jest.fn() };
+    wireMemoryActivityBroadcast({ embeddingService, broadcast: () => { throw new Error('ws down'); }, logger });
+    embeddingService.memoryActivity.emit('recall', { agentId: 'a1' });
+    await flush();
+    expect(logger.warn).toHaveBeenCalled();
+  });
+
+  test('called with no args returns a safe no-op', () => {
+    const off = wireMemoryActivityBroadcast();
+    expect(typeof off).toBe('function');
+    expect(() => off()).not.toThrow();
+  });
+
+  test('falls back to setTimeout when setImmediate is unavailable', async () => {
+    const orig = global.setImmediate;
+    global.setImmediate = undefined;
+    try {
+      const embeddingService = { memoryActivity: new EventEmitter() };
+      const sent = [];
+      wireMemoryActivityBroadcast({ embeddingService, broadcast: (m) => sent.push(m) });
+      embeddingService.memoryActivity.emit('recall', { agentId: 'a1' });
+      await new Promise((r) => setTimeout(r, 0));
+      expect(sent).toHaveLength(1);
+    } finally {
+      global.setImmediate = orig;
+    }
+  });
+});