neoagent 3.0.1-beta.9 → 3.1.0

package/server/routes/agents.js

@@ -10,6 +10,154 @@ const { buildAgentRunContext } = require('./_helpers/agentRunContext');
 
 router.use(requireAuth);
 
+const CHAT_HISTORY_DEFAULT_LIMIT = 40;
+const CHAT_HISTORY_MAX_LIMIT = 100;
+
+function normalizedTimestampExpression(valueSql) {
+  return `julianday(CASE WHEN instr(${valueSql}, 'T') > 0 THEN ${valueSql} ELSE replace(${valueSql}, ' ', 'T') || 'Z' END)`;
+}
+
+function normalizeChatHistoryLimit(rawLimit) {
+  const parsed = Number.parseInt(rawLimit, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return CHAT_HISTORY_DEFAULT_LIMIT;
+  }
+  return Math.min(parsed, CHAT_HISTORY_MAX_LIMIT);
+}
+
+function parseChatHistoryCursor(query) {
+  const createdAt = query.beforeCreatedAt?.toString().trim() || '';
+  const source = query.beforeSource?.toString().trim() || '';
+  const id = query.beforeId?.toString().trim() || '';
+  if (!createdAt || !source || !id) {
+    return null;
+  }
+  return { createdAt, source, id };
+}
+
+function chatHistoryTimestampMs(value) {
+  if (!value) return 0;
+  const normalized = value.includes('T') ? value : `${value.replace(' ', 'T')}Z`;
+  const parsed = new Date(normalized).getTime();
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function compareChatHistoryDesc(left, right) {
+  const timestampDiff =
+    chatHistoryTimestampMs(right.created_at) - chatHistoryTimestampMs(left.created_at);
+  if (timestampDiff !== 0) {
+    return timestampDiff;
+  }
+  const sourceDiff = String(right.history_source || '').localeCompare(
+    String(left.history_source || '')
+  );
+  if (sourceDiff !== 0) {
+    return sourceDiff;
+  }
+  return String(right.id || '').localeCompare(String(left.id || ''));
+}
+
+function buildRunUsageSummary(runId) {
+  const rows = db.prepare(`
+    SELECT
+      provider,
+      model,
+      phase,
+      input_tokens,
+      output_tokens,
+      reasoning_tokens,
+      cached_read_tokens,
+      cache_write_tokens,
+      total_tokens,
+      estimated_cost_usd,
+      latency_ms
+    FROM agent_model_usage
+    WHERE run_id = ?
+    ORDER BY id ASC
+  `).all(runId);
+  const totals = {
+    inputTokens: 0,
+    outputTokens: 0,
+    reasoningTokens: 0,
+    cachedReadTokens: 0,
+    cacheWriteTokens: 0,
+    totalTokens: 0,
+    estimatedCostUsd: 0,
+    pricedCallCount: 0,
+    latencyMs: 0,
+  };
+  const models = new Map();
+
+  for (const row of rows) {
+    const inputTokens = Number(row.input_tokens || 0);
+    const outputTokens = Number(row.output_tokens || 0);
+    const reasoningTokens = Number(row.reasoning_tokens || 0);
+    const cachedReadTokens = Number(row.cached_read_tokens || 0);
+    const cacheWriteTokens = Number(row.cache_write_tokens || 0);
+    const totalTokens = Number(row.total_tokens || 0);
+    const latencyMs = Number(row.latency_ms || 0);
+    const estimatedCostUsd = Number(row.estimated_cost_usd);
+    const key = `${row.provider}:${row.model}`;
+
+    totals.inputTokens += inputTokens;
+    totals.outputTokens += outputTokens;
+    totals.reasoningTokens += reasoningTokens;
+    totals.cachedReadTokens += cachedReadTokens;
+    totals.cacheWriteTokens += cacheWriteTokens;
+    totals.totalTokens += totalTokens;
+    totals.latencyMs += latencyMs;
+    if (Number.isFinite(estimatedCostUsd)) {
+      totals.estimatedCostUsd += estimatedCostUsd;
+      totals.pricedCallCount += 1;
+    }
+
+    if (!models.has(key)) {
+      models.set(key, {
+        provider: row.provider,
+        model: row.model,
+        callCount: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        reasoningTokens: 0,
+        cachedReadTokens: 0,
+        cacheWriteTokens: 0,
+        totalTokens: 0,
+        estimatedCostUsd: 0,
+        pricedCallCount: 0,
+        latencyMs: 0,
+        phases: new Set(),
+      });
+    }
+
+    const aggregate = models.get(key);
+    aggregate.callCount += 1;
+    aggregate.inputTokens += inputTokens;
+    aggregate.outputTokens += outputTokens;
+    aggregate.reasoningTokens += reasoningTokens;
+    aggregate.cachedReadTokens += cachedReadTokens;
+    aggregate.cacheWriteTokens += cacheWriteTokens;
+    aggregate.totalTokens += totalTokens;
+    aggregate.latencyMs += latencyMs;
+    aggregate.phases.add(String(row.phase || '').trim() || 'model_turn');
+    if (Number.isFinite(estimatedCostUsd)) {
+      aggregate.estimatedCostUsd += estimatedCostUsd;
+      aggregate.pricedCallCount += 1;
+    }
+  }
+
+  return {
+    totals: {
+      ...totals,
+      estimatedCostUsd: totals.pricedCallCount > 0 ? totals.estimatedCostUsd : null,
+    },
+    models: [...models.values()].map((entry) => ({
+      ...entry,
+      estimatedCostUsd: entry.pricedCallCount > 0 ? entry.estimatedCostUsd : null,
+      phases: [...entry.phases],
+    })),
+  };
+}
+
 // List agent runs
 router.get('/', (req, res) => {
   const limit = Math.min(parseInt(req.query.limit) || 50, 200);
@@ -23,9 +171,31 @@ router.get('/', (req, res) => {
 
 // Chat history (web + social messages merged)
 router.get('/chat-history', (req, res) => {
-  const limit = Math.min(parseInt(req.query.limit) || 100, 500);
+  const limit = normalizeChatHistoryLimit(req.query.limit);
+  const queryLimit = limit + 1;
   const userId = req.session.userId;
   const agentId = resolveAgentId(userId, getAgentIdFromRequest(req));
+  const cursor = parseChatHistoryCursor(req.query);
+  const cursorTimestampSql = normalizedTimestampExpression('@beforeCreatedAt');
+  const historyTimestampSql = normalizedTimestampExpression('created_at');
+  const cursorClause = (sourceName) =>
+    cursor
+      ? `AND (
+        ${historyTimestampSql} < ${cursorTimestampSql}
+        OR (${historyTimestampSql} = ${cursorTimestampSql} AND '${sourceName}' < @beforeSource)
+        OR (${historyTimestampSql} = ${cursorTimestampSql} AND '${sourceName}' = @beforeSource AND CAST(id AS TEXT) < @beforeId)
+      )`
+      : '';
+  const queryParams = cursor
+    ? {
+        userId,
+        agentId,
+        limit: queryLimit,
+        beforeCreatedAt: cursor.createdAt,
+        beforeSource: cursor.source,
+        beforeId: cursor.id,
+      }
+    : { userId, agentId, limit: queryLimit };
 
   const webMsgs = db.prepare(`
     SELECT
@@ -37,9 +207,14 @@ router.get('/chat-history', (req, res) => {
       created_at,
       agent_run_id AS run_id,
       metadata,
-      NULL AS tool_calls
-    FROM conversation_history WHERE user_id = ? AND agent_id = ? ORDER BY created_at DESC LIMIT ?
-  `).all(userId, agentId, limit);
+      NULL AS tool_calls,
+      'conversation' AS history_source
+    FROM conversation_history
+    WHERE user_id = @userId AND agent_id = @agentId
+    ${cursorClause('conversation')}
+    ORDER BY ${historyTimestampSql} DESC, CAST(id AS TEXT) DESC
+    LIMIT @limit
+  `).all(queryParams);
 
   const socialMsgs = db.prepare(`
     SELECT
@@ -51,23 +226,27 @@ router.get('/chat-history', (req, res) => {
       created_at,
       run_id,
       metadata,
-      tool_calls
-    FROM messages WHERE user_id = ? AND agent_id = ? AND platform != 'web'
-    ORDER BY created_at DESC LIMIT ?
-  `).all(userId, agentId, limit);
-
-  // Normalize SQL datetime ('YYYY-MM-DD HH:MM:SS', treated as local) and ISO-Z strings to ms
-  const toMs = (s) => {
-    if (!s) return 0;
-    const normalized = s.includes('T') ? s : s.replace(' ', 'T') + 'Z';
-    return new Date(normalized).getTime();
-  };
+      tool_calls,
+      'message' AS history_source
+    FROM messages
+    WHERE user_id = @userId AND agent_id = @agentId AND platform != 'web'
+    ${cursorClause('message')}
+    ORDER BY ${historyTimestampSql} DESC, CAST(id AS TEXT) DESC
+    LIMIT @limit
+  `).all(queryParams);
 
-  const all = [...webMsgs, ...socialMsgs]
-    .sort((a, b) => toMs(a.created_at) - toMs(b.created_at))
-    .slice(-limit);
+  const merged = [...webMsgs, ...socialMsgs].sort(compareChatHistoryDesc);
+  const page = merged.slice(0, limit);
+  const oldest = page.length > 0 ? page[page.length - 1] : null;
 
-  res.json({ messages: all, agentId });
+  res.json({
+    messages: [...page].reverse(),
+    agentId,
+    hasMore: merged.length > limit,
+    nextBeforeCreatedAt: oldest?.created_at || null,
+    nextBeforeSource: oldest?.history_source || null,
+    nextBeforeId: oldest?.id?.toString() || null,
+  });
 });
 
 // Create new agent run
@@ -130,6 +309,7 @@ router.post('/', async (req, res) => {
 
     res.json(result);
   } catch (err) {
+    console.error('[Agents] Run failed:', err?.stack || err);
     res.status(err?.statusCode || err?.status || 500).json({
       error: sanitizeError(err),
       code: err?.code,
@@ -145,8 +325,9 @@ router.get('/:id', (req, res) => {
 
   const steps = db.prepare('SELECT * FROM agent_steps WHERE run_id = ? ORDER BY step_index ASC').all(run.id);
   const history = db.prepare('SELECT * FROM conversation_history WHERE agent_run_id = ? ORDER BY created_at ASC').all(run.id);
+  const usage = buildRunUsageSummary(run.id);
 
-  res.json({ run, steps, history });
+  res.json({ run, steps, history, usage });
 });
 
 // Get detailed steps for a run (for activity history replay)
@@ -180,8 +361,9 @@ router.get('/:id/steps', (req, res) => {
     || latestHistoryAssistant?.content
     || run.final_response
     || null;
+  const usage = buildRunUsageSummary(run.id);
 
-  res.json({ run, steps, events: listRunEvents(run.id), response });
+  res.json({ run, steps, events: listRunEvents(run.id), response, usage });
 });
 
 // Abort a run

package/server/routes/billing.js

@@ -78,6 +78,11 @@ router.post('/trial', async (req, res) => {
   try {
     const { planId, deviceFingerprint } = req.body;
     if (!planId) return res.status(400).json({ error: 'planId is required.' });
+    if (deviceFingerprint !== undefined) {
+      if (typeof deviceFingerprint !== 'string' || deviceFingerprint.length > 256 || !/^[a-zA-Z0-9_+/=.-]{1,256}$/.test(deviceFingerprint)) {
+        return res.status(400).json({ error: 'Invalid deviceFingerprint.' });
+      }
+    }
     const ip = req.ip || req.socket?.remoteAddress;
     const result = await subs.startTrial(req.session.userId, planId, { ip, deviceFp: deviceFingerprint });
     res.json({ subscription: result.subscription });

package/server/routes/browser.js

@@ -103,7 +103,14 @@ router.post('/navigate', async (req, res) => {
       return res.status(403).json({ error: 'This URL is not permitted.' });
     }
 
-    const result = await bc.navigate(url, { waitUntil: waitFor || 'domcontentloaded' });
+    const result = await bc.navigate(url, {
+      waitUntil: waitFor || req.body?.waitUntil || 'domcontentloaded',
+      waitFor,
+      screenshot: req.body?.screenshot !== false,
+      fullPage: req.body?.fullPage === true,
+      referrerMode: req.body?.referrerMode,
+      challengeRetry: req.body?.challengeRetry,
+    });
     res.json(result);
   } catch (err) {
     res.status(500).json({ error: sanitizeError(err) });

package/server/routes/recordings.js

@@ -43,6 +43,62 @@ function getChunkMetadata(req) {
   };
 }
 
+const WAV_HEADER_BYTES = 44;
+
+// Reads the canonical 44-byte WAV header our recorders emit. Returns the audio
+// format fields, or null when the file is not a canonical WAV (in which case
+// the caller streams the chunks unmodified instead of rebuilding a WAV).
+async function readCanonicalWavHeader(filePath) {
+  let handle;
+  try {
+    handle = await fs.promises.open(filePath, 'r');
+    const { bytesRead, buffer } = await handle.read(
+      Buffer.alloc(WAV_HEADER_BYTES),
+      0,
+      WAV_HEADER_BYTES,
+      0,
+    );
+    if (
+      bytesRead < WAV_HEADER_BYTES
+      || buffer.toString('ascii', 0, 4) !== 'RIFF'
+      || buffer.toString('ascii', 8, 12) !== 'WAVE'
+      || buffer.toString('ascii', 36, 40) !== 'data'
+    ) {
+      return null;
+    }
+    return {
+      audioFormat: buffer.readUInt16LE(20),
+      channelCount: buffer.readUInt16LE(22),
+      sampleRate: buffer.readUInt32LE(24),
+      bitsPerSample: buffer.readUInt16LE(34),
+    };
+  } catch {
+    return null;
+  } finally {
+    await handle?.close();
+  }
+}
+
+function buildWavHeader({ audioFormat, channelCount, sampleRate, bitsPerSample }, dataLength) {
+  const blockAlign = channelCount * (bitsPerSample / 8);
+  const byteRate = sampleRate * blockAlign;
+  const header = Buffer.alloc(WAV_HEADER_BYTES);
+  header.write('RIFF', 0, 4, 'ascii');
+  header.writeUInt32LE(36 + dataLength, 4);
+  header.write('WAVE', 8, 4, 'ascii');
+  header.write('fmt ', 12, 4, 'ascii');
+  header.writeUInt32LE(16, 16);
+  header.writeUInt16LE(audioFormat, 20);
+  header.writeUInt16LE(channelCount, 22);
+  header.writeUInt32LE(sampleRate, 24);
+  header.writeUInt32LE(byteRate, 28);
+  header.writeUInt16LE(blockAlign, 32);
+  header.writeUInt16LE(bitsPerSample, 34);
+  header.write('data', 36, 4, 'ascii');
+  header.writeUInt32LE(dataLength, 40);
+  return header;
+}
+
 function buildInlineFilename(sourceKey) {
   const original = `${String(sourceKey || 'recording').normalize('NFKC')}.audio`
     .replace(/[\r\n]+/g, ' ')
@@ -136,10 +192,7 @@ router.get('/:sessionId/audio/:sourceKey', async (req, res) => {
       });
     }
 
-    res.setHeader('Content-Type', mimeType);
-    res.setHeader('Cache-Control', 'no-store');
-    res.setHeader('Content-Disposition', buildInlineFilename(source.source_key));
-
+    const readableChunks = [];
     for (const chunk of chunks) {
       const filePath = chunk.file_path;
       if (!filePath) {
@@ -147,10 +200,47 @@ router.get('/:sessionId/audio/:sourceKey', async (req, res) => {
       }
       try {
         await fs.promises.access(filePath, fs.constants.R_OK);
+        readableChunks.push(chunk);
       } catch {
-        continue;
+        // Skip chunk files that are missing or unreadable on disk.
       }
-      const stream = fs.createReadStream(filePath);
+    }
+    if (readableChunks.length === 0) {
+      return res.status(404).json({ error: 'No audio chunks available.' });
+    }
+
+    res.setHeader('Content-Type', mimeType);
+    res.setHeader('Cache-Control', 'no-store');
+    res.setHeader('Content-Disposition', buildInlineFilename(source.source_key));
+
+    // Each WAV chunk is a self-contained file with its own 44-byte RIFF header
+    // whose declared length only covers that chunk. Streaming them back-to-back
+    // produces a file most players stop reading after the first chunk. Rebuild
+    // one continuous WAV: a single header sized for the combined PCM, followed
+    // by the raw sample data from every chunk. Non-WAV containers (e.g. WebM)
+    // concatenate into a valid stream as-is.
+    const wavFormat = /wav/i.test(mimeType)
+      ? await readCanonicalWavHeader(readableChunks[0].file_path)
+      : null;
+
+    if (wavFormat) {
+      let totalPcmBytes = 0;
+      for (const chunk of readableChunks) {
+        const { size } = await fs.promises.stat(chunk.file_path);
+        totalPcmBytes += Math.max(0, size - WAV_HEADER_BYTES);
+      }
+      res.write(buildWavHeader(wavFormat, totalPcmBytes));
+      for (const chunk of readableChunks) {
+        const stream = fs.createReadStream(chunk.file_path, { start: WAV_HEADER_BYTES });
+        stream.pipe(res, { end: false });
+        await finished(stream);
+      }
+      res.end();
+      return;
+    }
+
+    for (const chunk of readableChunks) {
+      const stream = fs.createReadStream(chunk.file_path);
       stream.pipe(res, { end: false });
       await finished(stream);
     }

package/server/routes/screenHistory.js

@@ -10,6 +10,32 @@ const router = express.Router();
 
 router.use(requireAuth);
 
+function getTimelineService(req) {
+  return req.app?.locals?.timelineService || null;
+}
+
+function normalizeEntries(rawEntries) {
+  if (!Array.isArray(rawEntries)) {
+    throw new Error('entries must be an array.');
+  }
+  return rawEntries.map((entry) => {
+    if (!entry || typeof entry !== 'object') {
+      throw new Error('Each entry must be an object.');
+    }
+    const text = String(entry.text || '').replace(/\s+/g, ' ').trim();
+    if (!text) {
+      throw new Error('Entry text is required.');
+    }
+    return {
+      capturedAt: entry.capturedAt,
+      frontmostApp: entry.frontmostApp,
+      windowTitle: entry.windowTitle,
+      text,
+      ocrConfidence: entry.ocrConfidence,
+    };
+  });
+}
+
 router.get('/search', (req, res) => {
   const { q, limit = 50, offset = 0 } = req.query;
   const userId = req.session.userId;
@@ -21,7 +47,24 @@ router.get('/search', (req, res) => {
       // Full text search. buildFtsQuery sanitizes user input so FTS5 operator
       // characters (hyphens, AND/OR/NOT) don't throw and 500 the request.
       results = db.prepare(`
-        SELECT s.id, s.timestamp, s.app_name, s.text_content
+        SELECT
+          s.id,
+          s.timestamp,
+          s.captured_at,
+          s.captured_at AS capturedAt,
+          s.device_id,
+          s.device_id AS deviceId,
+          s.device_label,
+          s.device_label AS deviceLabel,
+          s.app_name,
+          s.app_name AS appName,
+          s.window_title,
+          s.window_title AS windowTitle,
+          s.text_content,
+          s.ocr_engine,
+          s.ocr_engine AS ocrEngine,
+          s.ocr_confidence,
+          s.ocr_confidence AS ocrConfidence
         FROM screen_history_fts fts
         JOIN screen_history s ON fts.rowid = s.id
         WHERE screen_history_fts MATCH ? AND s.user_id = ?
@@ -34,7 +77,24 @@ router.get('/search', (req, res) => {
     } else {
       // Recent history
       results = db.prepare(`
-        SELECT id, timestamp, app_name, text_content
+        SELECT
+          id,
+          timestamp,
+          captured_at,
+          captured_at AS capturedAt,
+          device_id,
+          device_id AS deviceId,
+          device_label,
+          device_label AS deviceLabel,
+          app_name,
+          app_name AS appName,
+          window_title,
+          window_title AS windowTitle,
+          text_content,
+          ocr_engine,
+          ocr_engine AS ocrEngine,
+          ocr_confidence,
+          ocr_confidence AS ocrConfidence
         FROM screen_history
         WHERE user_id = ?
         ORDER BY timestamp DESC
@@ -49,4 +109,82 @@ router.get('/search', (req, res) => {
   }
 });
 
+router.post('/entries', express.json(), (req, res) => {
+  try {
+    const userId = req.session.userId;
+    const deviceId = String(req.body?.deviceId || '').trim();
+    const activationId = String(req.body?.activationId || '').trim();
+    if (!deviceId) {
+      return res.status(400).json({ error: 'deviceId is required.' });
+    }
+    if (!activationId) {
+      return res.status(400).json({ error: 'activationId is required.' });
+    }
+
+    const device = db.prepare(
+      `SELECT device_id, label, activation_id, revoked_at, passive_history_enabled
+       FROM desktop_companion_devices
+       WHERE user_id = ? AND device_id = ?
+       LIMIT 1`
+    ).get(userId, deviceId);
+    if (!device) {
+      return res.status(404).json({ error: 'Desktop companion device not found.' });
+    }
+    if (device.revoked_at) {
+      return res.status(403).json({ error: 'Desktop companion device has been revoked.' });
+    }
+    if (String(device.activation_id || '') !== activationId) {
+      return res.status(403).json({ error: 'activationId does not match the registered desktop companion device.' });
+    }
+    if (device.passive_history_enabled !== 1) {
+      return res.status(403).json({ error: 'Passive screen history is disabled for this desktop companion device.' });
+    }
+
+    const entries = normalizeEntries(req.body?.entries);
+    const timelineService = getTimelineService(req);
+    if (!timelineService || typeof timelineService.storeScreenEntries !== 'function') {
+      return res.status(503).json({ error: 'Timeline service is unavailable.' });
+    }
+
+    const result = timelineService.storeScreenEntries({
+      userId,
+      deviceId,
+      deviceLabel: device.label || deviceId,
+      entries,
+    });
+    req.app?.locals?.desktopCompanionRegistry?.updatePassiveHistoryState?.(
+      userId,
+      deviceId,
+      {
+        enabled: true,
+        lastUploadedAt: new Date().toISOString(),
+        lastError: null,
+      },
+    );
+    res.status(201).json({
+      ok: true,
+      insertedCount: result.insertedCount,
+    });
+  } catch (err) {
+    const message = getErrorMessage(err);
+    const userId = req.session?.userId;
+    const deviceId = String(req.body?.deviceId || '').trim();
+    if (userId && deviceId) {
+      try {
+        req.app?.locals?.desktopCompanionRegistry?.updatePassiveHistoryState?.(
+          userId,
+          deviceId,
+          {
+            enabled: true,
+            lastError: message,
+          },
+        );
+      } catch {
+        // Best-effort; do not mask the original ingest failure.
+      }
+    }
+    res.status(400).json({ error: message });
+  }
+});
+
 module.exports = router;

package/server/routes/timeline.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const express = require('express');
+const { requireAuth } = require('../middleware/auth');
+
+const router = express.Router();
+
+router.use(requireAuth);
+
+router.get('/', (req, res) => {
+  const timelineService = req.app?.locals?.timelineService;
+  if (!timelineService || typeof timelineService.listEvents !== 'function') {
+    return res.status(503).json({ error: 'Timeline service is unavailable.' });
+  }
+
+  const rawSources = []
+    .concat(req.query.source || [])
+    .concat(req.query.sources || []);
+  const sources = rawSources
+    .flatMap((value) => String(value || '').split(','))
+    .map((value) => value.trim().toLowerCase())
+    .filter(Boolean);
+  const items = timelineService.listEvents(req.session.userId, {
+    limit: req.query.limit,
+    beforeOccurredAt: req.query.beforeOccurredAt,
+    beforeId: req.query.beforeId,
+    agentId: req.query.agentId,
+    sources,
+  });
+  const lastItem = items[items.length - 1] || null;
+
+  res.json({
+    items,
+    nextCursor: lastItem
+      ? {
+        beforeOccurredAt: lastItem.occurredAt,
+        beforeId: lastItem.id,
+      }
+      : null,
+  });
+});
+
+module.exports = router;