neoagent 2.3.1-beta.6 → 2.3.1-beta.61

package/server/services/wearable/firmware_manifest.js

@@ -0,0 +1,353 @@
+'use strict';
+
+const DEFAULT_GITHUB_REPOSITORY = 'NeoLabs-Systems/NeoAgent';
+const DEFAULT_ASSET_NAME = 'neoagent-wearable-firmware.bin';
+const MANIFEST_CACHE_TTL_MS = 5 * 60 * 1000;
+const manifestCache = new Map();
+
+function trimString(value, maxLength = 512) {
+  return String(value || '').trim().slice(0, maxLength);
+}
+
+function normalizeChannel(value) {
+  return trimString(value).toLowerCase() === 'beta' ? 'beta' : 'stable';
+}
+
+function parseRepositorySlug(value) {
+  const raw = trimString(value, 256);
+  if (!raw) {
+    return null;
+  }
+  const slugMatch = raw.match(/^([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)$/);
+  if (slugMatch) {
+    return slugMatch[1].replace(/\.git$/, '');
+  }
+  const urlMatch = raw.match(/github\.com[/:]([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+?)(?:\.git)?(?:\/.*)?$/i);
+  if (urlMatch) {
+    return urlMatch[1].replace(/\.git$/, '');
+  }
+  return null;
+}
+
+function getGithubRepository() {
+  return (
+    parseRepositorySlug(process.env.NEOAGENT_WEARABLE_FIRMWARE_GITHUB_REPOSITORY)
+    || DEFAULT_GITHUB_REPOSITORY
+  );
+}
+
+function getFirmwareAssetName() {
+  return trimString(process.env.NEOAGENT_WEARABLE_FIRMWARE_ASSET_NAME, 128) || DEFAULT_ASSET_NAME;
+}
+
+function getGithubToken() {
+  return trimString(
+    process.env.NEOAGENT_WEARABLE_GITHUB_TOKEN
+      || process.env.GITHUB_TOKEN
+      || process.env.GH_TOKEN,
+    2048
+  ) || null;
+}
+
+function stripHashPrefix(value) {
+  const text = trimString(value, 512);
+  if (!text) {
+    return null;
+  }
+  return text.replace(/^sha256:/i, '').toLowerCase();
+}
+
+function toBoolean(value, fallback = false) {
+  const normalized = String(value || '').trim().toLowerCase();
+  if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
+  return fallback;
+}
+
+function releaseHasAsset(release, assetName) {
+  return Boolean(selectReleaseAsset(release, assetName));
+}
+
+function selectGithubRelease(releases, channel, assetName = null) {
+  const normalizedChannel = normalizeChannel(channel);
+  if (normalizedChannel === 'stable') {
+    const candidates = Array.isArray(releases)
+      ? releases.filter((release) => release && release.prerelease === false && release.draft === false)
+      : [];
+    return candidates.find((release) => !assetName || releaseHasAsset(release, assetName)) || candidates[0] || null;
+  }
+
+  const betaPattern = /-beta(?:\.\d+)?$/i;
+  const candidates = Array.isArray(releases)
+    ? releases.filter((release) => release && release.prerelease === true && betaPattern.test(String(release.tag_name || '')))
+    : [];
+  candidates.sort((left, right) => {
+    const rightPublished = Date.parse(right?.published_at ?? right?.created_at ?? '') || 0;
+    const leftPublished = Date.parse(left?.published_at ?? left?.created_at ?? '') || 0;
+    return rightPublished - leftPublished;
+  });
+  return candidates.find((release) => !assetName || releaseHasAsset(release, assetName)) || candidates[0] || null;
+}
+
+function selectReleaseAsset(release, assetName) {
+  if (!release || !Array.isArray(release.assets) || release.assets.length === 0) {
+    return null;
+  }
+  const expectedName = trimString(assetName, 128) || DEFAULT_ASSET_NAME;
+  return release.assets.find((asset) => asset && asset.name === expectedName) || null;
+}
+
+function selectChecksumAsset(release, assetName) {
+  if (!release || !Array.isArray(release.assets) || release.assets.length === 0) {
+    return null;
+  }
+  const expectedName = trimString(assetName, 128) || DEFAULT_ASSET_NAME;
+  return release.assets.find((asset) => {
+    const name = String(asset?.name || '');
+    if (!name || name === expectedName) {
+      return false;
+    }
+    const lower = name.toLowerCase();
+    return (
+      lower === `${expectedName.toLowerCase()}.sha256`
+      || lower === `${expectedName.toLowerCase()}.sha256sum`
+      || lower.includes('checksum')
+      || lower.endsWith('.sha256')
+      || lower.endsWith('.sha256sum')
+    );
+  }) || null;
+}
+
+async function fetchGithubJson(fetchImpl, url, token) {
+  const response = await fetchImpl(url, {
+    headers: {
+      Accept: 'application/vnd.github+json',
+      ...(token ? { Authorization: `Bearer ${token}` } : {}),
+    },
+  });
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    const error = new Error(`GitHub API request failed with ${response.status}`);
+    error.status = response.status;
+    error.body = body;
+    throw error;
+  }
+  return response.json();
+}
+
+async function fetchText(fetchImpl, url, token) {
+  const response = await fetchImpl(url, {
+    headers: {
+      Accept: 'text/plain, application/octet-stream;q=0.9, */*;q=0.1',
+      ...(token ? { Authorization: `Bearer ${token}` } : {}),
+    },
+  });
+  if (!response.ok) {
+    const error = new Error(`Asset request failed with ${response.status}`);
+    error.status = response.status;
+    throw error;
+  }
+  return response.text();
+}
+
+function parseChecksumBody(body, assetName) {
+  const normalizedAssetName = String(assetName || '').trim();
+  const lines = String(body || '')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  for (const line of lines) {
+    const firstToken = line.split(/\s+/)[0];
+    const normalized = stripHashPrefix(firstToken);
+    if (normalized && /^[a-f0-9]{64}$/i.test(normalized)) {
+      if (!normalizedAssetName || line.includes(normalizedAssetName) || lines.length === 1) {
+        return normalized;
+      }
+    }
+  }
+  return null;
+}
+
+async function fetchGithubRelease(fetchImpl, repository, channel, token, assetName = null) {
+  const normalizedChannel = normalizeChannel(channel);
+  const releases = await fetchGithubJson(fetchImpl, `https://api.github.com/repos/${repository}/releases?per_page=100`, token);
+  const release = selectGithubRelease(releases, normalizedChannel, assetName);
+  if (!release) {
+    const error = new Error(`No ${normalizedChannel} firmware release found for ${repository}`);
+    error.status = 404;
+    throw error;
+  }
+  return release;
+}
+
+function cacheKey({ repository, channel, assetName, downloadUrlOverride }) {
+  return [repository, channel, assetName, downloadUrlOverride || ''].join('|');
+}
+
+function getCachedManifest(key) {
+  const cached = manifestCache.get(key);
+  if (!cached) {
+    return null;
+  }
+  if (cached.expiresAt <= Date.now()) {
+    manifestCache.delete(key);
+    return null;
+  }
+  return cached.manifest;
+}
+
+function setCachedManifest(key, manifest) {
+  manifestCache.set(key, {
+    manifest,
+    expiresAt: Date.now() + MANIFEST_CACHE_TTL_MS,
+  });
+}
+
+async function resolveFirmwareManifest({
+  channel,
+  downloadUrlOverride,
+  currentVersionOverride,
+  releaseNotesUrlOverride,
+  sha256Override,
+  repositoryOverride,
+  assetNameOverride,
+  fetchImpl = fetch,
+} = {}) {
+  const normalizedChannel = normalizeChannel(channel);
+  const repository = parseRepositorySlug(repositoryOverride) || getGithubRepository();
+  const assetName = trimString(assetNameOverride, 128) || getFirmwareAssetName();
+  const downloadUrl = trimString(downloadUrlOverride, 2000);
+  const cacheId = cacheKey({ repository, channel: normalizedChannel, assetName, downloadUrlOverride: downloadUrl });
+  const cached = getCachedManifest(cacheId);
+  if (cached) {
+    return cached;
+  }
+
+  if (downloadUrl) {
+    const manifest = {
+      configured: true,
+      source: 'static',
+      manifestVersion: 2,
+      channel: normalizedChannel,
+      currentVersion: trimString(currentVersionOverride, 120) || null,
+      releaseName: trimString(currentVersionOverride, 120) || null,
+      releaseTag: trimString(currentVersionOverride, 120) || null,
+      minimumServerVersion: trimString(process.env.NEOAGENT_WEARABLE_MIN_SERVER_VERSION, 120) || null,
+      downloadUrl,
+      releaseNotesUrl: trimString(releaseNotesUrlOverride, 2000) || null,
+      sha256: stripHashPrefix(sha256Override),
+      assetName,
+      repository,
+      generatedAt: new Date().toISOString(),
+      mandatory: toBoolean(process.env.NEOAGENT_WEARABLE_FIRMWARE_MANDATORY, false),
+    };
+    setCachedManifest(cacheId, manifest);
+    return manifest;
+  }
+
+  if (!repository) {
+    return {
+      configured: false,
+      source: 'github',
+      manifestVersion: 2,
+      channel: normalizedChannel,
+      downloadUrl: null,
+      releaseNotesUrl: null,
+      sha256: null,
+      assetName,
+      repository: null,
+      generatedAt: new Date().toISOString(),
+      error: 'GitHub repository is not configured.',
+      mandatory: false,
+    };
+  }
+
+  try {
+    const token = getGithubToken();
+    const release = await fetchGithubRelease(fetchImpl, repository, normalizedChannel, token, assetName);
+    const asset = selectReleaseAsset(release, assetName);
+    if (!asset || !asset.browser_download_url) {
+      return {
+        configured: false,
+        source: 'github',
+        manifestVersion: 2,
+        channel: normalizedChannel,
+        currentVersion: trimString(release?.tag_name, 120) || null,
+        releaseName: trimString(release?.name, 128) || trimString(release?.tag_name, 120) || null,
+        releaseTag: trimString(release?.tag_name, 120) || null,
+        minimumServerVersion: trimString(process.env.NEOAGENT_WEARABLE_MIN_SERVER_VERSION, 120) || null,
+        downloadUrl: null,
+        releaseNotesUrl: trimString(release?.html_url, 2000) || null,
+        sha256: null,
+        assetName,
+        repository,
+        generatedAt: new Date().toISOString(),
+        error: `Release asset ${assetName} was not found.`,
+        mandatory: false,
+        prerelease: Boolean(release?.prerelease),
+      };
+    }
+
+    let checksum = null;
+    const checksumAsset = selectChecksumAsset(release, asset.name);
+    if (checksumAsset?.browser_download_url) {
+      try {
+        checksum = parseChecksumBody(
+          await fetchText(fetchImpl, checksumAsset.browser_download_url, token),
+          asset.name,
+        );
+      } catch {
+        checksum = null;
+      }
+    }
+
+    const manifest = {
+      configured: true,
+      source: 'github',
+      manifestVersion: 2,
+      channel: normalizedChannel,
+      currentVersion: trimString(release?.tag_name, 120) || null,
+      releaseName: trimString(release?.name, 128) || trimString(release?.tag_name, 120) || null,
+      releaseTag: trimString(release?.tag_name, 120) || null,
+      minimumServerVersion: trimString(process.env.NEOAGENT_WEARABLE_MIN_SERVER_VERSION, 120) || null,
+      downloadUrl: asset.browser_download_url,
+      releaseNotesUrl: trimString(release?.html_url, 2000) || null,
+      sha256: checksum,
+      assetName: asset.name,
+      repository,
+      generatedAt: new Date().toISOString(),
+      mandatory: toBoolean(process.env.NEOAGENT_WEARABLE_FIRMWARE_MANDATORY, false),
+      prerelease: Boolean(release?.prerelease),
+    };
+    setCachedManifest(cacheId, manifest);
+    return manifest;
+  } catch (error) {
+    return {
+      configured: false,
+      source: 'github',
+      manifestVersion: 2,
+      channel: normalizedChannel,
+      downloadUrl: null,
+      releaseNotesUrl: null,
+      sha256: null,
+      assetName,
+      repository,
+      generatedAt: new Date().toISOString(),
+      error: error.message,
+      mandatory: false,
+    };
+  }
+}
+
+module.exports = {
+  DEFAULT_ASSET_NAME,
+  DEFAULT_GITHUB_REPOSITORY,
+  getFirmwareAssetName,
+  getGithubRepository,
+  normalizeChannel,
+  parseRepositorySlug,
+  resolveFirmwareManifest,
+  selectGithubRelease,
+  selectReleaseAsset,
+  stripHashPrefix,
+};

package/server/services/wearable/gateway.js

@@ -0,0 +1,350 @@
+'use strict';
+
+const { WebSocketServer } = require('ws');
+const { sanitizeError } = require('../../utils/security');
+const {
+  WEARABLE_WS_PATH,
+  isSupportedClientMessageType,
+  isWearableHello,
+  parseWearableMessage,
+} = require('./protocol');
+
+const UPGRADE_WINDOW_MS = 60 * 1000;
+const UPGRADE_MAX_ATTEMPTS = 30;
+const HELLO_TIMEOUT_MS = 5000;
+
+function rejectUpgrade(socket, statusCode, message) {
+  try {
+    socket.write(
+      `HTTP/1.1 ${statusCode} ${message}\r\n` +
+      'Connection: close\r\n' +
+      '\r\n',
+    );
+  } catch {}
+  try {
+    socket.destroy();
+  } catch {}
+}
+
+function remoteAddressFromRequest(req) {
+  const forwarded = req.headers?.['x-forwarded-for'];
+  if (typeof forwarded === 'string' && forwarded.trim()) {
+    return forwarded.split(',')[0].trim();
+  }
+  return req.socket?.remoteAddress || 'unknown';
+}
+
+function createUpgradeLimiter() {
+  const attempts = new Map();
+  return (remoteAddress) => {
+    const key = String(remoteAddress || 'unknown');
+    const now = Date.now();
+    for (const [entryKey, stats] of attempts.entries()) {
+      if (!stats?.windowStart || stats.windowStart + UPGRADE_WINDOW_MS <= now) {
+        attempts.delete(entryKey);
+      }
+    }
+    const current = attempts.get(key);
+    if (!current) {
+      attempts.set(key, { windowStart: now, count: 1 });
+      return true;
+    }
+    if (now - current.windowStart >= UPGRADE_WINDOW_MS) {
+      attempts.set(key, { windowStart: now, count: 1 });
+      return true;
+    }
+    if (current.count >= UPGRADE_MAX_ATTEMPTS) {
+      return false;
+    }
+    current.count += 1;
+    return true;
+  };
+}
+
+function sendJson(ws, payload) {
+  if (!ws || ws.readyState !== 1) return;
+  ws.send(JSON.stringify(payload));
+}
+
+function asObject(value) {
+  return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+
+function toOptionalString(value, maxLength = 512) {
+  if (value == null) return '';
+  return String(value).trim().slice(0, maxLength);
+}
+
+function toBoundedInt(value, fallback, min, max) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+function createWearableVoiceSink(ws, voiceRuntimeManager) {
+  return {
+    publishReady: async (session, extra = {}) => {
+      sendJson(ws, {
+        type: 'voice:session_ready',
+        sessionId: session.id,
+        ...extra,
+      });
+    },
+    setState: async (session, state, extra = {}) => {
+      sendJson(ws, {
+        type: 'voice:assistant_state',
+        sessionId: session.id,
+        state,
+        ...extra,
+      });
+    },
+    publishTranscriptPartial: async (session, content) => {
+      sendJson(ws, {
+        type: 'voice:transcript_partial',
+        sessionId: session.id,
+        content,
+      });
+    },
+    publishTranscriptFinal: async (session, content) => {
+      sendJson(ws, {
+        type: 'voice:transcript_final',
+        sessionId: session.id,
+        content,
+      });
+    },
+    publishAssistantOutput: async (session, content, options = {}) => {
+      await voiceRuntimeManager.deliverWearableAssistantOutput(ws, session.id, content, options);
+    },
+    interruptOutput: async (session) => {
+      sendJson(ws, {
+        type: 'voice:assistant_state',
+        sessionId: session.id,
+        state: 'interrupted',
+      });
+    },
+    publishError: async (session, message, extra = {}) => {
+      sendJson(ws, {
+        type: 'voice:error',
+        sessionId: session.id,
+        error: message,
+        ...extra,
+      });
+    },
+    close: async (session, reason = 'closed') => {
+      sendJson(ws, {
+        type: 'voice:assistant_state',
+        sessionId: session.id,
+        state: 'closed',
+        reason,
+      });
+    },
+  };
+}
+
+function bindWearableGateway(httpServer, app, sessionMiddleware) {
+  const wss = new WebSocketServer({ noServer: true });
+  const allowUpgradeAttempt = createUpgradeLimiter();
+
+  httpServer.on('upgrade', (req, socket, head) => {
+    let url;
+    try {
+      url = new URL(req.url, 'http://localhost');
+    } catch {
+      return;
+    }
+    if (url.pathname !== WEARABLE_WS_PATH) {
+      return;
+    }
+    const remoteAddress = remoteAddressFromRequest(req);
+    if (!allowUpgradeAttempt(remoteAddress)) {
+      rejectUpgrade(socket, 429, 'Too Many Requests');
+      return;
+    }
+    sessionMiddleware(req, {}, (err) => {
+      if (err) {
+        rejectUpgrade(socket, 500, 'Session Error');
+        return;
+      }
+      if (!req.session?.userId) {
+        rejectUpgrade(socket, 401, 'Unauthorized');
+        return;
+      }
+      const wearableService = app?.locals?.wearableService;
+      const voiceRuntimeManager = app?.locals?.voiceRuntimeManager;
+      if (!wearableService || !voiceRuntimeManager) {
+        rejectUpgrade(socket, 503, 'Service Unavailable');
+        return;
+      }
+
+      wss.handleUpgrade(req, socket, head, (ws) => {
+        ws.isAlive = true;
+        ws.on('pong', () => {
+          ws.isAlive = true;
+        });
+
+        let initialized = false;
+        let deviceId = '';
+        const activeSessionIds = new Set();
+        const helloTimer = setTimeout(() => {
+          if (!initialized) {
+            try {
+              ws.close(1008, 'Wearable hello timed out');
+            } catch {}
+          }
+        }, HELLO_TIMEOUT_MS);
+
+        const teardown = async () => {
+          clearTimeout(helloTimer);
+          if (deviceId) {
+            wearableService.unregisterConnection(req.session.userId, deviceId);
+          }
+          await Promise.allSettled(
+            Array.from(activeSessionIds).map((sessionId) =>
+              voiceRuntimeManager.closeSession(sessionId, 'wearable_socket_closed'),
+            ),
+          );
+        };
+
+        ws.on('close', () => {
+          void teardown();
+        });
+
+        ws.on('message', async (data) => {
+          try {
+            const message = parseWearableMessage(data);
+            if (!initialized) {
+              if (!isWearableHello(message)) {
+                throw new Error('wearable:hello is required before other messages.');
+              }
+              const connection = wearableService.registerConnection({
+                userId: req.session.userId,
+                ws,
+                remoteAddress,
+                userAgent: req.headers['user-agent'] || null,
+                hello: message,
+              });
+              initialized = true;
+              deviceId = connection.deviceId;
+              sendJson(ws, {
+                type: 'wearable:hello',
+                ok: true,
+                deviceId,
+                userId: req.session.userId,
+                serverTime: new Date().toISOString(),
+              });
+              return;
+            }
+
+            if (!isSupportedClientMessageType(message.type)) {
+              throw new Error(`Unsupported wearable message type "${message.type}".`);
+            }
+            wearableService.touchConnection(req.session.userId, deviceId);
+            const payload = asObject(message);
+            const sessionId = toOptionalString(payload.sessionId, 128);
+
+            switch (message.type) {
+              case 'voice:session_open': {
+                const resolvedSessionId = sessionId || null;
+                const session = await voiceRuntimeManager.openWearableSession({
+                  userId: req.session.userId,
+                  agentId: payload.agentId || payload.agent_id || null,
+                  sessionId: resolvedSessionId,
+                  sink: createWearableVoiceSink(ws, voiceRuntimeManager),
+                });
+                activeSessionIds.add(session.id);
+                break;
+              }
+              case 'voice:input_start':
+                if (!sessionId) throw new Error('sessionId is required');
+                await voiceRuntimeManager.beginInput(sessionId, {
+                  mimeType: toOptionalString(payload.mimeType, 128),
+                  turnId: toOptionalString(payload.turnId, 128),
+                });
+                break;
+              case 'voice:audio_chunk': {
+                if (!sessionId) throw new Error('sessionId is required');
+                const audioBase64 = toOptionalString(payload.audioBase64, 800000);
+                if (!audioBase64) throw new Error('audioBase64 is required');
+                const sequence = toBoundedInt(payload.sequence, -1, -1, 1000000);
+                if (sequence < 0) throw new Error('sequence is required');
+                const turnId = toOptionalString(payload.turnId, 128);
+                if (!turnId) throw new Error('turnId is required');
+                const audioBytes = Buffer.from(audioBase64, 'base64');
+                const appendResult = await voiceRuntimeManager.appendInputAudio(sessionId, audioBytes, {
+                  mimeType: toOptionalString(payload.mimeType, 128),
+                  turnId,
+                  sequence,
+                });
+                sendJson(ws, {
+                  type: 'voice:chunk_ack',
+                  sessionId,
+                  turnId,
+                  sequence,
+                  receivedThrough: appendResult?.receivedThrough ?? sequence,
+                });
+                break;
+              }
+              case 'voice:input_commit': {
+                if (!sessionId) throw new Error('sessionId is required');
+                await voiceRuntimeManager.commitInput(sessionId, {
+                  turnId: toOptionalString(payload.turnId, 128),
+                  finalSequence: toBoundedInt(payload.finalSequence, -1, -1, 1000000),
+                  promptHint: toOptionalString(payload.promptHint, 2000),
+                });
+                break;
+              }
+              case 'voice:interrupt':
+                if (!sessionId) throw new Error('sessionId is required');
+                await voiceRuntimeManager.interruptSession(sessionId);
+                break;
+              case 'voice:session_close':
+                if (!sessionId) throw new Error('sessionId is required');
+                activeSessionIds.delete(sessionId);
+                await voiceRuntimeManager.closeSession(sessionId, 'wearable_client_closed');
+                break;
+              default:
+                break;
+            }
+          } catch (error) {
+            sendJson(ws, {
+              type: 'voice:error',
+              error: sanitizeError(error),
+            });
+          }
+        });
+      });
+    });
+  });
+
+  const heartbeat = setInterval(() => {
+    for (const ws of wss.clients) {
+      if (ws.isAlive === false) {
+        try {
+          ws.terminate();
+        } catch {}
+        continue;
+      }
+      ws.isAlive = false;
+      try {
+        ws.ping();
+      } catch {}
+    }
+  }, 30000);
+  heartbeat.unref?.();
+
+  if (app?.locals) {
+    app.locals.wearableGateway = {
+      close: () =>
+        new Promise((resolve) => {
+          clearInterval(heartbeat);
+          wss.close(() => resolve());
+        }),
+    };
+  }
+
+  return wss;
+}
+
+module.exports = {
+  bindWearableGateway,
+};