neoagent 2.4.2-beta.5 → 2.4.2-beta.6

package/server/services/workspace/code_navigation.js

@@ -0,0 +1,194 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { spawnSync } = require('child_process');
+const { parseSync } = require('oxc-parser');
+const db = require('../../db/database');
+const {
+  getEmbedding,
+  cosineSimilarity,
+  serializeEmbedding,
+  deserializeEmbedding,
+  keywordSimilarity,
+} = require('../memory/embeddings');
+
+const CODE_EXTENSIONS = new Set(['.js', '.cjs', '.mjs', '.jsx', '.ts', '.tsx']);
+
+function lineAtOffset(content, offset) {
+  return content.slice(0, Math.max(0, offset)).split('\n').length;
+}
+
+function declarationInfo(node) {
+  const target = node?.type === 'ExportNamedDeclaration' || node?.type === 'ExportDefaultDeclaration'
+    ? node.declaration
+    : node;
+  if (!target) return null;
+  if (['FunctionDeclaration', 'ClassDeclaration', 'TSEnumDeclaration', 'TSInterfaceDeclaration', 'TSTypeAliasDeclaration'].includes(target.type)) {
+    return { node: target, name: target.id?.name || 'default', type: target.type };
+  }
+  if (target.type === 'VariableDeclaration') {
+    const first = target.declarations?.[0];
+    if (first?.id?.name) return { node: target, name: first.id.name, type: target.type };
+  }
+  return null;
+}
+
+class CodeNavigationService {
+  constructor(options = {}) {
+    this.workspaceManager = options.workspaceManager;
+  }
+
+  lexical(userId, options = {}) {
+    const root = this.workspaceManager.resolvePath(userId, options.path || '.', 'path');
+    const query = String(options.query || '').trim();
+    if (!query) throw new Error('query is required.');
+    const args = ['--json', '--line-number', '--max-count', '100'];
+    if (options.fixed !== false) args.push('--fixed-strings');
+    if (options.include) args.push('--glob', String(options.include));
+    args.push(query, root);
+    const result = spawnSync('rg', args, { encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 });
+    if (result.error?.code === 'ENOENT') {
+      return this.workspaceManager.searchFiles(userId, options);
+    }
+    const matches = String(result.stdout || '').split('\n').filter(Boolean).flatMap((line) => {
+      try {
+        const event = JSON.parse(line);
+        if (event.type !== 'match') return [];
+        const data = event.data;
+        return [{
+          path: path.relative(root, data.path.text).split(path.sep).join('/'),
+          line: data.line_number,
+          excerpt: String(data.lines.text || '').trim().slice(0, 1000),
+          submatches: data.submatches.map((item) => ({ start: item.start, end: item.end })),
+        }];
+      } catch {
+        return [];
+      }
+    });
+    return { mode: 'lexical', count: matches.length, results: matches };
+  }
+
+  extractStructure(userId, options = {}) {
+    const absolute = this.workspaceManager.resolvePath(userId, options.path || '', 'path');
+    const content = fs.readFileSync(absolute, 'utf8');
+    if (!CODE_EXTENSIONS.has(path.extname(absolute).toLowerCase())) {
+      throw new Error('Structural parsing currently supports JavaScript and TypeScript files.');
+    }
+    const parsed = parseSync(absolute, content);
+    const symbols = [];
+    for (const entry of parsed.program?.body || []) {
+      const info = declarationInfo(entry);
+      if (!info) continue;
+      const startLine = lineAtOffset(content, info.node.start);
+      const endLine = lineAtOffset(content, info.node.end);
+      symbols.push({
+        name: info.name,
+        type: info.type,
+        startLine,
+        endLine,
+        excerpt: content.slice(info.node.start, Math.min(info.node.end, info.node.start + 1600)),
+      });
+    }
+    return {
+      mode: 'structure',
+      path: absolute,
+      parseErrors: (parsed.errors || []).map((error) => String(error.message || error)).slice(0, 20),
+      symbols,
+    };
+  }
+
+  async indexWorkspace(userId, options = {}) {
+    const root = this.workspaceManager.resolvePath(userId, options.path || '.', 'path');
+    const files = [];
+    const walk = (dir) => {
+      if (files.length >= 500) return;
+      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name.startsWith('.neoagent-')) continue;
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) walk(fullPath);
+        else if (entry.isFile() && CODE_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) files.push(fullPath);
+      }
+    };
+    walk(root);
+    let indexed = 0;
+    for (const file of files) {
+      if (fs.statSync(file).size > 512 * 1024) continue;
+      const structure = this.extractStructure(userId, { path: file });
+      const relative = path.relative(this.workspaceManager._ensureWorkspaceRootSync(userId), file).split(path.sep).join('/');
+      for (const symbol of structure.symbols) {
+        const hash = crypto.createHash('sha256').update(symbol.excerpt).digest('hex');
+        const existing = db.prepare(
+          `SELECT id, content_hash FROM workspace_code_index
+           WHERE user_id = ? AND path = ? AND symbol = ? AND start_line = ?`
+        ).get(userId, relative, symbol.name, symbol.startLine);
+        if (existing?.content_hash === hash) continue;
+        const embedding = await getEmbedding(`${symbol.name}\n${symbol.excerpt}`);
+        db.prepare(
+          `INSERT INTO workspace_code_index (
+            user_id, path, symbol, symbol_type, start_line, end_line, content, content_hash, embedding
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+          ON CONFLICT(user_id, path, symbol, start_line) DO UPDATE SET
+            symbol_type = excluded.symbol_type,
+            end_line = excluded.end_line,
+            content = excluded.content,
+            content_hash = excluded.content_hash,
+            embedding = excluded.embedding,
+            updated_at = datetime('now')`
+        ).run(
+          userId,
+          relative,
+          symbol.name,
+          symbol.type,
+          symbol.startLine,
+          symbol.endLine,
+          symbol.excerpt,
+          hash,
+          embedding ? serializeEmbedding(embedding) : null,
+        );
+        indexed += 1;
+      }
+    }
+    return { filesScanned: files.length, symbolsIndexed: indexed };
+  }
+
+  async semantic(userId, options = {}) {
+    const query = String(options.query || '').trim();
+    if (!query) throw new Error('query is required.');
+    const indexStatus = await this.indexWorkspace(userId, options);
+    const queryEmbedding = await getEmbedding(query);
+    const rows = db.prepare(
+      `SELECT path, symbol, symbol_type, start_line, end_line, content, embedding
+       FROM workspace_code_index WHERE user_id = ?`
+    ).all(userId);
+    const limit = Math.min(Math.max(Number(options.limit || 12), 1), 50);
+    const results = rows.map((row) => {
+      const vector = deserializeEmbedding(row.embedding);
+      const score = queryEmbedding && vector
+        ? cosineSimilarity(queryEmbedding, vector)
+        : keywordSimilarity(query, `${row.symbol || ''} ${row.content}`);
+      return {
+        path: row.path,
+        symbol: row.symbol,
+        symbolType: row.symbol_type,
+        startLine: row.start_line,
+        endLine: row.end_line,
+        score,
+        excerpt: row.content.slice(0, 1600),
+      };
+    }).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).slice(0, limit);
+    return { mode: 'semantic', indexStatus, count: results.length, results };
+  }
+
+  navigate(userId, options = {}) {
+    const mode = String(options.mode || 'lexical');
+    if (mode === 'structure') return this.extractStructure(userId, options);
+    if (mode === 'semantic') return this.semantic(userId, options);
+    return this.lexical(userId, options);
+  }
+}
+
+module.exports = {
+  CodeNavigationService,
+};

package/server/services/workspace/structured_data.js

@@ -0,0 +1,93 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const Database = require('better-sqlite3');
+
+function parseDelimited(text, delimiter) {
+  const rows = [];
+  let row = [];
+  let value = '';
+  let quoted = false;
+  for (let index = 0; index < text.length; index += 1) {
+    const char = text[index];
+    if (char === '"') {
+      if (quoted && text[index + 1] === '"') {
+        value += '"';
+        index += 1;
+      } else {
+        quoted = !quoted;
+      }
+    } else if (char === delimiter && !quoted) {
+      row.push(value);
+      value = '';
+    } else if ((char === '\n' || char === '\r') && !quoted) {
+      if (char === '\r' && text[index + 1] === '\n') index += 1;
+      row.push(value);
+      if (row.some((cell) => cell !== '')) rows.push(row);
+      row = [];
+      value = '';
+    } else {
+      value += char;
+    }
+  }
+  row.push(value);
+  if (row.some((cell) => cell !== '')) rows.push(row);
+  const headers = rows.shift() || [];
+  return rows.map((cells) => Object.fromEntries(headers.map((header, index) => [header, cells[index] ?? ''])));
+}
+
+class StructuredDataService {
+  constructor(options = {}) {
+    this.workspaceManager = options.workspaceManager;
+  }
+
+  query(userId, options = {}) {
+    const filePath = this.workspaceManager.resolvePath(userId, options.path || '', 'path');
+    const extension = path.extname(filePath).toLowerCase();
+    const limit = Math.min(Math.max(Number(options.limit || 100), 1), 1000);
+    if (extension === '.sqlite' || extension === '.db' || extension === '.sqlite3') {
+      const sql = String(options.sql || '').trim();
+      if (!sql) throw new Error('sql is required for SQLite data.');
+      const database = new Database(filePath, { readonly: true, fileMustExist: true });
+      try {
+        const statement = database.prepare(sql);
+        if (!statement.reader || !statement.readonly) throw new Error('Only read-only queries are allowed.');
+        return {
+          format: 'sqlite',
+          columns: statement.columns().map((column) => column.name),
+          rows: statement.all(options.parameters || {}).slice(0, limit),
+        };
+      } finally {
+        database.close();
+      }
+    }
+    let rows;
+    if (extension === '.json') {
+      const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+      rows = Array.isArray(parsed) ? parsed : [parsed];
+    } else if (extension === '.csv' || extension === '.tsv') {
+      rows = parseDelimited(fs.readFileSync(filePath, 'utf8'), extension === '.tsv' ? '\t' : ',');
+    } else {
+      throw new Error('Supported structured-data formats are CSV, TSV, JSON, and SQLite.');
+    }
+    const selectedColumns = Array.isArray(options.columns) ? options.columns.map(String) : null;
+    const filtered = rows.filter((row) => {
+      if (!options.equals || typeof options.equals !== 'object') return true;
+      return Object.entries(options.equals).every(([key, value]) => row?.[key] === value);
+    }).slice(0, limit).map((row) => {
+      if (!selectedColumns) return row;
+      return Object.fromEntries(selectedColumns.map((column) => [column, row?.[column]]));
+    });
+    return {
+      format: extension.slice(1),
+      columns: filtered[0] && typeof filtered[0] === 'object' ? Object.keys(filtered[0]) : [],
+      rows: filtered,
+    };
+  }
+}
+
+module.exports = {
+  StructuredDataService,
+  parseDelimited,
+};

package/server/utils/cloud-security.js

@@ -28,14 +28,36 @@ const PRIVATE_IPV4 = [
 
 function isPrivateHost(hostname) {
   if (!hostname) return false;
-  const h = hostname.toLowerCase().replace(/^\[|\]$/g, '');
+  let h = hostname.toLowerCase().replace(/^\[|\]$/g, '');
 
   if (h === 'localhost' || h === 'localhost.localdomain') return true;
   if (h.endsWith('.local') || h.endsWith('.internal') || h.endsWith('.localhost')) return true;
 
-  // IPv6 loopback and link-local
+  // Unwrap IPv4-mapped/compatible IPv6 (e.g. ::ffff:127.0.0.1 or ::ffff:7f00:1)
+  // so the embedded IPv4 address is checked against the private ranges below.
+  const mapped = h.match(/^::(?:ffff:)?(?:0:)?([0-9a-f.:]+)$/);
+  if (mapped) {
+    const tail = mapped[1];
+    if (tail.includes('.')) {
+      // Dotted IPv4 form, e.g. ::ffff:127.0.0.1
+      h = tail;
+    } else {
+      // Hex form, e.g. ::ffff:7f00:1 -> reconstruct dotted IPv4.
+      const groups = tail.split(':');
+      if (groups.length === 2) {
+        const hi = parseInt(groups[0], 16);
+        const lo = parseInt(groups[1], 16);
+        if (Number.isFinite(hi) && Number.isFinite(lo) && hi <= 0xffff && lo <= 0xffff) {
+          h = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
+        }
+      }
+    }
+  }
+
+  // IPv6 loopback, link-local and unique-local
   if (h === '::1' || h === '::') return true;
   if (h.startsWith('fe80:')) return true;
+  if (/^f[cd][0-9a-f]*:/.test(h)) return true; // fc00::/7 unique-local
 
   for (const pattern of PRIVATE_IPV4) {
     if (pattern.test(h)) return true;