@danielblomma/cortex-mcp 2.4.0 → 2.4.2

package/scaffold/scripts/lib/ingest/arguments.mjs

@@ -0,0 +1,78 @@
+export function parseArgs(argv) {
+  const args = new Set(argv.slice(2));
+  if (args.has("--help") || args.has("-h")) {
+    printHelp();
+    process.exit(0);
+  }
+
+  return {
+    mode: args.has("--changed") ? "changed" : "full",
+    verbose: args.has("--verbose")
+  };
+}
+export function printHelp() {
+  console.log("Usage: ./scripts/ingest.sh [--changed] [--verbose]");
+  console.log("");
+  console.log("Options:");
+  console.log("  --changed   Ingest only changed/untracked files when git is available.");
+  console.log("  --verbose   Print skipped files and additional diagnostics.");
+  console.log("  -h, --help  Show this help message.");
+}
+
+export function parsePositiveIntegerEnv(name, fallback) {
+  const rawValue = process.env[name];
+  if (!rawValue) {
+    return fallback;
+  }
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 1) {
+    return fallback;
+  }
+  return Math.floor(parsed);
+}
+
+export function parseNonNegativeIntegerEnv(name, fallback) {
+  const rawValue = process.env[name];
+  if (!rawValue) {
+    return fallback;
+  }
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return fallback;
+  }
+  return Math.floor(parsed);
+}
+
+export function isTruthyEnv(value) {
+  if (typeof value !== "string") {
+    return false;
+  }
+  const normalized = value.trim().toLowerCase();
+  return normalized !== "" && normalized !== "0" && normalized !== "false" && normalized !== "no" && normalized !== "off";
+}
+
+export function createIngestMemoryTrace() {
+  const enabled = isTruthyEnv(process.env.CORTEX_INGEST_TRACE_MEMORY);
+
+  return {
+    enabled,
+    checkpoint(label, counts = {}) {
+      if (!enabled) {
+        return;
+      }
+
+      const memory = process.memoryUsage();
+      process.stderr.write(
+        `${JSON.stringify({
+          type: "cortex.ingest.memory",
+          label,
+          rss_bytes: memory.rss,
+          rss_mb: Number((memory.rss / 1024 / 1024).toFixed(2)),
+          heap_used_bytes: memory.heapUsed,
+          external_bytes: memory.external,
+          counts
+        })}\n`
+      );
+    }
+  };
+}

package/scaffold/scripts/lib/ingest/chunks.mjs

@@ -0,0 +1,212 @@
+import fs from "node:fs";
+import path from "node:path";
+import { MAX_BODY_CHARS } from "./constants.mjs";
+import { checksum, normalizeWhitespace } from "./files.mjs";
+import { REPO_ROOT } from "./runtime-paths.mjs";
+
+export function chunkIdFor(filePath, chunk) {
+  const startLine = Number.isFinite(chunk.startLine) ? chunk.startLine : 0;
+  const endLine = Number.isFinite(chunk.endLine) ? chunk.endLine : startLine;
+  return `chunk:${filePath}:${chunk.name}:${startLine}-${endLine}`;
+}
+
+export function generateChunkDescription(chunk) {
+  const parts = [chunk.kind];
+  if (chunk.exported) parts.push("exported");
+  if (chunk.async) parts.push("async");
+  parts.push(chunk.signature);
+
+  if (typeof chunk.description === "string" && chunk.description.trim().length > 10) {
+    parts.push(normalizeWhitespace(chunk.description).slice(0, 200));
+  }
+
+  // Extract leading JSDoc/comment from body
+  // Match leading JSDoc (/** */), block (/* */) and line (//) comments
+  const commentMatch = chunk.body.match(/^(?:\s*(?:\/\*[\s\S]*?\*\/|\/\/[^\n]*)[\s\n]*)+/);
+  if (commentMatch) {
+    const cleaned = commentMatch[0]
+      .replace(/\/\*\*|\*\/|\*|\/\//g, "")
+      .replace(/\s+/g, " ").trim()
+      .slice(0, 200);
+    if (cleaned.length > 10) parts.push(cleaned);
+  }
+
+  return parts.join(". ") + ".";
+}
+
+export function generateModuleSummary(dir, files, exportNames, repoRoot = REPO_ROOT) {
+  // Check for README.md in directory
+  const readmePath = path.join(repoRoot, dir, "README.md");
+  if (fs.existsSync(readmePath)) {
+    try {
+      const content = fs.readFileSync(readmePath, "utf8");
+      // Skip first heading line, take first 300 chars
+      const lines = content.split(/\r?\n/);
+      const startIdx = lines.findIndex(l => !l.startsWith("#") && l.trim().length > 0);
+      if (startIdx >= 0) {
+        const excerpt = lines.slice(startIdx).join(" ").trim().slice(0, 300);
+        if (excerpt.length > 20) return excerpt;
+      }
+    } catch {
+      // fall through to auto-generated summary
+    }
+  }
+
+  const name = path.basename(dir);
+  const codeFiles = files.filter(f => f.kind === "CODE");
+  const docFiles = files.filter(f => f.kind !== "CODE");
+
+  const parts = [`Module ${name}`];
+  parts.push(`Contains ${files.length} files (${codeFiles.length} code, ${docFiles.length} docs)`);
+
+  // Detect common file extension pattern
+  const exts = new Set(codeFiles.map(f => path.extname(f.path).toLowerCase()));
+  if (exts.size === 1) {
+    const ext = [...exts][0];
+    const extNames = {
+      ".ts": "TypeScript",
+      ".tsx": "TypeScript React",
+      ".mts": "TypeScript ESM",
+      ".cts": "TypeScript CommonJS",
+      ".js": "JavaScript",
+      ".jsx": "JavaScript React",
+      ".mjs": "JavaScript ESM",
+      ".cjs": "JavaScript CommonJS"
+    };
+    if (extNames[ext]) parts.push(`${extNames[ext]} source files`);
+  }
+
+  if (exportNames.length > 0) {
+    parts.push(`Key exports: ${exportNames.slice(0, 5).join(", ")}`);
+  }
+
+  return parts.join(". ") + ".";
+}
+
+export function generateModules(fileRecords, chunkRecords) {
+  const dirFiles = new Map();
+  const dirChunks = new Map();
+  const fileById = new Map(fileRecords.map(f => [f.id, f]));
+
+  for (const file of fileRecords) {
+    const dir = path.dirname(file.path);
+    if (!dirFiles.has(dir)) dirFiles.set(dir, []);
+    dirFiles.get(dir).push(file);
+  }
+
+  for (const chunk of chunkRecords) {
+    if (!chunk.exported || isWindowChunkId(chunk.id)) continue;
+    const file = fileById.get(chunk.file_id);
+    if (!file) continue;
+    const dir = path.dirname(file.path);
+    if (!dirChunks.has(dir)) dirChunks.set(dir, []);
+    dirChunks.get(dir).push(chunk);
+  }
+
+  const modules = [];
+  const containsRelations = [];
+  const containsModuleRelations = [];
+  const exportsRelations = [];
+
+  const MIN_MODULE_FILES = 2;
+
+  for (const [dir, files] of dirFiles) {
+    if (files.length < MIN_MODULE_FILES) continue;
+
+    const exports = dirChunks.get(dir) || [];
+    const exportNames = [...new Set(exports.slice(0, 20).map(c => c.name))];
+    const moduleId = `module:${dir}`;
+
+    modules.push({
+      id: moduleId,
+      path: dir,
+      name: path.basename(dir),
+      summary: generateModuleSummary(dir, files, exportNames),
+      file_count: files.length,
+      exported_symbols: exportNames.join(", "),
+      updated_at: files.reduce((latest, f) => f.updated_at > latest ? f.updated_at : latest, ""),
+      source_of_truth: false,
+      trust_level: 75,
+      status: "active"
+    });
+
+    // CONTAINS: Module -> File
+    for (const file of files) {
+      containsRelations.push({ from: moduleId, to: file.id });
+    }
+
+    // EXPORTS: Module -> Chunk
+    for (const chunk of exports) {
+      exportsRelations.push({ from: moduleId, to: chunk.id });
+    }
+  }
+
+  // CONTAINS_MODULE: parent Module -> child Module
+  const moduleDirs = new Set(modules.map(m => m.path));
+  for (const dir of moduleDirs) {
+    const parent = path.dirname(dir);
+    if (parent !== dir && moduleDirs.has(parent)) {
+      containsModuleRelations.push({
+        from: `module:${parent}`,
+        to: `module:${dir}`
+      });
+    }
+  }
+
+  return { modules, containsRelations, containsModuleRelations, exportsRelations };
+}
+
+export function isWindowChunkId(chunkId) {
+  return typeof chunkId === "string" && chunkId.includes(":window:");
+}
+
+export function splitChunkIntoWindows(chunkRecord, options) {
+  const { windowLines, overlapLines, splitMinLines, maxWindows, chunkBody } = options;
+  const sourceBody = typeof chunkBody === "string" ? chunkBody : chunkRecord.body;
+  const lines = sourceBody.split(/\r?\n/);
+  const totalLines = lines.length;
+  if (totalLines < splitMinLines || totalLines <= windowLines) {
+    return [];
+  }
+
+  const windows = [];
+  const safeOverlap = Math.max(0, Math.min(overlapLines, windowLines - 1));
+  let start = 0;
+  let windowIndex = 1;
+
+  while (start < totalLines && windows.length < maxWindows) {
+    const isLastAllowedWindow = windows.length + 1 >= maxWindows;
+    const end = isLastAllowedWindow ? totalLines : Math.min(totalLines, start + windowLines);
+    const windowStartLine = chunkRecord.start_line + start;
+    const windowEndLine = chunkRecord.start_line + Math.max(0, end - 1);
+    const windowBody = lines.slice(start, end).join("\n");
+    const persistedBody = isLastAllowedWindow ? windowBody : windowBody.slice(0, MAX_BODY_CHARS);
+    windows.push({
+      id: `${chunkRecord.id}:window:${windowIndex}:${windowStartLine}-${windowEndLine}`,
+      file_id: chunkRecord.file_id,
+      name: `${chunkRecord.name}#window${windowIndex}`,
+      kind: chunkRecord.kind,
+      signature: `${chunkRecord.signature} [window ${windowIndex}]`,
+      body: persistedBody,
+      description: chunkRecord.description || "",
+      start_line: windowStartLine,
+      end_line: windowEndLine,
+      language: chunkRecord.language,
+      exported: chunkRecord.exported || false,
+      checksum: checksum(Buffer.from(windowBody)),
+      updated_at: chunkRecord.updated_at,
+      trust_level: chunkRecord.trust_level,
+      status: chunkRecord.status,
+      source_of_truth: chunkRecord.source_of_truth
+    });
+
+    if (end >= totalLines) {
+      break;
+    }
+
+    start = end - safeOverlap;
+    windowIndex += 1;
+  }
+
+  return windows;
+}

package/scaffold/scripts/lib/ingest/config.mjs

@@ -0,0 +1,120 @@
+import { RULE_KEYWORD_LIMIT, STOP_WORDS } from "./constants.mjs";
+import { tokenizeKeywords, uniqueSorted } from "./files.mjs";
+
+export function parseSourcePaths(configText) {
+  const sourcePaths = [];
+  const lines = configText.split(/\r?\n/);
+  let inSourcePaths = false;
+
+  for (const line of lines) {
+    if (!inSourcePaths && /^source_paths:\s*$/.test(line.trim())) {
+      inSourcePaths = true;
+      continue;
+    }
+
+    if (!inSourcePaths) {
+      continue;
+    }
+
+    const entryMatch = line.match(/^\s*-\s*(.+?)\s*$/);
+    if (entryMatch) {
+      const unquoted = entryMatch[1].replace(/^['"]|['"]$/g, "");
+      sourcePaths.push(unquoted);
+      continue;
+    }
+
+    if (line.trim() !== "" && !/^\s/.test(line)) {
+      break;
+    }
+  }
+
+  return sourcePaths;
+}
+
+export function parseRules(rulesText) {
+  const lines = rulesText.split(/\r?\n/);
+  const rules = [];
+  let current = null;
+
+  const pushCurrent = () => {
+    if (!current || !current.id) {
+      return;
+    }
+    rules.push({
+      id: current.id,
+      description: current.description ?? "",
+      priority: Number.isFinite(current.priority) ? current.priority : 0,
+      enforce: current.enforce === true
+    });
+  };
+
+  for (const line of lines) {
+    const idMatch = line.match(/^\s*-\s*id:\s*(.+?)\s*$/);
+    if (idMatch) {
+      pushCurrent();
+      current = { id: idMatch[1].replace(/^['"]|['"]$/g, "") };
+      continue;
+    }
+
+    if (!current) {
+      continue;
+    }
+
+    const descriptionMatch = line.match(/^\s*description:\s*(.+?)\s*$/);
+    if (descriptionMatch) {
+      current.description = descriptionMatch[1].replace(/^['"]|['"]$/g, "");
+      continue;
+    }
+
+    const priorityMatch = line.match(/^\s*priority:\s*(\d+)\s*$/);
+    if (priorityMatch) {
+      current.priority = Number(priorityMatch[1]);
+      continue;
+    }
+
+    const enforceMatch = line.match(/^\s*enforce:\s*(true|false)\s*$/i);
+    if (enforceMatch) {
+      current.enforce = enforceMatch[1].toLowerCase() === "true";
+    }
+  }
+
+  pushCurrent();
+  return rules;
+}
+
+export function normalizeRuleTokens(ruleRecord) {
+  const idParts = ruleRecord.id.split(/[._-]+/g);
+  const descriptionTokens = tokenizeKeywords(ruleRecord.body);
+  const rawKeywords = [...idParts, ...descriptionTokens];
+  const normalized = rawKeywords
+    .map((token) => token.toLowerCase().replace(/[^a-z0-9]/g, ""))
+    .filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
+
+  return uniqueSorted(normalized).slice(0, RULE_KEYWORD_LIMIT);
+}
+
+export function fileTokenSet(fileRecord) {
+  const tokenSource = `${fileRecord.path}\n${fileRecord.content.slice(0, 12000)}`;
+  return new Set(tokenizeKeywords(tokenSource));
+}
+
+export function collectRuleKeywordMatch(ruleKeywords, tokens, minimumMatches) {
+  const required = Math.min(minimumMatches, Math.max(1, ruleKeywords.length));
+  let count = 0;
+  const sample = [];
+
+  for (const keyword of ruleKeywords) {
+    if (!tokens.has(keyword)) {
+      continue;
+    }
+    count += 1;
+    if (sample.length < 5) {
+      sample.push(keyword);
+    }
+    if (count >= required && sample.length >= 5) {
+      break;
+    }
+  }
+
+  return { matched: count >= required, sample };
+}

package/scaffold/scripts/lib/ingest/constants.mjs

@@ -0,0 +1,222 @@
+export const SUPPORTED_TEXT_EXTENSIONS = new Set([
+  ".md",
+  ".mdx",
+  ".txt",
+  ".adoc",
+  ".rst",
+  ".yaml",
+  ".yml",
+  ".json",
+  ".toml",
+  ".csv",
+  ".ts",
+  ".tsx",
+  ".mts",
+  ".cts",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".cjs",
+  ".py",
+  ".go",
+  ".java",
+  ".cs",
+  ".vb",
+  ".sln",
+  ".vbproj",
+  ".csproj",
+  ".fsproj",
+  ".props",
+  ".targets",
+  ".config",
+  ".resx",
+  ".settings",
+  ".rb",
+  ".rs",
+  ".php",
+  ".swift",
+  ".kt",
+  ".sql",
+  ".sh",
+  ".bash",
+  ".zsh",
+  ".ps1",
+  ".c",
+  ".h",
+  ".cpp",
+  ".hpp",
+  ".cc",
+  ".hh",
+  ".bas",
+  ".cls",
+  ".frm",
+  ".ctl"
+]);
+
+export const LEGACY_DOTNET_METADATA_EXTENSIONS = new Set([
+  ".sln",
+  ".vbproj",
+  ".csproj",
+  ".fsproj",
+  ".props",
+  ".targets",
+  ".config",
+  ".resx",
+  ".settings"
+]);
+
+export const PROJECT_DEFINITION_EXTENSIONS = new Set([".sln", ".vbproj", ".csproj", ".fsproj", ".vcxproj"]);
+export const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings", ".md", ".mdx"]);
+
+export const CODE_FILE_EXTENSIONS = new Set([
+  ".ts",
+  ".tsx",
+  ".mts",
+  ".cts",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".cjs",
+  ".py",
+  ".go",
+  ".java",
+  ".cs",
+  ".vb",
+  ".rb",
+  ".rs",
+  ".php",
+  ".swift",
+  ".kt",
+  ".sql",
+  ".sh",
+  ".bash",
+  ".zsh",
+  ".ps1",
+  ".c",
+  ".h",
+  ".cpp",
+  ".hpp",
+  ".cc",
+  ".hh",
+  ".bas",
+  ".cls",
+  ".frm",
+  ".ctl"
+]);
+
+export const SQL_REFERENCE_SOURCE_EXTENSIONS = new Set([
+  ".vb",
+  ".cs",
+  ".config",
+  ".resx",
+  ".settings"
+]);
+export const NAMED_RESOURCE_REFERENCE_SOURCE_EXTENSIONS = new Set([".vb", ".cs"]);
+
+export const SQL_OBJECT_REFERENCE_PATTERNS = [
+  /\b(?:SqlCommand|OleDbCommand|OdbcCommand)\s*\(\s*"([^"\r\n]{2,200})"/gi,
+  /\bCommandText\s*=\s*"([^"\r\n]{2,500})"/gi,
+  /\bCommandType\s*=\s*(?:CommandType\.)?StoredProcedure[\s\S]{0,240}?"([^"\r\n]{2,200})"/gi,
+  /"([^"\r\n]{2,200})"[\s\S]{0,240}?\bCommandType\s*=\s*(?:CommandType\.)?StoredProcedure/gi
+];
+
+export const SQL_STRING_REFERENCE_PATTERNS = [
+  /\bexec(?:ute)?\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
+  /\bfrom\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
+  /\bjoin\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
+  /\bupdate\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
+  /\binsert\s+into\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
+  /\bdelete\s+from\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
+  /\bmerge\s+into\s+([#@]?[A-Za-z0-9_[\].]+)/gi
+];
+
+export const SQL_RESOURCE_KEY_PATTERNS = [
+  /\bMy\.Resources\.([A-Za-z_][A-Za-z0-9_]*)/g,
+  /\bResources\.([A-Za-z_][A-Za-z0-9_]*)/g,
+  /\bMy\.Settings\.([A-Za-z_][A-Za-z0-9_]*)/g,
+  /\b(?:[A-Za-z_][A-Za-z0-9_]*\.)?Settings\.Default\.([A-Za-z_][A-Za-z0-9_]*)/g,
+  /\bGetString\(\s*"([^"\r\n]+)"/g,
+  /\bGetObject\(\s*"([^"\r\n]+)"/g
+];
+export const CONFIG_KEY_REFERENCE_PATTERNS = [
+  /\bConfigurationManager\.ConnectionStrings\s*\[\s*"([^"\r\n]+)"\s*\]/g,
+  /\bConfigurationManager\.ConnectionStrings\s*\(\s*"([^"\r\n]+)"\s*\)/g,
+  /\bConfigurationManager\.AppSettings\s*\[\s*"([^"\r\n]+)"\s*\]/g,
+  /\bConfigurationManager\.AppSettings\s*\(\s*"([^"\r\n]+)"\s*\)/g,
+  /\bGetConnectionString\(\s*"([^"\r\n]+)"\s*\)/g,
+  /\bGetAppSetting\(\s*"([^"\r\n]+)"\s*\)/g
+];
+
+export const SKIP_DIRECTORIES = new Set([
+  ".git",
+  ".idea",
+  ".vscode",
+  "node_modules",
+  "bin",
+  "obj",
+  "dist",
+  "build",
+  "coverage",
+  ".next",
+  ".cache",
+  ".context"
+]);
+
+export const MAX_FILE_BYTES = 1024 * 1024;
+export const MAX_CONTENT_CHARS = 60000;
+export const MAX_BODY_CHARS = 12000;
+export const RULE_KEYWORD_LIMIT = 20;
+export const DEFAULT_CHUNK_WINDOW_LINES = 80;
+export const DEFAULT_CHUNK_OVERLAP_LINES = 16;
+export const DEFAULT_CHUNK_SPLIT_MIN_LINES = 120;
+export const DEFAULT_CHUNK_MAX_WINDOWS = 8;
+export const IMPORT_RESOLUTION_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".json"];
+export const IMPORT_RUNTIME_JS_EXTENSIONS = new Set([".js", ".jsx", ".mjs", ".cjs"]);
+export const IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"];
+export const CPP_IMPORT_RESOLUTION_EXTENSIONS = [".h", ".hh", ".hpp", ".c", ".cc", ".cpp"];
+
+export const STOP_WORDS = new Set([
+  "the",
+  "and",
+  "for",
+  "with",
+  "from",
+  "that",
+  "this",
+  "must",
+  "when",
+  "where",
+  "into",
+  "used",
+  "using",
+  "only",
+  "true",
+  "false",
+  "unless",
+  "should",
+  "global",
+  "active",
+  "rule",
+  "rules",
+  "data",
+  "file",
+  "files",
+  "code",
+  "docs",
+  "context",
+  "och",
+  "det",
+  "att",
+  "som",
+  "med",
+  "för",
+  "utan",
+  "eller",
+  "inte",
+  "ska",
+  "skall",
+  "måste",
+  "kan",
+  "vid",
+  "alla"
+]);