@danielblomma/cortex-mcp 2.4.0 → 2.4.2

package/scaffold/scripts/lib/ingest/files.mjs

@@ -0,0 +1,387 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { execSync } from "node:child_process";
+import {
+  CODE_FILE_EXTENSIONS,
+  CPP_IMPORT_RESOLUTION_EXTENSIONS,
+  IMPORT_RESOLUTION_EXTENSIONS,
+  IMPORT_RUNTIME_JS_EXTENSIONS,
+  IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS,
+  LEGACY_DOTNET_METADATA_EXTENSIONS,
+  SKIP_DIRECTORIES,
+  STOP_WORDS,
+  SUPPORTED_TEXT_EXTENSIONS
+} from "./constants.mjs";
+import { REPO_ROOT } from "./runtime-paths.mjs";
+
+export function ensureDirectory(directoryPath) {
+  fs.mkdirSync(directoryPath, { recursive: true });
+}
+
+export function isTextFile(relPath) {
+  const ext = path.extname(relPath).toLowerCase();
+  const base = path.basename(relPath).toLowerCase();
+  if (SUPPORTED_TEXT_EXTENSIONS.has(ext)) {
+    return true;
+  }
+
+  return base === "readme" || base.startsWith("readme.");
+}
+
+export function isBinaryBuffer(buffer) {
+  const scanLength = Math.min(buffer.length, 4000);
+  for (let index = 0; index < scanLength; index += 1) {
+    if (buffer[index] === 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+export function toPosixPath(value) {
+  return value.split(path.sep).join("/");
+}
+
+export function normalizeToken(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+
+export function tokenizeKeywords(value) {
+  return value
+    .toLowerCase()
+    .split(/[^a-z0-9]+/g)
+    .map((token) => token.trim())
+    .filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
+}
+
+export function uniqueSorted(values) {
+  return [...new Set(values)].sort();
+}
+
+export function walkDirectory(directoryPath, files) {
+  const entries = fs.readdirSync(directoryPath, { withFileTypes: true });
+  for (const entry of entries) {
+    const absolutePath = path.join(directoryPath, entry.name);
+    if (entry.isDirectory() && shouldSkipDirectory(absolutePath, entry.name)) {
+      continue;
+    }
+
+    if (entry.isDirectory()) {
+      walkDirectory(absolutePath, files);
+      continue;
+    }
+
+    if (entry.isFile()) {
+      files.add(absolutePath);
+    }
+  }
+}
+
+export function shouldSkipDirectory(absolutePath, entryName) {
+  if (entryName === "bin" && path.resolve(path.dirname(absolutePath)) === REPO_ROOT) {
+    return false;
+  }
+  return SKIP_DIRECTORIES.has(entryName);
+}
+
+export function normalizeSourcePrefix(sourcePath) {
+  const source = toPosixPath(sourcePath).replace(/^\.\/+/, "").replace(/\/+$/, "");
+  return source === "." ? "" : source;
+}
+
+export function normalizeRelativePath(relPath) {
+  return toPosixPath(relPath).replace(/^\.\/+/, "").replace(/\/+$/, "");
+}
+
+export function hasSkippedDirectorySegment(relPath) {
+  const parts = normalizeRelativePath(relPath).split("/").filter(Boolean);
+  return parts.some((part, index) => {
+    if (part === "bin" && index === 0) {
+      return false;
+    }
+    return SKIP_DIRECTORIES.has(part);
+  });
+}
+
+export function hasSourcePrefix(relPath, sourcePaths) {
+  const normalizedRelPath = normalizeRelativePath(relPath);
+  if (!normalizedRelPath || hasSkippedDirectorySegment(normalizedRelPath)) {
+    return false;
+  }
+  return sourcePaths.some((sourcePath) => {
+    const source = normalizeSourcePrefix(sourcePath);
+    return source === "" || normalizedRelPath === source || normalizedRelPath.startsWith(`${source}/`);
+  });
+}
+
+export function pushImportResolutionCandidate(candidates, seenCandidates, candidatePath) {
+  if (!seenCandidates.has(candidatePath)) {
+    seenCandidates.add(candidatePath);
+    candidates.push(candidatePath);
+  }
+}
+
+export function isCppLikeFilePath(filePath) {
+  return [".c", ".h", ".cc", ".cpp", ".hh", ".hpp"].includes(path.posix.extname(filePath).toLowerCase());
+}
+
+export function resolveRelativeImportTargetId(filePath, importPath, indexedFileIds) {
+  const isCppLike = isCppLikeFilePath(filePath);
+  const isRelativeImport = importPath.startsWith(".");
+  const isLocalCppInclude =
+    isCppLike && !path.posix.isAbsolute(importPath) && !/^[A-Za-z]:[\\/]/.test(importPath);
+
+  if (!isRelativeImport && !isLocalCppInclude) {
+    return null;
+  }
+
+  const basePath = path.posix.normalize(path.posix.join(path.posix.dirname(filePath), importPath));
+  const candidates = [];
+  const seenCandidates = new Set();
+  pushImportResolutionCandidate(candidates, seenCandidates, basePath);
+
+  if (path.posix.extname(basePath) === "") {
+    const extensions = isCppLike ? CPP_IMPORT_RESOLUTION_EXTENSIONS : IMPORT_RESOLUTION_EXTENSIONS;
+    for (const extension of extensions) {
+      pushImportResolutionCandidate(candidates, seenCandidates, `${basePath}${extension}`);
+    }
+    if (!isCppLike) {
+      for (const extension of IMPORT_RESOLUTION_EXTENSIONS) {
+        pushImportResolutionCandidate(candidates, seenCandidates, path.posix.join(basePath, `index${extension}`));
+      }
+    }
+  } else if (IMPORT_RUNTIME_JS_EXTENSIONS.has(path.posix.extname(basePath))) {
+    const extension = path.posix.extname(basePath);
+    const stemPath = basePath.slice(0, -extension.length);
+    for (const candidateExtension of IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS) {
+      pushImportResolutionCandidate(candidates, seenCandidates, `${stemPath}${candidateExtension}`);
+    }
+  }
+
+  for (const candidate of candidates) {
+    const targetFileId = `file:${candidate}`;
+    if (indexedFileIds.has(targetFileId)) {
+      return targetFileId;
+    }
+  }
+
+  return null;
+}
+
+export function getGitChanges() {
+  try {
+    const output = execSync("git status --porcelain", {
+      cwd: REPO_ROOT,
+      stdio: ["ignore", "pipe", "ignore"],
+      encoding: "utf8"
+    });
+
+    const changed = new Set();
+    const deleted = new Set();
+
+    for (const line of output.split(/\r?\n/)) {
+      if (!line) continue;
+      const status = line.slice(0, 2);
+      const payload = line.slice(3).trim();
+      if (!payload) continue;
+
+      if (payload.includes(" -> ")) {
+        const [fromPath, toPath] = payload.split(" -> ");
+        deleted.add(path.resolve(REPO_ROOT, fromPath));
+        changed.add(path.resolve(REPO_ROOT, toPath));
+        continue;
+      }
+
+      const absolutePath = path.resolve(REPO_ROOT, payload);
+      if (status.includes("D")) {
+        deleted.add(absolutePath);
+      } else {
+        changed.add(absolutePath);
+      }
+    }
+
+    return {
+      changed: [...changed],
+      deleted: [...deleted]
+    };
+  } catch {
+    return {
+      changed: [],
+      deleted: []
+    };
+  }
+}
+
+export function collectCandidateFiles(sourcePaths, mode) {
+  const candidates = new Set();
+  const deletedRelPaths = new Set();
+
+  if (mode === "changed") {
+    const gitChanges = getGitChanges();
+    if (gitChanges.changed.length > 0 || gitChanges.deleted.length > 0) {
+      for (const absolutePath of gitChanges.changed) {
+        if (!fs.existsSync(absolutePath)) {
+          continue;
+        }
+
+        const stats = fs.statSync(absolutePath);
+        if (stats.isFile()) {
+          const relPath = toPosixPath(path.relative(REPO_ROOT, absolutePath));
+          if (hasSourcePrefix(relPath, sourcePaths)) {
+            candidates.add(absolutePath);
+          }
+          continue;
+        }
+
+        if (stats.isDirectory()) {
+          const nestedFiles = new Set();
+          walkDirectory(absolutePath, nestedFiles);
+          for (const nestedPath of nestedFiles) {
+            const nestedRelPath = toPosixPath(path.relative(REPO_ROOT, nestedPath));
+            if (hasSourcePrefix(nestedRelPath, sourcePaths)) {
+              candidates.add(nestedPath);
+            }
+          }
+        }
+      }
+
+      for (const deletedPath of gitChanges.deleted) {
+        const relPath = toPosixPath(path.relative(REPO_ROOT, deletedPath));
+        if (hasSourcePrefix(relPath, sourcePaths)) {
+          deletedRelPaths.add(relPath);
+        }
+      }
+
+      return {
+        candidates,
+        incrementalMode: true,
+        deletedRelPaths: [...deletedRelPaths]
+      };
+    }
+  }
+
+  for (const sourcePath of sourcePaths) {
+    if (hasSkippedDirectorySegment(sourcePath)) {
+      continue;
+    }
+    const absoluteSourcePath = path.resolve(REPO_ROOT, sourcePath);
+    if (!fs.existsSync(absoluteSourcePath)) {
+      continue;
+    }
+
+    const stats = fs.statSync(absoluteSourcePath);
+    if (stats.isFile()) {
+      candidates.add(absoluteSourcePath);
+      continue;
+    }
+
+    if (stats.isDirectory()) {
+      walkDirectory(absoluteSourcePath, candidates);
+    }
+  }
+
+  return {
+    candidates,
+    incrementalMode: false,
+    deletedRelPaths: []
+  };
+}
+
+export function detectKind(relPath) {
+  const lower = relPath.toLowerCase();
+  const ext = path.extname(lower);
+  const isAdrPath =
+    /(^|\/)(adr|adrs|decisions)(\/|$)/.test(lower) ||
+    /(^|\/)adr[-_ ]?\d+/.test(path.basename(lower));
+
+  if (isAdrPath) {
+    return "ADR";
+  }
+
+  if (
+    lower.startsWith("docs/") ||
+    ext === ".md" ||
+    ext === ".mdx" ||
+    ext === ".txt" ||
+    ext === ".adoc" ||
+    ext === ".rst"
+  ) {
+    return "DOC";
+  }
+
+  if (LEGACY_DOTNET_METADATA_EXTENSIONS.has(ext) || !CODE_FILE_EXTENSIONS.has(ext)) {
+    return "DOC";
+  }
+
+  return "CODE";
+}
+
+export function trustLevelForKind(kind) {
+  if (kind === "ADR") return 95;
+  if (kind === "CODE") return 80;
+  return 70;
+}
+
+export function checksum(buffer) {
+  return crypto.createHash("sha256").update(buffer).digest("hex");
+}
+
+export function normalizeWhitespace(value) {
+  return value.replace(/\s+/g, " ").trim();
+}
+
+export function extractTitle(content, fallbackTitle) {
+  const lines = content.split(/\r?\n/);
+  for (const line of lines) {
+    const match = line.match(/^#\s+(.+)\s*$/);
+    if (match) return match[1].trim();
+  }
+
+  return fallbackTitle;
+}
+
+export function parseDecisionDate(content, fallbackDate) {
+  const datePatterns = [
+    /^\s*date:\s*["']?(\d{4}-\d{2}-\d{2})["']?\s*$/im,
+    /^\s*decision[_\s-]*date:\s*["']?(\d{4}-\d{2}-\d{2})["']?\s*$/im
+  ];
+
+  for (const pattern of datePatterns) {
+    const match = content.match(pattern);
+    if (match && !Number.isNaN(Date.parse(match[1]))) {
+      return match[1];
+    }
+  }
+
+  return fallbackDate.slice(0, 10);
+}
+
+export function adrTokens(adrRecord) {
+  const fileBase = path.basename(adrRecord.path).replace(path.extname(adrRecord.path), "");
+  const tokens = new Set([
+    normalizeToken(adrRecord.id),
+    normalizeToken(fileBase),
+    normalizeToken(adrRecord.title)
+  ]);
+
+  const numberMatch = fileBase.match(/(\d+)/);
+  if (numberMatch) {
+    tokens.add(normalizeToken(`adr-${numberMatch[1]}`));
+    tokens.add(normalizeToken(numberMatch[1]));
+  }
+
+  return [...tokens].filter(Boolean);
+}
+
+export function findSupersedesReferences(content) {
+  const refs = new Set();
+  const pattern = /(?:supersedes|ersätter)\s*[:\-]?\s*([A-Za-z0-9._/-]+)/gi;
+  let match;
+  while ((match = pattern.exec(content)) !== null) {
+    refs.add(match[1]);
+  }
+
+  return [...refs];
+}

package/scaffold/scripts/lib/ingest/incremental-state.mjs

@@ -0,0 +1,131 @@
+import path from "node:path";
+import { readJsonlSafe } from "./io.mjs";
+import { relationKey } from "./relations.mjs";
+import { CACHE_DIR } from "./runtime-paths.mjs";
+
+export function removeChunkStateForFile(fileId, chunkRecordMap, definesRelationMap, callsRelationMap, importsRelationMap, callsSqlRelationMap) {
+  const removedChunkIds = new Set();
+
+  for (const [chunkId, chunkRecord] of chunkRecordMap.entries()) {
+    if (chunkRecord.file_id === fileId) {
+      removedChunkIds.add(chunkId);
+      chunkRecordMap.delete(chunkId);
+    }
+  }
+
+  if (removedChunkIds.size === 0) {
+    return;
+  }
+
+  for (const [key, relation] of definesRelationMap.entries()) {
+    if (relation.from === fileId || removedChunkIds.has(relation.to)) {
+      definesRelationMap.delete(key);
+    }
+  }
+
+  for (const [key, relation] of callsRelationMap.entries()) {
+    if (removedChunkIds.has(relation.from) || removedChunkIds.has(relation.to)) {
+      callsRelationMap.delete(key);
+    }
+  }
+
+  for (const [key, relation] of importsRelationMap.entries()) {
+    if (removedChunkIds.has(relation.from)) {
+      importsRelationMap.delete(key);
+    }
+  }
+
+  for (const [key, relation] of callsSqlRelationMap.entries()) {
+    if (relation.from === fileId || removedChunkIds.has(relation.to)) {
+      callsSqlRelationMap.delete(key);
+    }
+  }
+}
+
+export function hydrateIncrementalChunkState(fileRecords) {
+  const fileIdSet = new Set(fileRecords.map((record) => record.id));
+  const chunkRecordMap = new Map();
+  const definesRelationMap = new Map();
+  const callsRelationMap = new Map();
+  const importsRelationMap = new Map();
+  const callsSqlRelationMap = new Map();
+
+  for (const record of readJsonlSafe(path.join(CACHE_DIR, "entities.chunk.jsonl"))) {
+    if (!record || typeof record !== "object") continue;
+    const chunkId = String(record.id ?? "");
+    const fileId = String(record.file_id ?? "");
+    if (!chunkId || !fileIdSet.has(fileId)) {
+      continue;
+    }
+    chunkRecordMap.set(chunkId, {
+      ...record,
+      id: chunkId,
+      file_id: fileId
+    });
+  }
+
+  const chunkIdSet = new Set(chunkRecordMap.keys());
+
+  for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.defines.jsonl"))) {
+    if (!record || typeof record !== "object") continue;
+    const from = String(record.from ?? "");
+    const to = String(record.to ?? "");
+    if (!fileIdSet.has(from) || !chunkIdSet.has(to)) {
+      continue;
+    }
+    definesRelationMap.set(relationKey(from, to), { from, to });
+  }
+
+  for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.calls.jsonl"))) {
+    if (!record || typeof record !== "object") continue;
+    const from = String(record.from ?? "");
+    const to = String(record.to ?? "");
+    const callType = String(record.call_type ?? "direct");
+    if (!chunkIdSet.has(from) || !chunkIdSet.has(to)) {
+      continue;
+    }
+    callsRelationMap.set(relationKey(from, to, callType), {
+      from,
+      to,
+      call_type: callType
+    });
+  }
+
+  for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.imports.jsonl"))) {
+    if (!record || typeof record !== "object") continue;
+    const from = String(record.from ?? "");
+    const to = String(record.to ?? "");
+    const importName = String(record.import_name ?? "");
+    if (!chunkIdSet.has(from) || !fileIdSet.has(to)) {
+      continue;
+    }
+    importsRelationMap.set(relationKey(from, to, importName), {
+      from,
+      to,
+      import_name: importName
+    });
+  }
+
+  for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.calls_sql.jsonl"))) {
+    if (!record || typeof record !== "object") continue;
+    const from = String(record.from ?? "");
+    const to = String(record.to ?? "");
+    const note = String(record.note ?? "");
+    if (!fileIdSet.has(from) || !chunkIdSet.has(to)) {
+      continue;
+    }
+    callsSqlRelationMap.set(relationKey(from, to, note), {
+      from,
+      to,
+      note
+    });
+  }
+
+  return {
+    chunkRecordMap,
+    definesRelationMap,
+    callsRelationMap,
+    importsRelationMap,
+    callsSqlRelationMap
+  };
+}

package/scaffold/scripts/lib/ingest/io.mjs

@@ -0,0 +1,78 @@
+import fs from "node:fs";
+
+export function writeJsonl(filePath, records) {
+  const fd = fs.openSync(filePath, "w");
+  try {
+    for (const record of records) {
+      fs.writeSync(fd, `${JSON.stringify(record)}\n`, undefined, "utf8");
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+export function stageJsonl(filePath, records) {
+  const stagedPath = `${filePath}.tmp-${process.pid}-${Date.now()}`;
+  writeJsonl(stagedPath, records);
+  return {
+    commit() {
+      fs.renameSync(stagedPath, filePath);
+    },
+    discard() {
+      fs.rmSync(stagedPath, { force: true });
+    }
+  };
+}
+
+export function countFileContentRecords(fileRecords) {
+  let count = 0;
+  for (const fileRecord of fileRecords) {
+    if (Object.prototype.hasOwnProperty.call(fileRecord, "content")) {
+      count += 1;
+    }
+  }
+  return count;
+}
+
+export function sanitizeTsvCell(value) {
+  if (value === null || value === undefined) return "";
+  return String(value).replace(/\t/g, " ").replace(/\r?\n/g, " ");
+}
+
+export function writeTsv(filePath, headers, rows) {
+  const fd = fs.openSync(filePath, "w");
+  try {
+    fs.writeSync(fd, `${headers.join("\t")}\n`, undefined, "utf8");
+    for (const row of rows) {
+      fs.writeSync(fd, `${row.map((value) => sanitizeTsvCell(value)).join("\t")}\n`, undefined, "utf8");
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+export function* mapRows(records, project) {
+  for (const record of records) {
+    yield project(record);
+  }
+}
+
+export function readJsonlSafe(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return [];
+  }
+
+  return fs
+    .readFileSync(filePath, "utf8")
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter((record) => record !== null);
+}

package/scaffold/scripts/lib/ingest/main.mjs

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+import {
+  createIngestPipelineState,
+  runCacheWriteStage,
+  runDatabaseWriteStage,
+  runFileCacheStagingStage,
+  runManifestCompletionStage,
+  runMaterializationStage,
+  runParseStage,
+  runScanHydrationStage,
+  runTokenMatchingStage
+} from "./pipeline-stages.mjs";
+import {
+  buildChunkAliasIndexes,
+  buildSqlResourceReferenceMap,
+  extractSqlObjectReferencesFromContent,
+  generateConfigIncludeRelations,
+  generateConfigTransformKeyRelations,
+  generateMachineConfigRelations,
+  generateConfigTransformRelations,
+  generateNamedResourceRelations,
+  generateSectionHandlerRelations
+} from "./relations.mjs";
+import {
+  detectKind,
+  resolveRelativeImportTargetId
+} from "./files.mjs";
+import {
+  generateChunkDescription,
+  generateModuleSummary,
+  generateModules
+} from "./chunks.mjs";
+import { generateProjects } from "./projects.mjs";
+import {
+  getChunkParserForExtension,
+  initializeParserComposition
+} from "./parser-composition.mjs";
+import {
+  parseFilesInWorkers,
+  resolveIngestWorkerCount
+} from "./workers.mjs";
+
+async function main() {
+  await initializeParserComposition();
+  const state = createIngestPipelineState();
+  runScanHydrationStage(state);
+  await runParseStage(state);
+  runMaterializationStage(state);
+  runFileCacheStagingStage(state);
+  runTokenMatchingStage(state);
+  runCacheWriteStage(state);
+  runDatabaseWriteStage(state);
+  runManifestCompletionStage(state);
+}
+
+export {
+  buildChunkAliasIndexes,
+  buildSqlResourceReferenceMap,
+  detectKind,
+  extractSqlObjectReferencesFromContent,
+  generateChunkDescription,
+  generateConfigIncludeRelations,
+  generateConfigTransformKeyRelations,
+  generateMachineConfigRelations,
+  generateConfigTransformRelations,
+  generateModuleSummary,
+  generateModules,
+  generateNamedResourceRelations,
+  generateProjects,
+  generateSectionHandlerRelations,
+  getChunkParserForExtension,
+  main,
+  parseFilesInWorkers,
+  resolveIngestWorkerCount,
+  resolveRelativeImportTargetId
+};