@swarmvaultai/engine 0.1.31 → 0.1.33

package/README.md

@@ -174,11 +174,13 @@ This matters because many "OpenAI-compatible" backends only implement part of th
 - `addInput(rootDir, input, { author, contributor })` captures supported URLs into normalized markdown before ingesting them, or falls back to generic URL ingest
 - `ingestDirectory(rootDir, inputDir, { repoRoot, include, exclude, maxFiles, gitignore, extractClasses })` recursively ingests a local directory as a repo-aware code/content source tree
 - `importInbox(rootDir, inputDir?)` recursively imports supported inbox files plus markdown and HTML browser-clipper style bundles
-- JavaScript, TypeScript, Python, Go, Rust, Java, C#, C, C++, PHP, Ruby, and PowerShell inputs are treated as code sources and compiled into both source pages and `wiki/code/` module pages
+- JavaScript, TypeScript, Python, Go, Rust, Java, C#, C, C++, PHP, Ruby, PowerShell, Kotlin, and Scala inputs are treated as code sources and compiled into both source pages and `wiki/code/` module pages
+- `.rst` and `.rest` inputs are treated as first-class text sources with lightweight heading and directive normalization before analysis
 - code manifests can carry `repoRelativePath`, and compile writes `state/code-index.json` so local imports can resolve across an ingested repo tree
 - repo-aware manifests, graph nodes, and graph pages can also carry `sourceClass` so first-party, third-party, resource, and generated material can be filtered and reported separately
 - HTML and markdown URL ingests localize remote image references into `raw/assets/<sourceId>/` by default and rewrite the stored markdown to local relative paths
 - PDF and DOCX ingests now write extracted-text and metadata sidecars under `state/extracts/`, and image ingest keeps the same sidecar model for vision extraction
+- Tree-sitter-backed languages now verify runtime and grammar compatibility per language; failures stay local to the affected source and surface as diagnostics instead of aborting the whole compile
 
 ### Compile + Query
 
@@ -206,6 +208,7 @@ This matters because many "OpenAI-compatible" backends only implement part of th
 - `getWatchStatus(rootDir)` reads the latest watch-status artifact plus pending semantic refresh entries
 - `syncTrackedRepos(rootDir)` refreshes previously ingested repo roots, updates changed manifests, and removes deleted repo manifests
 - `syncTrackedReposForWatch(rootDir)` is the repo-watch sync path that defers non-code semantic refresh into `state/watch/`
+- large ingest and compile passes emit low-noise progress on TTYs, and report presentation rolls up tiny fragmented communities without mutating the canonical graph artifact
 - `installGitHooks(rootDir)`, `uninstallGitHooks(rootDir)`, and `getGitHookStatus(rootDir)` manage local `post-commit` and `post-checkout` hook blocks for the nearest git repository
 - `installAgent(rootDir, agent, { hook })` writes agent instructions and returns the primary `target`, all touched `targets`, and optional merge warnings for agents such as Aider
 - `lintVault(rootDir, options)` runs structural lint, optional deep lint, optional contradiction-only filtering through `{ conflicts: true }`, and optional web-augmented evidence gathering

package/dist/index.d.ts

@@ -57,7 +57,7 @@ type ApprovalChangeType = "create" | "update" | "delete" | "promote";
 type SourceKind = "markdown" | "text" | "pdf" | "image" | "html" | "docx" | "binary" | "code";
 type SourceCaptureType = "arxiv" | "doi" | "tweet" | "article" | "url";
 type SourceClass = "first_party" | "third_party" | "resource" | "generated";
-type CodeLanguage = "javascript" | "jsx" | "typescript" | "tsx" | "python" | "go" | "rust" | "java" | "csharp" | "c" | "cpp" | "php" | "ruby" | "powershell";
+type CodeLanguage = "javascript" | "jsx" | "typescript" | "tsx" | "python" | "go" | "rust" | "java" | "kotlin" | "scala" | "csharp" | "c" | "cpp" | "php" | "ruby" | "powershell";
 type CodeSymbolKind = "function" | "class" | "interface" | "type_alias" | "enum" | "variable" | "struct" | "trait";
 type OrchestrationRole = "research" | "audit" | "context" | "safety";
 declare const webSearchProviderTypeSchema: z.ZodEnum<{
@@ -966,6 +966,12 @@ interface GraphReportArtifact {
         path?: string;
         title?: string;
     }>;
+    fragmentedCommunityRollup?: {
+        totalCommunities: number;
+        rolledUpCount: number;
+        rolledUpNodes: number;
+        exampleLabels: string[];
+    };
     surprisingConnections: Array<{
         id: string;
         sourceNodeId: string;

package/dist/index.js

@@ -1741,7 +1741,9 @@ import fs5 from "fs/promises";
 import { createRequire } from "module";
 import path5 from "path";
 var require2 = createRequire(import.meta.url);
-var TREE_SITTER_PACKAGE_ROOT = path5.dirname(path5.dirname(require2.resolve("@vscode/tree-sitter-wasm")));
+var TREE_SITTER_RUNTIME_PACKAGE = "@vscode/tree-sitter-wasm";
+var TREE_SITTER_EXTRA_GRAMMARS_PACKAGE = "tree-sitter-wasms";
+var packageRootCache = /* @__PURE__ */ new Map();
 var RATIONALE_MARKERS = ["NOTE:", "IMPORTANT:", "HACK:", "WHY:", "RATIONALE:"];
 function stripKnownCommentPrefix(line) {
   let next = line.trim();
@@ -1755,21 +1757,41 @@ function stripKnownCommentPrefix(line) {
 var treeSitterModulePromise;
 var treeSitterInitPromise;
 var languageCache = /* @__PURE__ */ new Map();
-var grammarFileByLanguage = {
-  python: "tree-sitter-python.wasm",
-  go: "tree-sitter-go.wasm",
-  rust: "tree-sitter-rust.wasm",
-  java: "tree-sitter-java.wasm",
-  csharp: "tree-sitter-c-sharp.wasm",
-  c: "tree-sitter-cpp.wasm",
-  cpp: "tree-sitter-cpp.wasm",
-  php: "tree-sitter-php.wasm",
-  ruby: "tree-sitter-ruby.wasm",
-  powershell: "tree-sitter-powershell.wasm"
+var grammarAssetByLanguage = {
+  python: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-python.wasm" },
+  go: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-go.wasm" },
+  rust: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-rust.wasm" },
+  java: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-java.wasm" },
+  kotlin: { packageName: TREE_SITTER_EXTRA_GRAMMARS_PACKAGE, relativePath: "out/tree-sitter-kotlin.wasm" },
+  scala: { packageName: TREE_SITTER_EXTRA_GRAMMARS_PACKAGE, relativePath: "out/tree-sitter-scala.wasm" },
+  csharp: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-c-sharp.wasm" },
+  c: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-cpp.wasm" },
+  cpp: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-cpp.wasm" },
+  php: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-php.wasm" },
+  ruby: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-ruby.wasm" },
+  powershell: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-powershell.wasm" }
 };
+function resolvePackageRoot(packageName) {
+  const cached = packageRootCache.get(packageName);
+  if (cached) {
+    return cached;
+  }
+  let resolved;
+  try {
+    resolved = path5.dirname(require2.resolve(`${packageName}/package.json`));
+  } catch {
+    resolved = path5.dirname(path5.dirname(require2.resolve(packageName)));
+  }
+  packageRootCache.set(packageName, resolved);
+  return resolved;
+}
+function grammarAssetPath(language) {
+  const asset = grammarAssetByLanguage[language];
+  return path5.join(resolvePackageRoot(asset.packageName), asset.relativePath);
+}
 async function getTreeSitterModule() {
   if (!treeSitterModulePromise) {
-    treeSitterModulePromise = import(require2.resolve("@vscode/tree-sitter-wasm")).then(
+    treeSitterModulePromise = import(require2.resolve(TREE_SITTER_RUNTIME_PACKAGE)).then(
       (module) => module.default ?? module
     );
   }
@@ -1777,8 +1799,9 @@ async function getTreeSitterModule() {
 }
 async function ensureTreeSitterInit(module) {
   if (!treeSitterInitPromise) {
+    const runtimeRoot = resolvePackageRoot(TREE_SITTER_RUNTIME_PACKAGE);
     treeSitterInitPromise = module.Parser.init({
-      locateFile: () => path5.join(TREE_SITTER_PACKAGE_ROOT, "wasm", "tree-sitter.wasm")
+      locateFile: () => path5.join(runtimeRoot, "wasm", "tree-sitter.wasm")
     });
   }
   return treeSitterInitPromise;
@@ -1791,7 +1814,7 @@ async function loadLanguage(language) {
   const loader = (async () => {
     const module = await getTreeSitterModule();
     await ensureTreeSitterInit(module);
-    const bytes = await fs5.readFile(path5.join(TREE_SITTER_PACKAGE_ROOT, "wasm", grammarFileByLanguage[language]));
+    const bytes = await fs5.readFile(grammarAssetPath(language));
     return module.Language.load(bytes);
   })();
   languageCache.set(language, loader);
@@ -1805,7 +1828,7 @@ function normalizeSymbolReference(value) {
   return lastSegment.replace(/[,:;]+$/g, "").trim();
 }
 function stripCodeExtension(filePath) {
-  return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
+  return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|kt|kts|scala|sc|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
 }
 function manifestModuleName(manifest, language) {
   const repoPath = manifest.repoRelativePath ?? path5.basename(manifest.originalPath ?? manifest.storedPath);
@@ -2044,6 +2067,7 @@ function extractIdentifier(node) {
   }
   if ([
     "identifier",
+    "simple_identifier",
     "field_identifier",
     "type_identifier",
     "name",
@@ -2057,6 +2081,7 @@ function extractIdentifier(node) {
   const preferred = node.childForFieldName("name") ?? node.namedChildren.find(
     (child) => child && [
       "identifier",
+      "simple_identifier",
       "field_identifier",
       "type_identifier",
       "name",
@@ -2119,6 +2144,25 @@ function diagnosticsFromTree(rootNode) {
   visit(rootNode);
   return diagnostics.slice(0, 20);
 }
+function treeSitterCompatibilityMessage(language, error) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (typeof error === "object" && error && "code" in error && error.code === "MODULE_NOT_FOUND") {
+    return `Tree-sitter runtime support for ${language} is unavailable. Reinstall @swarmvaultai/engine so the packaged parser runtime is present.`;
+  }
+  if (typeof error === "object" && error && "code" in error && error.code === "ENOENT") {
+    return `Missing tree-sitter grammar asset for ${language}. Reinstall @swarmvaultai/engine so the packaged grammar files are present.`;
+  }
+  return `Tree-sitter support for ${language} could not load: ${truncate(normalizeWhitespace(message), 220)}.`;
+}
+function treeSitterCompatibilityDiagnostic(language, error) {
+  return {
+    code: 9010,
+    category: "error",
+    message: treeSitterCompatibilityMessage(language, error),
+    line: 1,
+    column: 1
+  };
+}
 function parsePythonImportStatement(text) {
   const match = text.trim().match(/^import\s+(.+)$/);
   if (!match) {
@@ -2188,6 +2232,36 @@ function parseJavaImport(text) {
     reExport: false
   };
 }
+function parseKotlinImport(text) {
+  const cleaned = text.trim().replace(/^import\s+/, "");
+  if (!cleaned) {
+    return void 0;
+  }
+  const aliasMatch = cleaned.match(/^(.+?)\s+as\s+([A-Za-z_]\w*)$/);
+  const specifier = (aliasMatch ? aliasMatch[1] : cleaned).trim();
+  if (!specifier) {
+    return void 0;
+  }
+  return {
+    specifier,
+    importedSymbols: [],
+    namespaceImport: aliasMatch?.[2],
+    isExternal: !specifier.startsWith("."),
+    reExport: false
+  };
+}
+function parseScalaImport(text) {
+  const cleaned = text.trim().replace(/^import\s+/, "");
+  if (!cleaned) {
+    return [];
+  }
+  return cleaned.split(",").map((item) => item.trim()).filter(Boolean).map((item) => ({
+    specifier: item.replace(/\s*=>\s*/g, " => "),
+    importedSymbols: [],
+    isExternal: !item.startsWith("."),
+    reExport: false
+  }));
+}
 function parseCSharpUsing(text) {
   const aliasMatch = text.trim().match(/^using\s+([A-Za-z_]\w*)\s*=\s*([^;]+);$/);
   if (aliasMatch) {
@@ -2280,6 +2354,38 @@ function parsePowerShellImport(commandNode) {
   }
   return void 0;
 }
+function keywordVisible(text, hiddenKeywords) {
+  return !hiddenKeywords.some((keyword) => new RegExp(`\\b${keyword}\\b`).test(text));
+}
+function declarationVisible(node, hiddenKeywords) {
+  const modifierText = nodeText(findNamedChild(node, "modifiers") ?? node.childForFieldName("modifiers"));
+  return modifierText ? keywordVisible(modifierText, hiddenKeywords) : true;
+}
+function kotlinClassKind(text) {
+  const trimmed = text.trimStart();
+  if (trimmed.startsWith("interface ")) {
+    return "interface";
+  }
+  if (trimmed.startsWith("enum class ")) {
+    return "enum";
+  }
+  return "class";
+}
+function scalaDefinitionKind(node) {
+  if (node.type === "trait_definition") {
+    return "trait";
+  }
+  if (node.type === "class_definition") {
+    return /\bcase\s+class\b/.test(node.text) ? "class" : "class";
+  }
+  if (node.type === "object_definition") {
+    return "class";
+  }
+  if (node.type === "function_definition") {
+    return "function";
+  }
+  return void 0;
+}
 function pythonCodeAnalysis(manifest, rootNode, diagnostics) {
   const imports = [];
   const draftSymbols = [];
@@ -2533,6 +2639,181 @@ function javaCodeAnalysis(manifest, rootNode, diagnostics) {
     namespace: packageName
   });
 }
+function kotlinCodeAnalysis(manifest, rootNode, diagnostics) {
+  const imports = [];
+  const draftSymbols = [];
+  const exportLabels = [];
+  let packageName;
+  const pushBodyFunctions = (bodyNode, scopeName) => {
+    if (!bodyNode) {
+      return;
+    }
+    for (const child of bodyNode.namedChildren) {
+      if (!child || child.type !== "function_declaration") {
+        continue;
+      }
+      const functionName = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "simple_identifier"));
+      if (!functionName) {
+        continue;
+      }
+      const exported = declarationVisible(child, ["private", "internal", "protected"]);
+      const symbolName = scopeName ? `${scopeName}.${functionName}` : functionName;
+      draftSymbols.push({
+        name: symbolName,
+        kind: "function",
+        signature: singleLineSignature(child.text),
+        exported,
+        callNames: [],
+        extendsNames: [],
+        implementsNames: [],
+        bodyText: nodeText(child.childForFieldName("body") ?? findNamedChild(child, "function_body"))
+      });
+      if (exported) {
+        exportLabels.push(symbolName);
+      }
+    }
+  };
+  for (const child of rootNode.namedChildren) {
+    if (!child) {
+      continue;
+    }
+    if (child.type === "package_header") {
+      packageName = nodeText(findNamedChild(child, "identifier") ?? child.namedChildren.at(0) ?? null) || packageName;
+      continue;
+    }
+    if (child.type === "import_list") {
+      for (const importNode of child.descendantsOfType("import_header").filter((item) => item !== null)) {
+        const parsed = parseKotlinImport(importNode.text);
+        if (parsed) {
+          imports.push(parsed);
+        }
+      }
+      continue;
+    }
+    if (child.type === "function_declaration") {
+      pushBodyFunctions({
+        ...child,
+        namedChildren: [child]
+      });
+      continue;
+    }
+    if (child.type !== "class_declaration" && child.type !== "object_declaration") {
+      continue;
+    }
+    const name = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "type_identifier"));
+    if (!name) {
+      continue;
+    }
+    const kind = child.type === "object_declaration" ? "class" : kotlinClassKind(child.text);
+    const delegationNames = uniqueBy(
+      child.namedChildren.filter((item) => item !== null && item.type === "delegation_specifier").flatMap((item) => descendantTypeNames(item)),
+      (item) => item
+    );
+    const exported = declarationVisible(child, ["private", "internal"]);
+    const bodyNode = findNamedChild(child, "class_body") ?? child.childForFieldName("body");
+    draftSymbols.push({
+      name,
+      kind,
+      signature: singleLineSignature(child.text),
+      exported,
+      callNames: [],
+      extendsNames: kind === "interface" ? delegationNames : delegationNames.slice(0, 1),
+      implementsNames: kind === "class" ? delegationNames.slice(1) : [],
+      bodyText: nodeText(bodyNode) || child.text
+    });
+    if (exported) {
+      exportLabels.push(name);
+    }
+    pushBodyFunctions(bodyNode, name);
+  }
+  return finalizeCodeAnalysis(manifest, "kotlin", imports, draftSymbols, exportLabels, diagnostics, {
+    namespace: packageName
+  });
+}
+function scalaCodeAnalysis(manifest, rootNode, diagnostics) {
+  const imports = [];
+  const draftSymbols = [];
+  const exportLabels = [];
+  let packageName;
+  const pushTemplateFunctions = (bodyNode, scopeName) => {
+    if (!bodyNode) {
+      return;
+    }
+    for (const child of bodyNode.namedChildren) {
+      if (!child || child.type !== "function_definition") {
+        continue;
+      }
+      const functionName = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "identifier"));
+      if (!functionName) {
+        continue;
+      }
+      const exported = declarationVisible(child, ["private", "protected"]);
+      const symbolName = scopeName ? `${scopeName}.${functionName}` : functionName;
+      draftSymbols.push({
+        name: symbolName,
+        kind: "function",
+        signature: singleLineSignature(child.text),
+        exported,
+        callNames: [],
+        extendsNames: [],
+        implementsNames: [],
+        bodyText: child.text
+      });
+      if (exported) {
+        exportLabels.push(symbolName);
+      }
+    }
+  };
+  for (const child of rootNode.namedChildren) {
+    if (!child) {
+      continue;
+    }
+    if (child.type === "package_clause") {
+      packageName = nodeText(findNamedChild(child, "package_identifier") ?? child.namedChildren.at(0) ?? null) || packageName;
+      continue;
+    }
+    if (child.type === "import_declaration") {
+      imports.push(...parseScalaImport(child.text));
+      continue;
+    }
+    if (child.type === "function_definition") {
+      pushTemplateFunctions({
+        ...child,
+        namedChildren: [child]
+      });
+      continue;
+    }
+    if (!["trait_definition", "class_definition", "object_definition"].includes(child.type)) {
+      continue;
+    }
+    const name = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "identifier"));
+    const kind = scalaDefinitionKind(child);
+    if (!name || !kind) {
+      continue;
+    }
+    const extendsClause = findNamedChild(child, "extends_clause") ?? child.childForFieldName("extends");
+    const inheritance = uniqueBy(descendantTypeNames(extendsClause), (item) => item);
+    const bodyNode = findNamedChild(child, "template_body") ?? child.childForFieldName("body");
+    const exported = declarationVisible(child, ["private", "protected"]);
+    draftSymbols.push({
+      name,
+      kind,
+      signature: singleLineSignature(child.text),
+      exported,
+      callNames: [],
+      extendsNames: kind === "trait" ? inheritance : inheritance.slice(0, 1),
+      implementsNames: kind === "class" ? inheritance.slice(1) : [],
+      bodyText: nodeText(bodyNode) || child.text
+    });
+    if (exported) {
+      exportLabels.push(name);
+    }
+    pushTemplateFunctions(bodyNode, name);
+  }
+  return finalizeCodeAnalysis(manifest, "scala", imports, draftSymbols, exportLabels, diagnostics, {
+    namespace: packageName
+  });
+}
 function csharpCodeAnalysis(manifest, rootNode, diagnostics) {
   const imports = [];
   const draftSymbols = [];
@@ -2915,11 +3196,19 @@ function cFamilyCodeAnalysis(manifest, language, rootNode, diagnostics) {
   return finalizeCodeAnalysis(manifest, language, imports, draftSymbols, exportLabels, diagnostics);
 }
 async function analyzeTreeSitterCode(manifest, content, language) {
-  const module = await getTreeSitterModule();
-  await ensureTreeSitterInit(module);
-  const parser = new module.Parser();
-  parser.setLanguage(await loadLanguage(language));
-  const tree = parser.parse(content);
+  let tree = null;
+  try {
+    const module = await getTreeSitterModule();
+    await ensureTreeSitterInit(module);
+    const parser = new module.Parser();
+    parser.setLanguage(await loadLanguage(language));
+    tree = parser.parse(content);
+  } catch (error) {
+    return {
+      code: finalizeCodeAnalysis(manifest, language, [], [], [], [treeSitterCompatibilityDiagnostic(language, error)]),
+      rationales: []
+    };
+  }
   if (!tree) {
     return {
       code: finalizeCodeAnalysis(
@@ -2953,6 +3242,10 @@ async function analyzeTreeSitterCode(manifest, content, language) {
         return { code: rustCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
       case "java":
         return { code: javaCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
+      case "kotlin":
+        return { code: kotlinCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
+      case "scala":
+        return { code: scalaCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
       case "csharp":
         return { code: csharpCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
       case "php":
@@ -2964,6 +3257,26 @@ async function analyzeTreeSitterCode(manifest, content, language) {
       case "c":
       case "cpp":
         return { code: cFamilyCodeAnalysis(manifest, language, tree.rootNode, diagnostics), rationales };
+      default:
+        return {
+          code: finalizeCodeAnalysis(
+            manifest,
+            language,
+            [],
+            [],
+            [],
+            [
+              {
+                code: 9011,
+                category: "error",
+                message: `No parser-backed analyzer is registered for ${language}.`,
+                line: 1,
+                column: 1
+              }
+            ]
+          ),
+          rationales
+        };
     }
   } finally {
     tree.delete();
@@ -3219,7 +3532,7 @@ function makeRationale2(manifest, index, text, kind, symbolName) {
   };
 }
 function stripCodeExtension2(filePath) {
-  return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
+  return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|kt|kts|scala|sc|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
 }
 function manifestModuleName2(manifest, language) {
   const repoPath = manifest.repoRelativePath ?? path6.basename(manifest.originalPath ?? manifest.storedPath);
@@ -3550,6 +3863,12 @@ function inferCodeLanguage(filePath, mimeType = "") {
   if (extension === ".java") {
     return "java";
   }
+  if (extension === ".kt" || extension === ".kts") {
+    return "kotlin";
+  }
+  if (extension === ".scala" || extension === ".sc") {
+    return "scala";
+  }
   if (extension === ".cs") {
     return "csharp";
   }
@@ -3654,6 +3973,10 @@ function candidateExtensionsFor(language) {
       return [".rs"];
     case "java":
       return [".java"];
+    case "kotlin":
+      return [".kt", ".kts"];
+    case "scala":
+      return [".scala", ".sc"];
     case "csharp":
       return [".cs"];
     case "php":
@@ -3714,10 +4037,17 @@ async function buildCodeIndex(rootDir, manifests, analyses) {
         break;
       }
       case "java":
+      case "kotlin":
+      case "scala":
       case "csharp":
         if (normalizedNamespace) {
           recordAlias(aliases, `${normalizedNamespace}.${basename}`);
         }
+        if (normalizedNamespace) {
+          for (const symbol of analysis.code.symbols) {
+            recordAlias(aliases, `${normalizedNamespace}.${symbol.name}`);
+          }
+        }
         break;
       case "php":
         if (normalizedNamespace) {
@@ -3824,6 +4154,8 @@ function findImportCandidates(manifest, codeImport, lookup) {
       return aliasMatches(lookup, codeImport.specifier);
     case "go":
     case "java":
+    case "kotlin":
+    case "scala":
     case "csharp":
       return aliasMatches(lookup, codeImport.specifier);
     case "php":
@@ -3870,6 +4202,8 @@ function importLooksLocal(manifest, codeImport, candidates) {
     case "powershell":
     case "c":
     case "cpp":
+    case "kotlin":
+    case "scala":
       return !codeImport.isExternal;
     default:
       return false;
@@ -4520,6 +4854,9 @@ async function markPagesStaleForSources(rootDir, sourceIds) {
 var DEFAULT_MAX_ASSET_SIZE = 10 * 1024 * 1024;
 var DEFAULT_MAX_DIRECTORY_FILES = 5e3;
 var HARD_REPO_IGNORES = /* @__PURE__ */ new Set([".git", ".venv"]);
+var PROGRESS_FILE_THRESHOLD = 150;
+var PROGRESS_UPDATE_INTERVAL = 100;
+var RST_HEADING_MARKERS = /* @__PURE__ */ new Set(["=", "-", "~", "^", '"', "#", "*", "+"]);
 function uniqueStrings(values) {
   return [...new Set(values.filter(Boolean))];
 }
@@ -4527,6 +4864,9 @@ function inferKind(mimeType, filePath) {
   if (inferCodeLanguage(filePath, mimeType)) {
     return "code";
   }
+  if (isRstFilePath(filePath)) {
+    return "text";
+  }
   if (mimeType.includes("markdown")) {
     return "markdown";
   }
@@ -4547,13 +4887,146 @@ function inferKind(mimeType, filePath) {
   }
   return "binary";
 }
-function titleFromText(fallback, content) {
+function isRstFilePath(filePath) {
+  const extension = path11.extname(filePath).toLowerCase();
+  return extension === ".rst" || extension === ".rest";
+}
+function titleFromText(fallback, content, filePath) {
+  if (filePath && isRstFilePath(filePath)) {
+    const rstTitle = titleFromRst(fallback, content);
+    if (rstTitle) {
+      return rstTitle;
+    }
+  }
   const heading = content.match(/^#\s+(.+)$/m)?.[1]?.trim();
   return heading || fallback;
 }
 function guessMimeType(target) {
+  if (isRstFilePath(target)) {
+    return "text/x-rst";
+  }
   return mime.lookup(target) || "application/octet-stream";
 }
+function rstAdornmentLine(line) {
+  const trimmed = line.trim();
+  if (trimmed.length < 3) {
+    return void 0;
+  }
+  const marker = trimmed[0] ?? "";
+  if (!RST_HEADING_MARKERS.has(marker) || ![...trimmed].every((char) => char === marker)) {
+    return void 0;
+  }
+  return marker;
+}
+function rstHeadingLevel(marker) {
+  switch (marker) {
+    case "=":
+      return "#";
+    case "-":
+      return "##";
+    case "~":
+      return "###";
+    case "^":
+      return "####";
+    default:
+      return "##";
+  }
+}
+function normalizeRstDirective(line) {
+  const trimmed = line.trim();
+  if (!trimmed.startsWith(".. ")) {
+    return void 0;
+  }
+  const directiveMatch = trimmed.match(/^\.\.\s+([A-Za-z][\w-]*)::\s*(.*)$/);
+  if (!directiveMatch) {
+    return "";
+  }
+  const label = directiveMatch[1].replace(/-/g, " ").split(/\s+/).filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(" ");
+  const detail = directiveMatch[2]?.trim();
+  return detail ? `${label}: ${detail}` : `${label}:`;
+}
+function normalizeRstExtractedText(content) {
+  const lines = content.split(/\r?\n/);
+  const normalized = [];
+  for (let index = 0; index < lines.length; index += 1) {
+    const current = lines[index] ?? "";
+    const next = lines[index + 1] ?? "";
+    const afterNext = lines[index + 2] ?? "";
+    const trimmed = current.trim();
+    if (!trimmed) {
+      normalized.push("");
+      continue;
+    }
+    const currentAdornment = rstAdornmentLine(current);
+    const nextAdornment = rstAdornmentLine(next);
+    const afterNextAdornment = rstAdornmentLine(afterNext);
+    if (currentAdornment && next.trim() && afterNextAdornment && currentAdornment === afterNextAdornment) {
+      normalized.push(`${rstHeadingLevel(currentAdornment)} ${next.trim()}`);
+      normalized.push("");
+      index += 2;
+      continue;
+    }
+    if (nextAdornment && trimmed.length > 0) {
+      normalized.push(`${rstHeadingLevel(nextAdornment)} ${trimmed}`);
+      normalized.push("");
+      index += 1;
+      continue;
+    }
+    const directive = normalizeRstDirective(current);
+    if (directive !== void 0) {
+      if (directive) {
+        normalized.push(directive);
+      }
+      continue;
+    }
+    normalized.push(current);
+  }
+  return normalized.join("\n").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+function titleFromRst(fallback, content) {
+  const normalized = normalizeRstExtractedText(content);
+  const heading = normalized.match(/^#+\s+(.+)$/m)?.[1]?.trim();
+  return heading || fallback;
+}
+function extractedTextForPlainSource(filePath, sourceKind, content) {
+  if (sourceKind === "text" && isRstFilePath(filePath)) {
+    return normalizeRstExtractedText(content);
+  }
+  return content;
+}
+function shouldEmitProgress(totalItems) {
+  return totalItems >= PROGRESS_FILE_THRESHOLD && Boolean(process.stderr?.isTTY);
+}
+function createProgressReporter(prefix, totalItems) {
+  if (!shouldEmitProgress(totalItems)) {
+    return {
+      tick: () => {
+      },
+      finish: () => {
+      }
+    };
+  }
+  let completed = 0;
+  let nextUpdate = Math.min(PROGRESS_UPDATE_INTERVAL, totalItems);
+  process.stderr.write(`[swarmvault ${prefix}] starting ${totalItems} file(s)
+`);
+  return {
+    tick: () => {
+      completed += 1;
+      if (completed >= nextUpdate || completed === totalItems) {
+        process.stderr.write(`[swarmvault ${prefix}] ${completed}/${totalItems}
+`);
+        while (completed >= nextUpdate) {
+          nextUpdate += PROGRESS_UPDATE_INTERVAL;
+        }
+      }
+    },
+    finish: (summary) => {
+      process.stderr.write(`[swarmvault ${prefix}] finished ${totalItems} file(s)${summary ? ` (${summary})` : ""}
+`);
+    }
+  };
+}
 function normalizeIngestOptions(options) {
   return {
     includeAssets: options?.includeAssets ?? true,
@@ -5490,6 +5963,7 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
       }))
     );
     scannedCount += files.length;
+    const progress = createProgressReporter("sync", files.length);
     const currentPaths = new Set(files.map((absolutePath) => path11.resolve(absolutePath)));
     for (const absolutePath of files) {
       const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path11.relative(repoRoot, absolutePath));
@@ -5500,7 +5974,9 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
       } else if (result.wasUpdated) {
         updated.push(result.manifest);
       }
+      progress.tick();
     }
+    progress.finish(`repo=${toPosix(path11.relative(rootDir, repoRoot)) || "."}`);
     for (const manifest of repoManifests) {
       const originalPath = manifest.originalPath ? path11.resolve(manifest.originalPath) : null;
       if (originalPath && !currentPaths.has(originalPath)) {
@@ -5590,6 +6066,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
       }))
     );
     scannedCount += files.length;
+    const progress = createProgressReporter("sync-watch", files.length);
     const currentPaths = new Set(files.map((absolutePath) => path11.resolve(absolutePath)));
     for (const absolutePath of files) {
       const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path11.relative(repoRoot, absolutePath));
@@ -5616,6 +6093,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
             staleSourceIds.add(existing.sourceId);
           }
         }
+        progress.tick();
         continue;
       }
       const result = await persistPreparedInput(rootDir, prepared, paths);
@@ -5624,7 +6102,9 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
       } else if (result.wasUpdated) {
         updated.push(result.manifest);
       }
+      progress.tick();
     }
+    progress.finish(`repo=${toPosix(path11.relative(rootDir, repoRoot)) || "."}`);
     for (const manifest of repoManifests) {
       const originalPath = manifest.originalPath ? path11.resolve(manifest.originalPath) : null;
       if (originalPath && !currentPaths.has(originalPath)) {
@@ -5685,8 +6165,8 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
   let extractedText;
   let extractionArtifact;
   if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
-    extractedText = payloadBytes.toString("utf8");
-    title = titleFromText(path11.basename(absoluteInput, path11.extname(absoluteInput)), extractedText);
+    extractedText = extractedTextForPlainSource(absoluteInput, sourceKind, payloadBytes.toString("utf8"));
+    title = titleFromText(path11.basename(absoluteInput, path11.extname(absoluteInput)), extractedText, absoluteInput);
     extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
   } else if (sourceKind === "html") {
     const html = payloadBytes.toString("utf8");
@@ -5796,8 +6276,8 @@ async function prepareUrlInput(rootDir, input, options) {
     const extension = path11.extname(inputUrl.pathname);
     storedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
     if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
-      extractedText = payloadBytes.toString("utf8");
-      title = titleFromText(title || inputUrl.hostname, extractedText);
+      extractedText = extractedTextForPlainSource(inputUrl.pathname, sourceKind, payloadBytes.toString("utf8"));
+      title = titleFromText(title || inputUrl.hostname, extractedText, inputUrl.pathname);
       extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
       if (sourceKind === "markdown" && options.includeAssets) {
         const { attachments: remoteAttachments, skippedCount } = await collectRemoteImageAttachments(
@@ -6090,6 +6570,7 @@ async function ingestDirectory(rootDir, inputDir, options) {
   const { files, skipped } = await collectDirectoryFiles(rootDir, absoluteInputDir, repoRoot, normalizedOptions);
   const imported = [];
   const updated = [];
+  const progress = createProgressReporter("ingest", files.length);
   for (const absolutePath of files) {
     const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path11.relative(repoRoot, absolutePath));
     const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
@@ -6101,7 +6582,9 @@ async function ingestDirectory(rootDir, inputDir, options) {
     } else {
       skipped.push({ path: toPosix(path11.relative(rootDir, absolutePath)), reason: "duplicate_content" });
     }
+    progress.tick();
   }
+  progress.finish(`imported=${imported.length}, updated=${updated.length}, skipped=${skipped.length}`);
   await appendLogEntry(rootDir, "ingest_directory", toPosix(path11.relative(rootDir, absoluteInputDir)) || ".", [
     `repo_root=${toPosix(path11.relative(rootDir, repoRoot)) || "."}`,
     `scanned=${files.length}`,
@@ -8889,6 +9372,35 @@ function topGroupPatterns(graph) {
     (left, right) => right.confidence - left.confidence || right.nodeIds.length - left.nodeIds.length || left.label.localeCompare(right.label)
   ).slice(0, 8);
 }
+function fragmentedCommunityPresentation(graph, communityPages) {
+  const thinCommunities = (graph.communities ?? []).filter((community) => community.nodeIds.length <= 2).sort((left, right) => right.nodeIds.length - left.nodeIds.length || left.label.localeCompare(right.label));
+  const visibleCommunities = thinCommunities.slice(0, 6).map((community) => {
+    const page = communityPages.find((candidate) => candidate.id === `graph:${community.id}`);
+    return {
+      id: community.id,
+      label: community.label,
+      nodeCount: community.nodeIds.length,
+      pageId: page?.id,
+      path: page?.path,
+      title: page?.title
+    };
+  });
+  const rolledUp = thinCommunities.slice(visibleCommunities.length);
+  if (!rolledUp.length) {
+    return {
+      thinCommunities: visibleCommunities
+    };
+  }
+  return {
+    thinCommunities: visibleCommunities,
+    fragmentedCommunityRollup: {
+      totalCommunities: graph.communities?.length ?? 0,
+      rolledUpCount: rolledUp.length,
+      rolledUpNodes: rolledUp.reduce((sum, community) => sum + community.nodeIds.length, 0),
+      exampleLabels: rolledUp.slice(0, 4).map((community) => community.label)
+    }
+  };
+}
 function suggestedGraphQuestions(graph) {
   const thinCommunities = (graph.communities ?? []).filter((community) => community.nodeIds.length <= 2);
   const bridgeNodes = graph.nodes.filter((node) => (node.bridgeScore ?? 0) > 0).sort((left, right) => (right.bridgeScore ?? 0) - (left.bridgeScore ?? 0)).slice(0, 3);
@@ -8903,17 +9415,7 @@ function buildGraphReportArtifact(input) {
   const pagesById = new Map(reportGraph.pages.map((page) => [page.id, page]));
   const godNodes = reportGraph.nodes.filter((node) => node.isGodNode).sort((left, right) => (right.degree ?? 0) - (left.degree ?? 0)).slice(0, 8);
   const bridgeNodes = reportGraph.nodes.filter((node) => (node.bridgeScore ?? 0) > 0).sort((left, right) => (right.bridgeScore ?? 0) - (left.bridgeScore ?? 0)).slice(0, 8);
-  const thinCommunities = (reportGraph.communities ?? []).filter((community) => community.nodeIds.length <= 2).map((community) => {
-    const page = input.communityPages.find((candidate) => candidate.id === `graph:${community.id}`);
-    return {
-      id: community.id,
-      label: community.label,
-      nodeCount: community.nodeIds.length,
-      pageId: page?.id,
-      path: page?.path,
-      title: page?.title
-    };
-  });
+  const communityPresentation = fragmentedCommunityPresentation(reportGraph, input.communityPages);
   const surprisingConnections = topSurprisingConnections(reportGraph, pagesById);
   const groupPatterns = topGroupPatterns(reportGraph);
   const breakdown = sourceClassBreakdown(input.graph);
@@ -8927,6 +9429,11 @@ function buildGraphReportArtifact(input) {
       `Non-first-party material accounts for ${(nonFirstPartyNodes / Math.max(1, input.graph.nodes.length) * 100).toFixed(1)}% of graph nodes.`
     );
   }
+  if (communityPresentation.fragmentedCommunityRollup) {
+    warnings.push(
+      `First-party report view is fragmented: ${communityPresentation.fragmentedCommunityRollup.rolledUpCount} tiny communities were rolled up for readability.`
+    );
+  }
   return {
     generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
     graphHash: input.graphHash,
@@ -8964,10 +9471,11 @@ function buildGraphReportArtifact(input) {
       degree: node.degree,
       bridgeScore: node.bridgeScore
     })),
-    thinCommunities,
+    thinCommunities: communityPresentation.thinCommunities,
+    fragmentedCommunityRollup: communityPresentation.fragmentedCommunityRollup,
     surprisingConnections,
     groupPatterns,
-    suggestedQuestions: suggestedGraphQuestions(input.graph),
+    suggestedQuestions: suggestedGraphQuestions(reportGraph),
     communityPages: input.communityPages.map((page) => ({
       id: page.id,
       path: page.path,
@@ -9094,6 +9602,10 @@ function buildGraphReportPage(input) {
     ...input.report.thinCommunities.length ? input.report.thinCommunities.map(
       (community) => community.path ? `- [[${community.path.replace(/\.md$/, "")}|${community.title ?? community.label}]] (${community.nodeCount} node(s))` : `- ${community.label} (${community.nodeCount} node(s))`
     ) : ["- No thin communities detected."],
+    ...input.report.fragmentedCommunityRollup ? [
+      `- Rolled up ${input.report.fragmentedCommunityRollup.rolledUpCount} additional tiny communities covering ${input.report.fragmentedCommunityRollup.rolledUpNodes} node(s).`,
+      `- Example rolled-up labels: ${input.report.fragmentedCommunityRollup.exampleLabels.join(", ")}`
+    ] : [],
     "",
     "## Surprising Connections",
     "",
@@ -10236,9 +10748,41 @@ function searchPages(dbPath, query, limitOrOptions = 5) {
 }
 
 // src/vault.ts
+var COMPILE_PROGRESS_THRESHOLD = 120;
+var COMPILE_PROGRESS_UPDATE_INTERVAL = 50;
 function uniqueStrings3(values) {
   return uniqueBy(values.filter(Boolean), (value) => value);
 }
+function createCompileProgressReporter(phase, totalItems) {
+  if (totalItems < COMPILE_PROGRESS_THRESHOLD || !process.stderr?.isTTY) {
+    return {
+      tick: () => {
+      },
+      finish: () => {
+      }
+    };
+  }
+  let completed = 0;
+  let nextUpdate = Math.min(COMPILE_PROGRESS_UPDATE_INTERVAL, totalItems);
+  process.stderr.write(`[swarmvault compile] ${phase}: 0/${totalItems}
+`);
+  return {
+    tick: (label) => {
+      completed += 1;
+      if (completed >= nextUpdate || completed === totalItems) {
+        process.stderr.write(`[swarmvault compile] ${phase}: ${completed}/${totalItems}${label ? ` (${label})` : ""}
+`);
+        while (completed >= nextUpdate) {
+          nextUpdate += COMPILE_PROGRESS_UPDATE_INTERVAL;
+        }
+      }
+    },
+    finish: (summary) => {
+      process.stderr.write(`[swarmvault compile] ${phase}: ${totalItems}/${totalItems}${summary ? ` (${summary})` : ""}
+`);
+    }
+  };
+}
 function normalizeOutputFormat2(format) {
   return format === "report" || format === "slides" || format === "chart" || format === "image" ? format : "markdown";
 }
@@ -13117,34 +13661,41 @@ async function compileVault(rootDir, options = {}) {
       candidatePageCount: (graph?.pages ?? []).filter((page) => page.status === "candidate").length
     };
   }
+  const analysisProgress = createCompileProgressReporter("analyze", manifests.length);
   const [dirtyAnalyses, cleanAnalyses] = await Promise.all([
     Promise.all(
-      dirty.map(
-        async (manifest) => analyzeSource(
+      dirty.map(async (manifest) => {
+        const analysis = await analyzeSource(
           manifest,
           await readExtractedText(rootDir, manifest),
           provider,
           paths,
           getEffectiveSchema(schemas, sourceProjects[manifest.sourceId] ?? null)
-        )
-      )
+        );
+        analysisProgress.tick(manifest.title);
+        return analysis;
+      })
     ),
     Promise.all(
       clean.map(async (manifest) => {
         const cached = await readJsonFile(path21.join(paths.analysesDir, `${manifest.sourceId}.json`));
         if (cached) {
+          analysisProgress.tick(manifest.title);
           return cached;
         }
-        return analyzeSource(
+        const analysis = await analyzeSource(
           manifest,
           await readExtractedText(rootDir, manifest),
           provider,
           paths,
           getEffectiveSchema(schemas, sourceProjects[manifest.sourceId] ?? null)
         );
+        analysisProgress.tick(manifest.title);
+        return analysis;
       })
     )
   ]);
+  analysisProgress.finish(`dirty=${dirty.length}, clean=${clean.length}`);
   const initialAnalyses = [...dirtyAnalyses, ...cleanAnalyses];
   const codeIndex = await buildCodeIndex(rootDir, manifests, initialAnalyses);
   const analyses = await Promise.all(
@@ -13913,7 +14464,7 @@ async function bootstrapDemo(rootDir, input) {
 }
 
 // src/mcp.ts
-var SERVER_VERSION = "0.1.31";
+var SERVER_VERSION = "0.1.33";
 async function createMcpServer(rootDir) {
   const server = new McpServer({
     name: "swarmvault",
@@ -14743,6 +15294,7 @@ async function watchVault(rootDir, options = {}) {
   let consecutiveFailures = 0;
   let currentDebounceMs = baseDebounceMs;
   const reasons = /* @__PURE__ */ new Set();
+  let activeCycle = null;
   const watcher = chokidar.watch(watchTargets, {
     ignoreInitial: true,
     usePolling: true,
@@ -14787,7 +15339,12 @@ async function watchVault(rootDir, options = {}) {
       clearTimeout(timer);
     }
     timer = setTimeout(() => {
-      void runCycle();
+      const cycle = runCycle();
+      activeCycle = cycle.finally(() => {
+        if (activeCycle === cycle) {
+          activeCycle = null;
+        }
+      });
     }, currentDebounceMs);
   };
   const runCycle = async () => {
@@ -14955,6 +15512,7 @@ async function watchVault(rootDir, options = {}) {
         clearTimeout(timer);
       }
       await watcher.close();
+      await activeCycle;
     }
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swarmvaultai/engine",
-  "version": "0.1.31",
+  "version": "0.1.33",
   "description": "Core engine for SwarmVault: ingest, compile, query, lint, and provider abstractions.",
   "type": "module",
   "main": "dist/index.js",
@@ -57,6 +57,7 @@
     "mime-types": "^3.0.1",
     "neo4j-driver": "^5.28.3",
     "pdfjs-dist": "^5.4.394",
+    "tree-sitter-wasms": "^0.1.13",
     "turndown": "^7.2.1",
     "typescript": "^5.9.3",
     "yaml": "^2.8.1",