genesis-compiler 1.0.0 → 1.2.0

package/src/index/code-index.js

@@ -0,0 +1,283 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import { astGrepCodeIndexers } from './code-indexers/ast-grep.js';
+import { asDiagnostic } from './errors.js';
+import { gitContext } from './git.js';
+import { isProjectContentPath } from './paths.js';
+import { inspectProgram } from './program.js';
+import { gitVisibleFileStates } from './project-files.js';
+import { readStack } from './stack.js';
+import { sha256, stableJson, writeFileAtomic } from './utils.js';
+
+export const MACHINE_CITY_PATH = '.genesis/machine-city.json';
+export const PROGRAM_CITY_PATH = '.genesis/program-city.json';
+
+const INDEXERS = astGrepCodeIndexers;
+
+function directoryId(value) {
+  return `directory:${value || '.'}`;
+}
+
+function fileId(value) {
+  return `file:${value}`;
+}
+
+function subsystemId(value) {
+  return `subsystem:${value}`;
+}
+
+function operationId(module) {
+  return `operation:${module.subsystem}/${module.name}`;
+}
+
+function parentDirectory(value) {
+  const parent = path.posix.dirname(value);
+  return parent === '.' ? '' : parent;
+}
+
+function directoryRecords(paths) {
+  const directories = new Set(['']);
+  for (const value of paths) {
+    let current = parentDirectory(value);
+    while (true) {
+      directories.add(current);
+      if (!current) break;
+      current = parentDirectory(current);
+    }
+  }
+  return [...directories].sort().map((directory) => ({
+    id: directoryId(directory),
+    path: directory,
+    title: directory ? path.posix.basename(directory) : 'Project',
+    parentId: directory ? directoryId(parentDirectory(directory)) : null,
+  }));
+}
+
+function subsystemRecords(subsystems) {
+  const paths = new Set();
+  for (const subsystem of subsystems) {
+    const segments = subsystem.split('/');
+    for (let count = 1; count <= segments.length; count += 1) {
+      paths.add(segments.slice(0, count).join('/'));
+    }
+  }
+  return [...paths].sort().map((subsystem) => ({
+    id: subsystemId(subsystem),
+    path: subsystem,
+    title: path.posix.basename(subsystem).replace(/-/gu, ' '),
+    parentId: subsystem.includes('/') ? subsystemId(parentDirectory(subsystem)) : null,
+  }));
+}
+
+function normalizedFunction(entry, file, extractor) {
+  const qualifiedName = entry.qualifiedName || entry.name;
+  return {
+    id: `function:${file.path}:${qualifiedName}:${entry.kind}:${entry.line}:${entry.column}`,
+    name: entry.name,
+    qualifiedName,
+    kind: entry.kind,
+    visibility: entry.visibility,
+    container: entry.container || null,
+    path: file.path,
+    fileId: fileId(file.path),
+    line: entry.line,
+    column: entry.column,
+    parameters: entry.parameters || [],
+    async: entry.async === true,
+    generator: entry.generator === true,
+    static: entry.static === true,
+    role: file.role,
+    language: file.language,
+    extractor,
+  };
+}
+
+function mergeIndexedFiles(contributions) {
+  const files = new Map();
+  const functions = [];
+  for (const contribution of contributions) {
+    for (const file of contribution.files || []) {
+      const existing = files.get(file.path);
+      const record = existing || {
+        id: fileId(file.path),
+        path: file.path,
+        language: file.language,
+        role: file.role,
+        mode: file.mode,
+        bytes: file.bytes,
+        lines: file.lines,
+        hash: file.hash,
+        extractors: [],
+        functionIds: [],
+      };
+      if (!record.extractors.includes(contribution.extractor)) record.extractors.push(contribution.extractor);
+      for (const entry of file.functions || []) {
+        const normalized = normalizedFunction(entry, file, contribution.extractor);
+        functions.push(normalized);
+        record.functionIds.push(normalized.id);
+      }
+      files.set(file.path, record);
+    }
+  }
+  return {
+    files: [...files.values()].sort((left, right) => left.path.localeCompare(right.path)),
+    functions: functions.sort((left, right) => (
+      left.path.localeCompare(right.path) || left.line - right.line || left.qualifiedName.localeCompare(right.qualifiedName)
+    )),
+  };
+}
+
+function machineCity({ components, contributions, diagnostics, indexers }) {
+  const merged = mergeIndexedFiles(contributions);
+  const visibility = new Map(merged.functions.map((entry) => [entry.id, entry.visibility]));
+  const codeHash = sha256(stableJson(merged.files.map(({ path: filePath, hash, mode }) => ({
+    path: filePath,
+    hash,
+    mode,
+  }))));
+  return {
+    schema: 'genesis.machine-city.v1',
+    schemaVersion: 1,
+    status: indexers.length === 0 ? 'unconfigured' : diagnostics.length > 0 ? 'completed-with-warning' : 'current',
+    codeHash,
+    stackComponents: components,
+    indexers,
+    diagnostics,
+    functions: merged.functions,
+    districts: directoryRecords(merged.files.map(({ path: filePath }) => filePath)),
+    buildings: merged.files.map((file) => ({
+      ...file,
+      districtId: directoryId(parentDirectory(file.path)),
+      title: path.posix.basename(file.path),
+      publicFunctionCount: file.functionIds.filter((id) => visibility.get(id) === 'public').length,
+      internalFunctionCount: file.functionIds.filter((id) => visibility.get(id) === 'internal').length,
+    })),
+  };
+}
+
+async function programCity(projectRoot) {
+  try {
+    const program = await inspectProgram(projectRoot);
+    const operations = program.modules.map((module) => ({
+      id: operationId(module),
+      name: module.name,
+      title: module.title,
+      description: module.description,
+      publicContract: module.publicContract,
+      implementationMap: module.implementationMap,
+      path: module.path,
+      subsystem: module.subsystem,
+      districtId: subsystemId(module.subsystem),
+      sources: module.sources,
+      sourceFileIds: module.sources.map(fileId),
+    }));
+    return {
+      schema: 'genesis.program-city.v1',
+      schemaVersion: 1,
+      status: program.status,
+      programHash: sha256(stableJson(operations)),
+      diagnostics: [],
+      districts: subsystemRecords(program.subsystems),
+      buildings: operations,
+      links: operations.flatMap((operation) => operation.sourceFileIds.map((targetId) => ({
+        kind: 'implemented-by',
+        fromId: operation.id,
+        toId: targetId,
+      }))),
+    };
+  } catch (error) {
+    return {
+      schema: 'genesis.program-city.v1',
+      schemaVersion: 1,
+      status: 'invalid',
+      programHash: sha256('invalid'),
+      diagnostics: [asDiagnostic(error)],
+      districts: [],
+      buildings: [],
+      links: [],
+    };
+  }
+}
+
+async function writeIfChanged(projectRoot, relativePath, value) {
+  const location = path.join(projectRoot, relativePath);
+  const source = stableJson(value);
+  try {
+    if (await readFile(location, 'utf8') === source) return false;
+  } catch (error) {
+    if (!['ENOENT', 'ENOTDIR'].includes(error?.code)) throw error;
+  }
+  await writeFileAtomic(location, source);
+  return true;
+}
+
+function functionMatches(entry, queries) {
+  if (queries.length === 0) return false;
+  const haystack = [entry.name, entry.qualifiedName, entry.path, entry.kind, entry.visibility]
+    .join('\n')
+    .toLowerCase();
+  return queries.every((query) => haystack.includes(query.toLowerCase()));
+}
+
+function formatCount(value, singular) {
+  return `${value} ${singular}${value === 1 ? '' : 's'}`;
+}
+
+export async function buildProjectIndex({ projectRoot, queries = [], write = true } = {}) {
+  const root = (await gitContext(projectRoot)).repositoryRoot;
+  const stack = await readStack(root);
+  const states = await gitVisibleFileStates(root, { includePath: isProjectContentPath });
+  const files = [...states]
+    .filter(([, state]) => state.exists && !state.symlink && !state.special && state.hash)
+    .map(([filePath, state]) => ({ path: filePath, hash: state.hash, mode: state.mode }));
+  const indexers = [...new Set(stack.components.flatMap((component) => component.indexers || []))].sort();
+  const contributions = [];
+  const diagnostics = [];
+  for (const id of indexers) {
+    const indexer = INDEXERS.get(id);
+    if (!indexer) {
+      diagnostics.push({ code: 'CODE_INDEXER_UNAVAILABLE', message: `No installed code indexer exists for ${id}.` });
+      continue;
+    }
+    try {
+      const contribution = await indexer.extract({ files, projectRoot: root });
+      contributions.push({ ...contribution, extractor: id });
+      diagnostics.push(...(contribution.diagnostics || []).map((diagnostic) => ({ ...diagnostic, extractor: id })));
+    } catch (error) {
+      diagnostics.push({ ...asDiagnostic(error), code: 'CODE_INDEXER_FAILED', extractor: id });
+    }
+  }
+  const machine = machineCity({
+    components: stack.components.map(({ id }) => id),
+    contributions,
+    diagnostics,
+    indexers,
+  });
+  const program = await programCity(root);
+  const changedFiles = [];
+  if (write) {
+    if (await writeIfChanged(root, MACHINE_CITY_PATH, machine)) changedFiles.push(MACHINE_CITY_PATH);
+    if (await writeIfChanged(root, PROGRAM_CITY_PATH, program)) changedFiles.push(PROGRAM_CITY_PATH);
+  }
+  const allMatches = machine.functions.filter((entry) => functionMatches(entry, queries));
+  const allDiagnostics = [...diagnostics, ...program.diagnostics];
+  return {
+    status: allDiagnostics.length > 0 ? 'completed-with-warning' : 'ready',
+    summary: [
+      `Indexed ${formatCount(machine.functions.length, 'function')}`,
+      `in ${formatCount(machine.buildings.length, 'file')}`,
+      `and ${formatCount(program.buildings.length, 'Program operation')}.`,
+    ].join(' '),
+    paths: { machine: MACHINE_CITY_PATH, program: PROGRAM_CITY_PATH },
+    changedFiles,
+    diagnostics: allDiagnostics,
+    functionCount: machine.functions.length,
+    fileCount: machine.buildings.length,
+    operationCount: program.buildings.length,
+    totalMatches: allMatches.length,
+    matches: allMatches.slice(0, 200),
+    machine,
+    program,
+  };
+}

package/src/index/code-indexers/ast-grep.js

@@ -0,0 +1,414 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import bashLanguage from '@ast-grep/lang-bash';
+import cLanguage from '@ast-grep/lang-c';
+import cppLanguage from '@ast-grep/lang-cpp';
+import csharpLanguage from '@ast-grep/lang-csharp';
+import goLanguage from '@ast-grep/lang-go';
+import javaLanguage from '@ast-grep/lang-java';
+import kotlinLanguage from '@ast-grep/lang-kotlin';
+import phpLanguage from '@ast-grep/lang-php';
+import pythonLanguage from '@ast-grep/lang-python';
+import rubyLanguage from '@ast-grep/lang-ruby';
+import rustLanguage from '@ast-grep/lang-rust';
+import { parseAsync, registerDynamicLanguage } from '@ast-grep/napi';
+
+const MAX_SOURCE_BYTES = 4 * 1024 * 1024;
+const OPENING_PARAMETER_DELIMITERS = '([{<';
+const CLOSING_PARAMETER_DELIMITERS = ')]}>';
+const EXCLUDED_DIRECTORIES = new Set([
+  '.next', '.nuxt', '.output', '.svelte-kit', 'build', 'coverage', 'dist',
+  'node_modules', 'storage', 'target', 'vendor',
+]);
+
+registerDynamicLanguage({
+  bash: bashLanguage,
+  c: cLanguage,
+  cpp: cppLanguage,
+  csharp: csharpLanguage,
+  go: goLanguage,
+  java: javaLanguage,
+  kotlin: kotlinLanguage,
+  php: phpLanguage,
+  python: pythonLanguage,
+  ruby: rubyLanguage,
+  rust: rustLanguage,
+});
+
+function javascriptParser(filePath) {
+  if (/\.tsx$/u.test(filePath)) return 'Tsx';
+  if (/\.ts$/u.test(filePath)) return 'TypeScript';
+  if (/\.jsx$/u.test(filePath)) return 'Tsx';
+  return 'JavaScript';
+}
+
+function cFamilyParser(filePath) {
+  return /\.(?:c|h)$/u.test(filePath) ? 'c' : 'cpp';
+}
+
+const DEFINITIONS = Object.freeze({
+  javascript: {
+    extensions: ['.cjs', '.js', '.jsx', '.mjs', '.ts', '.tsx', '.vue'],
+    kinds: ['function_declaration', 'generator_function_declaration', 'method_definition', 'arrow_function', 'function_expression'],
+    parser: javascriptParser,
+  },
+  python: { extensions: ['.py', '.pyi'], kinds: ['function_definition'], parser: 'python' },
+  java: { extensions: ['.java'], kinds: ['method_declaration', 'constructor_declaration'], parser: 'java' },
+  csharp: { extensions: ['.cs'], kinds: ['method_declaration', 'constructor_declaration', 'local_function_statement'], parser: 'csharp' },
+  cpp: {
+    extensions: ['.c', '.cc', '.cpp', '.cxx', '.h', '.hh', '.hpp', '.hxx'],
+    kinds: ['function_definition'],
+    parser: cFamilyParser,
+  },
+  php: { extensions: ['.php'], kinds: ['function_definition', 'method_declaration'], parser: 'php' },
+  go: { extensions: ['.go'], kinds: ['function_declaration', 'method_declaration'], parser: 'go' },
+  rust: { extensions: ['.rs'], kinds: ['function_item'], parser: 'rust' },
+  ruby: { extensions: ['.rb', '.rake'], kinds: ['method', 'singleton_method'], parser: 'ruby' },
+  kotlin: { extensions: ['.kt', '.kts'], kinds: ['function_declaration', 'secondary_constructor'], parser: 'kotlin' },
+  shell: {
+    extensions: ['.bash', '.sh'],
+    kinds: ['function_definition'],
+    parser: 'bash',
+    shebang: /^#![^\n]*\b(?:bash|dash|ksh|sh|zsh)\b/u,
+  },
+});
+const CALLABLE_KINDS = new Set(Object.values(DEFINITIONS).flatMap(({ kinds }) => kinds));
+
+function safeField(node, name) {
+  try { return node.field(name); } catch { return null; }
+}
+
+function firstDescendant(node, kinds) {
+  const wanted = new Set(kinds);
+  const pending = [...(node?.children() || [])];
+  while (pending.length > 0) {
+    const candidate = pending.shift();
+    if (wanted.has(candidate.kind())) return candidate;
+    pending.unshift(...candidate.children());
+  }
+  return null;
+}
+
+function declaredName(node) {
+  const direct = safeField(node, 'name');
+  if (direct) return direct.text().trim();
+  const parent = node.parent();
+  if (['arrow_function', 'function_expression'].includes(node.kind())) {
+    if (parent?.kind() === 'variable_declarator') {
+      return safeField(parent, 'name')?.text().trim() || parent.child(0)?.text().trim() || '';
+    }
+    if (parent?.kind() === 'pair') {
+      return safeField(parent, 'key')?.text().trim() || parent.child(0)?.text().trim() || '';
+    }
+    if (parent?.kind() === 'assignment_expression') {
+      const target = safeField(parent, 'left')?.text().trim() || parent.child(0)?.text().trim() || '';
+      return target.match(/[A-Za-z_$][\w$]*$/u)?.[0] || '';
+    }
+    return '';
+  }
+  const declarator = safeField(node, 'declarator');
+  if (declarator) {
+    const identifier = firstDescendant(declarator, [
+      'destructor_name', 'field_identifier', 'identifier', 'operator_name', 'qualified_identifier',
+    ]);
+    if (identifier) return identifier.text().trim();
+  }
+  const ownIdentifier = node.children().find((child) => [
+    'field_identifier', 'identifier', 'name', 'property_identifier', 'simple_identifier',
+  ].includes(child.kind()));
+  if (ownIdentifier) return ownIdentifier.text().trim();
+  return '';
+}
+
+function parameterText(node) {
+  const direct = safeField(node, 'parameters');
+  if (direct) return direct.text().trim();
+  const declarator = safeField(node, 'declarator');
+  return firstDescendant(declarator, [
+    'formal_parameters', 'parameter_list', 'parameters', 'function_value_parameters',
+  ])?.text().trim() || '';
+}
+
+function splitParameters(value) {
+  const source = value.replace(/^\(/u, '').replace(/\)$/u, '').trim();
+  if (!source) return [];
+  const result = [];
+  let start = 0;
+  let depth = 0;
+  for (let index = 0; index < source.length; index += 1) {
+    if (OPENING_PARAMETER_DELIMITERS.includes(source[index])) depth += 1;
+    else if (CLOSING_PARAMETER_DELIMITERS.includes(source[index])) depth = Math.max(0, depth - 1);
+    else if (source[index] === ',' && depth === 0) {
+      result.push(source.slice(start, index).trim());
+      start = index + 1;
+    }
+  }
+  result.push(source.slice(start).trim());
+  return result.filter(Boolean);
+}
+
+function ancestor(node, kinds) {
+  const wanted = new Set(kinds);
+  return node.ancestors().find((candidate) => wanted.has(candidate.kind())) || null;
+}
+
+const CONTAINER_KINDS = new Set([
+  'class', 'class_declaration', 'class_definition', 'class_specifier', 'enum_declaration',
+  'impl_item', 'interface_declaration', 'module', 'object_declaration', 'struct_item', 'trait_item',
+]);
+
+function containerName(node, language) {
+  if (language === 'go' && node.kind() === 'method_declaration') {
+    const receiver = safeField(node, 'receiver')?.text().trim();
+    if (receiver) return receiver.replace(/^\(|\)$/gu, '').replace(/^\w+\s+\*?/u, '').trim();
+  }
+  const container = node.ancestors().find((candidate) => CONTAINER_KINDS.has(candidate.kind()));
+  if (!container) {
+    const parentFunction = node.ancestors().find((candidate) => DEFINITIONS.javascript.kinds.includes(candidate.kind())
+      || candidate.kind() === 'function_definition');
+    return parentFunction ? declaredName(parentFunction) : null;
+  }
+  if (container.kind() === 'impl_item') {
+    return safeField(container, 'type')?.text().trim()
+      || container.text().match(/^\s*impl(?:<[^>]+>)?\s+([^\s{]+)/u)?.[1]
+      || null;
+  }
+  return safeField(container, 'name')?.text().trim()
+    || container.children().find((child) => [
+      'constant', 'identifier', 'name', 'simple_identifier', 'type_identifier',
+    ].includes(child.kind()))?.text().trim()
+    || null;
+}
+
+function directText(node) {
+  return node.children().slice(0, 4).map((child) => child.text()).join(' ');
+}
+
+function cPlusPlusVisibility(node) {
+  const classBody = ancestor(node, ['field_declaration_list']);
+  if (!classBody) return /^\s*static\b/u.test(node.text()) ? 'internal' : 'public';
+  const classNode = ancestor(node, ['class_specifier', 'struct_specifier', 'union_specifier']);
+  let current = /^\s*class\b/u.test(classNode?.text() || '') ? 'private' : 'public';
+  for (const sibling of classBody.children()) {
+    if (sibling.range().start.index >= node.range().start.index) break;
+    if (sibling.kind() === 'access_specifier') current = sibling.text().replace(':', '').trim();
+  }
+  return current === 'public' ? 'public' : 'internal';
+}
+
+function rubyVisibility(node) {
+  const body = ancestor(node, ['body_statement']);
+  if (!body) return declaredName(node).startsWith('_') ? 'internal' : 'public';
+  let current = 'public';
+  for (const sibling of body.children()) {
+    if (sibling.range().start.index >= node.range().start.index) break;
+    const text = sibling.text().trim();
+    if (['private', 'protected', 'public'].includes(text)) current = text;
+  }
+  return current === 'public' && !declaredName(node).startsWith('_') ? 'public' : 'internal';
+}
+
+function javascriptExportedNames(source) {
+  const names = new Set();
+  for (const match of source.matchAll(/\bexport\s*\{([^}]+)\}/gu)) {
+    for (const item of match[1].split(',')) {
+      const local = item.trim().split(/\s+as\s+/u)[0];
+      if (local) names.add(local);
+    }
+  }
+  for (const match of source.matchAll(/\b(?:module\.)?exports\.([A-Za-z_$][\w$]*)\s*=/gu)) names.add(match[1]);
+  for (const match of source.matchAll(/\bmodule\.exports\s*=\s*\{([^}]+)\}/gu)) {
+    for (const item of match[1].split(',')) {
+      const value = item.trim().split(':').at(-1)?.trim();
+      if (/^[A-Za-z_$][\w$]*$/u.test(value || '')) names.add(value);
+    }
+  }
+  return names;
+}
+
+function javascriptVisibility(node, name, exportedNames) {
+  if (name.startsWith('#') || /\b(?:private|protected)\s+/u.test(directText(node))) return 'internal';
+  const classNode = ancestor(node, ['class_declaration', 'class_expression']);
+  if (classNode) {
+    const className = declaredName(classNode);
+    const exported = classNode.parent()?.kind() === 'export_statement' || exportedNames.has(className);
+    return exported ? 'public' : 'internal';
+  }
+  const declaration = ['arrow_function', 'function_expression'].includes(node.kind())
+    ? ancestor(node, ['lexical_declaration', 'variable_declaration'])
+    : node;
+  return declaration?.parent()?.kind() === 'export_statement' || exportedNames.has(name)
+    ? 'public'
+    : 'internal';
+}
+
+function visibility(node, name, language, exportedNames) {
+  const head = directText(node);
+  if (language === 'javascript') return javascriptVisibility(node, name, exportedNames);
+  if (language === 'python') {
+    const nestedFunction = node.ancestors().some((candidate) => candidate.kind() === 'function_definition');
+    return !nestedFunction && !name.startsWith('_') ? 'public' : 'internal';
+  }
+  if (['java', 'csharp'].includes(language)) return /\bpublic\b/u.test(head) ? 'public' : 'internal';
+  if (language === 'cpp') return cPlusPlusVisibility(node);
+  if (language === 'php') return /\b(?:private|protected)\b/u.test(head) ? 'internal' : 'public';
+  if (language === 'go') return /^\p{Lu}/u.test(name) ? 'public' : 'internal';
+  if (language === 'rust') return /^\s*pub(?:\([^)]*\))?\s+/u.test(node.text()) ? 'public' : 'internal';
+  if (language === 'ruby') return rubyVisibility(node);
+  if (language === 'kotlin') return /\b(?:private|protected|internal)\b/u.test(head) ? 'internal' : 'public';
+  if (language === 'shell') return name.startsWith('_') ? 'internal' : 'public';
+  return 'internal';
+}
+
+function callableKind(node, name) {
+  if (node.kind().includes('constructor') || ['__construct', '__init__', 'constructor'].includes(name)) return 'constructor';
+  if (node.kind().includes('method')) return 'method';
+  const owner = node.ancestors().find((candidate) => (
+    CONTAINER_KINDS.has(candidate.kind()) || CALLABLE_KINDS.has(candidate.kind())
+  ));
+  return owner && CONTAINER_KINDS.has(owner.kind()) ? 'method' : 'function';
+}
+
+function scriptUnits(source, filePath, definition) {
+  if (!filePath.endsWith('.vue')) {
+    const parser = typeof definition.parser === 'function' ? definition.parser(filePath) : definition.parser;
+    return [{ source, lineOffset: 0, parser }];
+  }
+  const units = [];
+  for (const match of source.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script\s*>/giu)) {
+    const content = match[2] || '';
+    const contentIndex = (match.index || 0) + match[0].indexOf(content);
+    const typescript = /\blang\s*=\s*["']tsx?["']/iu.test(match[1] || '');
+    const jsx = /\blang\s*=\s*["'](?:jsx|tsx)["']/iu.test(match[1] || '');
+    units.push({
+      source: content,
+      lineOffset: source.slice(0, contentIndex).split('\n').length - 1,
+      parser: jsx ? 'Tsx' : typescript ? 'TypeScript' : 'JavaScript',
+    });
+  }
+  return units;
+}
+
+function functionsInTree(root, definition, { language, lineOffset, source }) {
+  const exportedNames = language === 'javascript' ? javascriptExportedNames(source) : new Set();
+  const nodes = new Map();
+  for (const kind of definition.kinds) {
+    for (const node of root.findAll({ rule: { kind } })) nodes.set(node.id(), node);
+  }
+  return [...nodes.values()].sort((left, right) => left.range().start.index - right.range().start.index)
+    .map((node) => {
+      const name = declaredName(node);
+      if (!name) return null;
+      const container = containerName(node, language);
+      const location = node.range().start;
+      const parameters = splitParameters(parameterText(node));
+      const resolvedVisibility = visibility(node, name, language, exportedNames);
+      return {
+        name,
+        qualifiedName: [container, name].filter(Boolean).join(language === 'php' ? '::' : '.'),
+        kind: callableKind(node, name),
+        visibility: resolvedVisibility,
+        container,
+        line: location.line + lineOffset + 1,
+        column: location.column + 1,
+        parameters,
+        async: /^\s*async\b/u.test(node.text()),
+        generator: /\bfunction\s*\*|\byield\b/u.test(node.text()),
+        static: /\bstatic\b/u.test(directText(node)),
+      };
+    }).filter(Boolean);
+}
+
+function hasSupportedExtension(filePath, definition) {
+  const normalized = filePath.toLowerCase();
+  return definition.extensions.some((extension) => normalized.endsWith(extension));
+}
+
+function isCandidate(file, definition) {
+  if (file.path.split('/').some((segment) => EXCLUDED_DIRECTORIES.has(segment))) return false;
+  return hasSupportedExtension(file.path, definition)
+    || (definition.shebang && file.mode === 0o100755 && path.posix.extname(file.path) === '');
+}
+
+function fileRole(filePath) {
+  return /(?:^|\/)(?:__tests__|tests?|spec)(?:\/|$)|\.(?:spec|test)\.[^.]+$|(?:Test|Spec)\.[^.]+$/u.test(filePath)
+    ? 'test'
+    : 'source';
+}
+
+function syntaxDiagnostics(root, filePath, lineOffset) {
+  return root.findAll({ rule: { kind: 'ERROR' } }).slice(0, 20).map((node) => ({
+    code: 'CODE_INDEX_PARSE_RECOVERED',
+    message: 'The parser recovered from invalid or unsupported syntax.',
+    path: filePath,
+    line: node.range().start.line + lineOffset + 1,
+  }));
+}
+
+function createIndexer(id) {
+  const definition = DEFINITIONS[id];
+  return {
+    id,
+    async extract({ files, projectRoot }) {
+      const indexedFiles = [];
+      const diagnostics = [];
+      for (const file of files.filter((candidate) => isCandidate(candidate, definition))) {
+        let source;
+        try {
+          source = await readFile(path.join(projectRoot, file.path), 'utf8');
+        } catch (error) {
+          diagnostics.push({
+            code: 'CODE_INDEX_FILE_READ_FAILED',
+            message: error.message,
+            path: file.path,
+          });
+          continue;
+        }
+        if (Buffer.byteLength(source) > MAX_SOURCE_BYTES) {
+          diagnostics.push({
+            code: 'CODE_INDEX_FILE_TOO_LARGE',
+            message: `Skipped a source file larger than ${MAX_SOURCE_BYTES} bytes.`,
+            path: file.path,
+          });
+          continue;
+        }
+        if (!hasSupportedExtension(file.path, definition) && !definition.shebang?.test(source)) continue;
+        const functions = [];
+        for (const unit of scriptUnits(source, file.path, definition)) {
+          try {
+            const parsed = await parseAsync(unit.parser, unit.source);
+            const root = parsed.root();
+            functions.push(...functionsInTree(root, definition, {
+              language: id,
+              lineOffset: unit.lineOffset,
+              source: unit.source,
+            }));
+            diagnostics.push(...syntaxDiagnostics(root, file.path, unit.lineOffset));
+          } catch (error) {
+            diagnostics.push({
+              code: 'CODE_INDEX_PARSE_FAILED',
+              message: error.message,
+              path: file.path,
+            });
+          }
+        }
+        indexedFiles.push({
+          path: file.path,
+          language: id,
+          role: fileRole(file.path),
+          mode: file.mode,
+          bytes: Buffer.byteLength(source),
+          lines: source.length === 0 ? 0 : source.split('\n').length - (source.endsWith('\n') ? 1 : 0),
+          hash: file.hash,
+          functions,
+        });
+      }
+      return { files: indexedFiles, diagnostics };
+    },
+  };
+}
+
+export const astGrepCodeIndexers = new Map(
+  Object.keys(DEFINITIONS).map((id) => [id, createIndexer(id)]),
+);