@ontrails/source 1.0.0-beta.45 → 1.0.0-beta.47

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @ontrails/source
 
+## 1.0.0-beta.47
+
+### Minor Changes
+
+- [`90d394c`](https://github.com/outfitter-dev/trails/commit/90d394c005fdf6b898ba7052d0b56755af0f4954): Derive nested worktree, repository, and submodule collection boundaries in the
+  shared Source walker. Regrade and Warden now observe one directly targeted
+  working tree per run, and Regrade audit summaries expose boundary skip counts.
+
+## 1.0.0-beta.46
+
+### Minor Changes
+
+- [`54d259b`](https://github.com/outfitter-dev/trails/commit/54d259be81fb6c41d85be48a6cb2100c746a7126): Expose parser-native comment spans from `parseWithDiagnostics` so source-aware
+  tooling can distinguish exact JavaScript and TypeScript comment trivia without
+  reimplementing a lexer.
+
+  Use the shared spans in Warden's public-example rule while keeping leading
+  comment ownership fail-closed across JavaScript line terminators.
+
 ## 1.0.0-beta.45
 
 ## 1.0.0-beta.44

package/README.md

@@ -9,7 +9,8 @@ Shared source-code machinery for Trails packages and repo tooling.
 `@ontrails/source` owns reusable source-code mechanics:
 
 - AST node guards and accessors for the OXC node shapes Trails tooling uses.
-- `parse` and `parseWithDiagnostics` wrappers over `oxc-parser`.
+- `parse` and `parseWithDiagnostics` wrappers over `oxc-parser`, including
+  parser-native comment spans for tools that must distinguish source trivia.
 - `walk`, parent-aware walking, and scope-aware walking over `oxc-walker`.
 - Source locations, source edits, literal extraction, and generic Trails syntax recognition.
 - Generic trail/entity discovery helpers such as `findTrailDefinitions`, `findImplementationBodies`, `findEntityDefinitions`, and `isImplementationCall`.
@@ -44,6 +45,21 @@ const ast = parse(
 const trailIds = ast ? findTrailDefinitions(ast).map((trail) => trail.id) : [];
 ```
 
+Inspect exact comment spans without rebuilding a JavaScript or TypeScript lexer:
+
+```ts
+import { parseWithDiagnostics } from '@ontrails/source';
+
+const sourceCode = '/** Describe a trail. */\nexport const show = 1;\n';
+const parsed = parseWithDiagnostics('example.ts', sourceCode);
+const comments = parsed.comments.map((comment) => ({
+  ...comment,
+  source: sourceCode.slice(comment.start, comment.end),
+}));
+```
+
+Comment spans are returned only when the parser reports no diagnostics. Tools must treat an empty comment inventory on a recovered parse as unknown rather than as proof that the source contains no comments.
+
 Walk source with parent context:
 
 ```ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ontrails/source",
-  "version": "1.0.0-beta.45",
+  "version": "1.0.0-beta.47",
   "description": "Shared source-code AST parsing, walking, location, edit, literal, and Trails syntax helpers.",
   "repository": {
     "type": "git",

package/src/collection.ts

@@ -0,0 +1,312 @@
+import { spawnSync } from 'node:child_process';
+import { lstatSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import type { Dirent } from 'node:fs';
+import { join, posix, relative, resolve, sep } from 'node:path';
+
+/** Git-derived reasons a nested directory is outside the current observation tree. */
+export type SourceCollectionBoundaryReason =
+  | 'nested-repository'
+  | 'nested-worktree'
+  | 'submodule-boundary';
+
+/** Filesystem entry kinds exposed to a source-collection policy. */
+export type SourceCollectionEntryKind = 'directory' | 'file' | 'other';
+
+/** A root-relative filesystem entry offered to a source-collection policy. */
+export interface SourceCollectionEntry {
+  readonly kind: SourceCollectionEntryKind;
+  readonly name: string;
+  readonly path: string;
+}
+
+/** The action a source-collection policy chooses for one entry. */
+export type SourceCollectionDecision =
+  | { readonly action: 'collect' }
+  | { readonly action: 'recurse' }
+  | { readonly action: 'skip'; readonly reason: string };
+
+/** One collected file, with both absolute and root-relative identities. */
+export interface CollectedSourceFile {
+  readonly absolutePath: string;
+  readonly path: string;
+}
+
+/** One visible collection skip. */
+export interface SkippedSourceEntry {
+  readonly path: string;
+  readonly reason: string;
+}
+
+/** Deterministic result of observing one source tree. */
+export interface SourceTreeCollection {
+  readonly files: readonly CollectedSourceFile[];
+  readonly root: string;
+  readonly skipped: readonly SkippedSourceEntry[];
+}
+
+/** Consumer policy for ordinary entries after Git boundaries are derived. */
+export interface CollectSourceTreeOptions {
+  readonly classify?: (
+    entry: SourceCollectionEntry
+  ) => SourceCollectionDecision;
+}
+
+const toPosixRelative = (root: string, absolutePath: string): string => {
+  const path = relative(root, absolutePath);
+  return sep === posix.sep ? path : path.split(sep).join(posix.sep);
+};
+
+const entryKind = (entry: {
+  isDirectory(): boolean;
+  isFile(): boolean;
+}): SourceCollectionEntryKind => {
+  if (entry.isDirectory()) {
+    return 'directory';
+  }
+  return entry.isFile() ? 'file' : 'other';
+};
+
+const defaultDecision = (
+  entry: SourceCollectionEntry
+): SourceCollectionDecision => {
+  if (entry.kind === 'directory') {
+    return { action: 'recurse' };
+  }
+  return entry.kind === 'file'
+    ? { action: 'collect' }
+    : { action: 'skip', reason: 'unsupported-entry' };
+};
+
+interface SubmodulePathSnapshot {
+  readonly paths: ReadonlySet<string>;
+  readonly readable: boolean;
+}
+
+const readSubmodulePaths = (root: string): SubmodulePathSnapshot => {
+  const gitmodulesPath = join(root, '.gitmodules');
+  let metadataStats: ReturnType<typeof lstatSync>;
+  try {
+    metadataStats = lstatSync(gitmodulesPath);
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === 'ENOENT'
+      ? { paths: new Set(), readable: true }
+      : { paths: new Set(), readable: false };
+  }
+  if (!metadataStats.isFile()) {
+    return { paths: new Set(), readable: false };
+  }
+
+  let source: string;
+  try {
+    source = readFileSync(gitmodulesPath, 'utf8');
+  } catch {
+    return { paths: new Set(), readable: false };
+  }
+  const parsed = spawnSync(
+    'git',
+    ['config', '--file', '-', '--null', '--get-regexp', '^submodule\\.'],
+    { encoding: 'utf8', input: source }
+  );
+  if (parsed.status === 1) {
+    const hasOnlyComments = source
+      .split(/\r?\n/)
+      .every((line) => !line.trim() || /^[#;]/.test(line.trim()));
+    return {
+      paths: new Set(),
+      readable: hasOnlyComments,
+    };
+  }
+  if (parsed.status !== 0) {
+    return { paths: new Set(), readable: false };
+  }
+
+  const paths = new Set<string>();
+  const owners = new Set<string>();
+  const ownersWithPaths = new Set<string>();
+  const authoredSubmodules = new Set(
+    [
+      ...source.matchAll(
+        /^\s*\[\s*submodule(?:\s+"((?:[^"\\]|\\.)*)"|\.([^\]]+?))\s*\]/gim
+      ),
+    ].map((match) => match[1] ?? match[2]?.trim())
+  );
+  for (const record of parsed.stdout.split('\0').filter(Boolean)) {
+    const separator = record.indexOf('\n');
+    if (separator === -1) {
+      return { paths: new Set(), readable: false };
+    }
+    const key = record.slice(0, separator);
+    const fieldSeparator = key.lastIndexOf('.');
+    if (fieldSeparator <= 'submodule.'.length) {
+      return { paths: new Set(), readable: false };
+    }
+    const owner = key.slice('submodule.'.length, fieldSeparator);
+    owners.add(owner);
+    if (key.slice(fieldSeparator + 1).toLowerCase() !== 'path') {
+      continue;
+    }
+    const value = record.slice(separator + 1);
+    if (!value) {
+      return { paths: new Set(), readable: false };
+    }
+    ownersWithPaths.add(owner);
+    paths.add(posix.normalize(value.replaceAll('\\', '/')).replace(/\/+$/, ''));
+  }
+  return owners.size === ownersWithPaths.size &&
+    authoredSubmodules.size === ownersWithPaths.size
+    ? { paths, readable: true }
+    : { paths: new Set(), readable: false };
+};
+
+const boundaryReason = (
+  absoluteDirectory: string,
+  path: string,
+  submodules: SubmodulePathSnapshot
+): SourceCollectionBoundaryReason | 'unreadable-git-boundary' | undefined => {
+  if (!submodules.readable) {
+    return 'unreadable-git-boundary';
+  }
+  if (submodules.paths.has(path)) {
+    return 'submodule-boundary';
+  }
+
+  const marker = join(absoluteDirectory, '.git');
+  let markerStats: ReturnType<typeof lstatSync>;
+  try {
+    markerStats = lstatSync(marker);
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === 'ENOENT'
+      ? undefined
+      : 'unreadable-git-boundary';
+  }
+
+  try {
+    if (markerStats.isDirectory()) {
+      return statSync(join(marker, 'HEAD')).isFile() &&
+        statSync(join(marker, 'objects')).isDirectory()
+        ? 'nested-repository'
+        : 'unreadable-git-boundary';
+    }
+    if (!markerStats.isFile()) {
+      return 'unreadable-git-boundary';
+    }
+
+    const pointer = readFileSync(marker, 'utf8').match(/^gitdir:\s*(.+?)\s*$/);
+    const gitDirectory = pointer?.[1];
+    if (!gitDirectory) {
+      return 'unreadable-git-boundary';
+    }
+    const resolvedGitDirectory = resolve(absoluteDirectory, gitDirectory);
+    return statSync(resolvedGitDirectory).isDirectory() &&
+      statSync(join(resolvedGitDirectory, 'HEAD')).isFile()
+      ? 'nested-worktree'
+      : 'unreadable-git-boundary';
+  } catch {
+    return 'unreadable-git-boundary';
+  }
+};
+
+const comparePath = (
+  left: { readonly path: string },
+  right: {
+    readonly path: string;
+  }
+): number => {
+  if (left.path < right.path) {
+    return -1;
+  }
+  return left.path > right.path ? 1 : 0;
+};
+
+/**
+ * Collect files from exactly one working tree, pruning nested Git observations.
+ *
+ * Git boundaries take precedence over consumer policy so a tool cannot scan a
+ * nested checkout and hide that fact behind an authored exclude. The supplied
+ * root remains first-class: only directories encountered beneath it are
+ * classified as boundaries.
+ *
+ * @example
+ * ```ts
+ * const result = collectSourceTree(process.cwd(), {
+ *   classify: (entry) =>
+ *     entry.kind === 'file' && entry.path.endsWith('.ts')
+ *       ? { action: 'collect' }
+ *       : entry.kind === 'directory'
+ *         ? { action: 'recurse' }
+ *         : { action: 'skip', reason: 'unsupported-extension' },
+ * });
+ * ```
+ */
+export const collectSourceTree = (
+  root: string,
+  options: CollectSourceTreeOptions = {}
+): SourceTreeCollection | null => {
+  const absoluteRoot = resolve(root);
+  let rootEntries: readonly Dirent<string>[];
+  try {
+    rootEntries = readdirSync(absoluteRoot, { withFileTypes: true });
+  } catch {
+    return null;
+  }
+
+  const classify = options.classify ?? defaultDecision;
+  const submodules = readSubmodulePaths(absoluteRoot);
+  const files: CollectedSourceFile[] = [];
+  const skipped: SkippedSourceEntry[] = submodules.readable
+    ? []
+    : [{ path: '.gitmodules', reason: 'unreadable-git-metadata' }];
+  const queue: {
+    readonly absolutePath: string;
+    readonly entries?: readonly Dirent<string>[];
+  }[] = [{ absolutePath: absoluteRoot, entries: rootEntries }];
+
+  while (queue.length > 0) {
+    const current = queue.shift() as (typeof queue)[number];
+    let { entries } = current;
+    if (!entries) {
+      try {
+        entries = readdirSync(current.absolutePath, { withFileTypes: true });
+      } catch {
+        skipped.push({
+          path: toPosixRelative(absoluteRoot, current.absolutePath),
+          reason: 'unreadable-directory',
+        });
+        continue;
+      }
+    }
+
+    for (const entry of entries) {
+      const absolutePath = join(current.absolutePath, entry.name);
+      const path = toPosixRelative(absoluteRoot, absolutePath);
+      const kind = entryKind(entry);
+      if (path === '.gitmodules' && !submodules.readable) {
+        continue;
+      }
+      if (kind === 'directory') {
+        const reason = boundaryReason(absolutePath, path, submodules);
+        if (reason) {
+          skipped.push({ path, reason });
+          continue;
+        }
+      }
+      if (entry.name === '.git') {
+        skipped.push({ path, reason: 'ignored-directory' });
+        continue;
+      }
+
+      const decision = classify({ kind, name: entry.name, path });
+      if (decision.action === 'collect') {
+        files.push({ absolutePath, path });
+      } else if (decision.action === 'recurse') {
+        queue.push({ absolutePath });
+      } else {
+        skipped.push({ path, reason: decision.reason });
+      }
+    }
+  }
+
+  files.sort(comparePath);
+  skipped.sort(comparePath);
+  return { files, root: absoluteRoot, skipped };
+};

package/src/index.ts

@@ -5,6 +5,7 @@ export * from './scopes.js';
 export * from './locations.js';
 export * from './edits.js';
 export * from './literals.js';
+export * from './collection.js';
 export type {
   EntityDefinition,
   FindEntityDefinitionsOptions,

package/src/nodes.ts

@@ -245,8 +245,19 @@ export interface AstParseDiagnostic {
   readonly severity: string;
 }
 
+/** Parser-native source comment span. Offsets include the comment delimiters. */
+export interface SourceComment {
+  readonly end: number;
+  readonly start: number;
+  readonly type: 'Block' | 'Line';
+  /** Comment text without the line or block delimiters. */
+  readonly value: string;
+}
+
 export interface AstParseResult {
   readonly ast: AstNode | null;
+  /** Exact parser-native comments for a diagnostic-free parse; empty on errors. */
+  readonly comments: readonly SourceComment[];
   readonly diagnostics: readonly AstParseDiagnostic[];
 }
 

package/src/parse.ts

@@ -25,22 +25,25 @@ export const parseWithDiagnostics = (
 ): AstParseResult => {
   try {
     const result = parseSync(filePath, sourceCode, { sourceType: 'module' });
+    const diagnostics = result.errors.map((error) => ({
+      helpMessage: error.helpMessage,
+      labels: error.labels.map((label) => ({
+        end: label.end,
+        message: label.message,
+        start: label.start,
+      })),
+      message: error.message,
+      severity: error.severity,
+    }));
     return {
       ast: result.program as unknown as AstNode,
-      diagnostics: result.errors.map((error) => ({
-        helpMessage: error.helpMessage,
-        labels: error.labels.map((label) => ({
-          end: label.end,
-          message: label.message,
-          start: label.start,
-        })),
-        message: error.message,
-        severity: error.severity,
-      })),
+      comments: diagnostics.length === 0 ? result.comments : [],
+      diagnostics,
     };
   } catch (error) {
     return {
       ast: null,
+      comments: [],
       diagnostics: [
         {
           helpMessage: null,