@saluzi/codegraph 0.1.0 → 0.2.1

package/dist/extraction/index.d.ts

@@ -3,47 +3,150 @@
  *
  * Coordinates file scanning, parsing, and database storage.
  */
-import * as fs from 'fs'
-import { ExtractionResult, ExtractionError } from '../types'
-import { QueryBuilder } from '../db/queries'
+import * as fs from 'fs';
+import { ExtractionResult, ExtractionError } from '../types';
+import { QueryBuilder } from '../db/queries';
+import { Ignore } from 'ignore';
 /**
  * Progress callback for indexing operations
  */
 export interface IndexProgress {
-  phase: 'scanning' | 'parsing' | 'storing' | 'resolving'
-  current: number
-  total: number
-  currentFile?: string
+    phase: 'scanning' | 'parsing' | 'storing' | 'resolving';
+    current: number;
+    total: number;
+    currentFile?: string;
 }
 /**
  * Result of an indexing operation
  */
 export interface IndexResult {
-  success: boolean
-  filesIndexed: number
-  filesSkipped: number
-  filesErrored: number
-  nodesCreated: number
-  edgesCreated: number
-  errors: ExtractionError[]
-  durationMs: number
+    success: boolean;
+    filesIndexed: number;
+    filesSkipped: number;
+    filesErrored: number;
+    /**
+     * How many indexable files the scan discovered — the ground truth the
+     * indexed/skipped/errored tallies must add up to. A shortfall means files
+     * were silently dropped mid-pipeline (e.g. a killed worker under load) and
+     * the index is PARTIAL; callers surface that rather than trusting the
+     * counts. Only set by full-index runs (indexAll), not indexFiles/sync.
+     */
+    filesDiscovered?: number;
+    nodesCreated: number;
+    edgesCreated: number;
+    errors: ExtractionError[];
+    durationMs: number;
 }
 /**
  * Result of a sync operation
  */
 export interface SyncResult {
-  filesChecked: number
-  filesAdded: number
-  filesModified: number
-  filesRemoved: number
-  nodesUpdated: number
-  durationMs: number
-  changedFilePaths?: string[]
+    filesChecked: number;
+    filesAdded: number;
+    filesModified: number;
+    filesRemoved: number;
+    nodesUpdated: number;
+    durationMs: number;
+    changedFilePaths?: string[];
 }
 /**
  * Calculate SHA256 hash of file contents
  */
-export declare function hashContent(content: string): string
+export declare function hashContent(content: string): string;
+/**
+ * An `ignore` matcher seeded with the built-in defaults, merged with the project's
+ * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
+ * by both enumeration paths so behavior is identical with or without git — and so
+ * the defaults apply to tracked files too (committing a dependency dir doesn't make
+ * it project code; the explicit `.gitignore` negation is the only opt-in).
+ */
+export declare function buildDefaultIgnore(rootDir: string): Ignore;
+/**
+ * Workspace-scope ignore matcher. Ordinary paths get the root's matcher
+ * (built-in defaults + root `.gitignore`); paths inside an EMBEDDED repo get
+ * that repo's own matcher (defaults + its root `.gitignore`) — the parent's
+ * `.gitignore` hides a child repo from git, not from the index (#514). A
+ * directory path (trailing slash) that is an ANCESTOR of an embedded root is
+ * never ignored, so directory-pruning callers (the Linux per-directory
+ * watcher) still descend to reach the embedded repos.
+ *
+ * Single source of truth for indexer and watcher scope — they must not diverge.
+ */
+export declare class ScopeIgnore {
+    private rootMatcher;
+    /**
+     * Project `codegraph.json` `exclude` patterns (#999), matched against the
+     * full root-relative path. Wins over everything else — an explicit user
+     * exclude applies even to tracked files and even inside embedded repos.
+     */
+    private exclude;
+    /**
+     * Project `codegraph.json` `include` patterns — first-party source forced
+     * INTO the index despite `.gitignore`. When a path matches, it is NOT
+     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
+     * — but never `exclude` (checked first) and never a built-in default-ignored
+     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
+     * directory of an included subtree still isn't pruned by the directory
+     * walker/watcher.
+     */
+    private include;
+    private includeRoots;
+    private embedded;
+    private defaults;
+    constructor(rootMatcher: Ignore, embedded: Array<{
+        root: string;
+        matcher: Ignore;
+    }>, 
+    /**
+     * Project `codegraph.json` `exclude` patterns (#999), matched against the
+     * full root-relative path. Wins over everything else — an explicit user
+     * exclude applies even to tracked files and even inside embedded repos.
+     */
+    exclude?: Ignore | null, 
+    /**
+     * Project `codegraph.json` `include` patterns — first-party source forced
+     * INTO the index despite `.gitignore`. When a path matches, it is NOT
+     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
+     * — but never `exclude` (checked first) and never a built-in default-ignored
+     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
+     * directory of an included subtree still isn't pruned by the directory
+     * walker/watcher.
+     */
+    include?: Ignore | null, includeRoots?: string[]);
+    ignores(rel: string): boolean;
+}
+/**
+ * Build the workspace-scope matcher. When the caller already knows the
+ * embedded roots (the scanner discovers them during collection), pass them to
+ * skip rediscovery; otherwise they're discovered here (the watcher path).
+ */
+export declare function buildScopeIgnore(rootDir: string, embeddedRoots?: Iterable<string>): ScopeIgnore;
+/**
+ * Standalone discovery of every embedded repo root under `rootDir` (relative,
+ * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
+ * (#514) only for directories the project opted in via `codegraph.json`
+ * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
+ * are not discovered (#970, #976). Recursive (an embedded repo can embed further
+ * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
+ * there already.
+ */
+export declare function discoverEmbeddedRepoRoots(rootDir: string): string[];
+/**
+ * The INVERSE of the gitignored side of {@link discoverEmbeddedRepoRoots}:
+ * nested git repositories under a gitignored directory that the project has NOT
+ * opted into via `codegraph.json` `includeIgnored`. These are real repos the
+ * default `init`/`index` deliberately skips because `.gitignore` excludes them
+ * (#970, #976) — most visibly the "super-repo `.gitignore`s its child repos"
+ * layout (#1156), where `init` at the parent correctly indexes ~nothing while
+ * `init` inside each child works. The CLI uses this to turn that silent empty
+ * index into an actionable hint: it names the skipped repos and offers to opt
+ * them in. Paths are `rootDir`-relative and trailing-slashed (valid
+ * `includeIgnored` patterns as-is). Returns `[]` for a non-git root (a
+ * filesystem walk already descends into nested repos there), skips built-in
+ * default-ignored dirs (`node_modules`, …), and is bounded so it never stalls
+ * on a giant ignored tree.
+ */
+export declare function findUnindexedIgnoredRepos(rootDir: string): string[];
 /**
  * Recursively scan a directory for source files.
  *
@@ -51,98 +154,79 @@ export declare function hashContent(content: string): string
  * levels), then keeps files with a supported source extension. For non-git
  * projects, falls back to a filesystem walk that parses .gitignore itself.
  */
-export declare function scanDirectory(
-  rootDir: string,
-  onProgress?: (current: number, file: string) => void,
-): string[]
+export declare function scanDirectory(rootDir: string, onProgress?: (current: number, file: string) => void): string[];
 /**
  * Async variant of scanDirectory that yields to the event loop periodically,
  * allowing worker threads to receive and render progress messages.
  */
-export declare function scanDirectoryAsync(
-  rootDir: string,
-  onProgress?: (current: number, file: string) => void,
-): Promise<string[]>
+export declare function scanDirectoryAsync(rootDir: string, onProgress?: (current: number, file: string) => void): Promise<string[]>;
 /**
  * Extraction orchestrator
  */
 export declare class ExtractionOrchestrator {
-  private rootDir
-  private queries
-  /**
-   * Names of frameworks detected for this project, populated by indexAll().
-   * Passed to extractFromSource so framework-specific extractors (route nodes,
-   * middleware, etc.) run after the tree-sitter pass. Cleared if detection
-   * hasn't run yet so single-file re-index paths can detect on the spot.
-   */
-  private detectedFrameworkNames
-  constructor(rootDir: string, queries: QueryBuilder)
-  /**
-   * Build a filesystem-backed ResolutionContext sufficient for framework
-   * detection. Graph-query methods (getNodesByName etc.) return empty because
-   * the DB hasn't been populated yet, but detect() only uses readFile,
-   * fileExists, and getAllFiles, so that's fine.
-   */
-  private buildDetectionContext
-  /**
-   * Detect frameworks on demand using the current scanned files (or a fresh
-   * scan if none are provided). Cached on the orchestrator so repeat calls
-   * inside a single run don't re-scan.
-   */
-  private ensureDetectedFrameworks
-  /**
-   * Index all files in the project
-   */
-  indexAll(
-    onProgress?: (progress: IndexProgress) => void,
-    signal?: AbortSignal,
-    verbose?: boolean,
-  ): Promise<IndexResult>
-  /**
-   * Index specific files
-   */
-  indexFiles(filePaths: string[]): Promise<IndexResult>
-  /**
-   * Index a single file
-   */
-  indexFile(relativePath: string): Promise<ExtractionResult>
-  /**
-   * Index a single file with pre-read content and stats.
-   * Used by the parallel batch reader to avoid redundant file I/O.
-   */
-  indexFileWithContent(
-    relativePath: string,
-    content: string,
-    stats: fs.Stats,
-  ): Promise<ExtractionResult>
-  /**
-   * Store extraction result in database
-   */
-  private storeExtractionResult
-  /**
-   * Sync with current file state.
-   * Uses git status as a fast path when available, falling back to full scan.
-   */
-  sync(onProgress?: (progress: IndexProgress) => void): Promise<SyncResult>
-  /**
-   * Get files that have changed since last index.
-   * Uses git status as a fast path when available, falling back to full scan.
-   */
-  getChangedFiles(): {
-    added: string[]
-    modified: string[]
-    removed: string[]
-  }
+    private rootDir;
+    private queries;
+    /**
+     * Names of frameworks detected for this project, populated by indexAll().
+     * Passed to extractFromSource so framework-specific extractors (route nodes,
+     * middleware, etc.) run after the tree-sitter pass. Cleared if detection
+     * hasn't run yet so single-file re-index paths can detect on the spot.
+     */
+    private detectedFrameworkNames;
+    constructor(rootDir: string, queries: QueryBuilder);
+    /**
+     * Build a filesystem-backed ResolutionContext sufficient for framework
+     * detection. Graph-query methods (getNodesByName etc.) return empty because
+     * the DB hasn't been populated yet, but detect() only uses readFile,
+     * fileExists, and getAllFiles, so that's fine.
+     */
+    private buildDetectionContext;
+    /**
+     * Detect frameworks on demand using the current scanned files (or a fresh
+     * scan if none are provided). Cached on the orchestrator so repeat calls
+     * inside a single run don't re-scan.
+     */
+    private ensureDetectedFrameworks;
+    /**
+     * Index all files in the project
+     */
+    indexAll(onProgress?: (progress: IndexProgress) => void, signal?: AbortSignal, verbose?: boolean, walBackpressure?: () => Promise<void> | null): Promise<IndexResult>;
+    /**
+     * Index specific files
+     */
+    indexFiles(filePaths: string[]): Promise<IndexResult>;
+    /**
+     * Index a single file
+     */
+    indexFile(relativePath: string): Promise<ExtractionResult>;
+    /**
+     * Index a single file with pre-read content and stats.
+     * Used by the parallel batch reader to avoid redundant file I/O.
+     */
+    indexFileWithContent(relativePath: string, content: string, stats: fs.Stats): Promise<ExtractionResult>;
+    /**
+     * Store extraction result in database
+     */
+    private storeExtractionResult;
+    /**
+     * Sync the index with the current file state.
+     *
+     * Change detection is filesystem-based, never git: a (size, mtime) stat
+     * pre-filter skips unchanged files, then a content-hash compare confirms real
+     * changes. This works in non-git projects and catches committed changes from
+     * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
+     */
+    sync(onProgress?: (progress: IndexProgress) => void): Promise<SyncResult>;
+    /**
+     * Get files that have changed since last index.
+     * Uses git status as a fast path when available, falling back to full scan.
+     */
+    getChangedFiles(): {
+        added: string[];
+        modified: string[];
+        removed: string[];
+    };
 }
-export { extractFromSource } from './tree-sitter'
-export {
-  detectLanguage,
-  isSourceFile,
-  isLanguageSupported,
-  isGrammarLoaded,
-  getSupportedLanguages,
-  initGrammars,
-  loadGrammarsForLanguages,
-  loadAllGrammars,
-} from './grammars'
-//# sourceMappingURL=index.d.ts.map
+export { extractFromSource } from './tree-sitter';
+export { detectLanguage, isSourceFile, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './grammars';
+//# sourceMappingURL=index.d.ts.map

package/dist/extraction/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/extraction/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAMzB,OAAO,EAGL,gBAAgB,EAChB,eAAe,EAIhB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAQ7C,OAAe,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA0CxC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD;AAoKD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAK1D;AA2TD;;;;;;;;;;GAUG;AACH,qBAAa,WAAW;IAIpB,OAAO,CAAC,WAAW;IAEnB;;;;OAIG;IACH,OAAO,CAAC,OAAO;IACf;;;;;;;;OAQG;IACH,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,YAAY;IArBtB,OAAO,CAAC,QAAQ,CAA2C;IAC3D,OAAO,CAAC,QAAQ,CAAgC;gBAEtC,WAAW,EAAE,MAAM,EAC3B,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD;;;;OAIG;IACK,OAAO,GAAE,MAAM,GAAG,IAAW;IACrC;;;;;;;;OAQG;IACK,OAAO,GAAE,MAAM,GAAG,IAAW,EAC7B,YAAY,GAAE,MAAM,EAAO;IAMrC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;CAiC9B;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,WAAW,CAU/F;AAoCD;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAwDnE;AASD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAkBnE;AA2TD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GACnD,MAAM,EAAE,CAqBV;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GACnD,OAAO,CAAC,MAAM,EAAE,CAAC,CAuBnB;AA8KD;;GAEG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAe;IAC9B;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB,CAAyB;gBAE3C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY;IAKlD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAkD7B;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAQhC;;OAEG;IACG,QAAQ,CACZ,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,EAC9C,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,OAAO,EAKjB,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GAC3C,OAAO,CAAC,WAAW,CAAC;IAkfvB;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IA4C3D;;OAEG;IACG,SAAS,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwChE;;;OAGG;IACG,oBAAoB,CACxB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,EAAE,CAAC,KAAK,GACd,OAAO,CAAC,gBAAgB,CAAC;IA0D5B;;OAEG;YACW,qBAAqB;IA+InC;;;;;;;OAOG;IACG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC;IA8J/E;;;OAGG;IACH,eAAe,IAAI;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE;CAwF9E;AAGD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,qBAAqB,EAAE,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}