@dataparade/cli 0.0.1

package/dist/src/data-flow/terraform-flows.d.ts

@@ -0,0 +1,32 @@
+import type { DetectedComponent } from "../core/types/component";
+import type { DetectedDataFlow } from "../core/types/data-flow";
+/**
+ * Infer edges between Terraform-managed assets from `terraform_references`
+ * merged onto components (see Terraform detector).
+ */
+export declare function appendTerraformDataFlows(components: DetectedComponent[], startIndex: number): {
+    flows: DetectedDataFlow[];
+    nextIndex: number;
+};
+export declare function terraformProviderLocalNameFromResourceType(resourceType: string): string | undefined;
+/**
+ * Resolves the `provider.<local>` third_party for a Terraform-managed asset
+ * (e.g. `aws_s3_bucket.*` → `provider.aws`).
+ */
+export declare function findTerraformProviderForResourceAsset(components: DetectedComponent[], asset: DetectedComponent): DetectedComponent | undefined;
+/**
+ * Same role as the former `appendTerraformProviderAttachmentFlows`, but runs
+ * from {@link applyDeterministicInferenceFallbacks} **after** provider-topology
+ * managed nodes (so flow types / dedupe match the TypeScript path). Skips
+ * assets already tagged with `managed_by_provider` and skips pairs already in
+ * `pairKeys` (e.g. provider → resource edges emitted as `file_transfer`).
+ *
+ * Skips resources declared under a `module.*` address so the provider node does
+ * not fan out to every nested resource (references / topology already link the
+ * graph).
+ */
+export declare function appendTerraformBareProviderAttachmentFlows(input: {
+    components: DetectedComponent[];
+    flows: DetectedDataFlow[];
+    pairKeys: Set<string>;
+}): void;

package/dist/src/data-flow/terraform-flows.js

@@ -0,0 +1,170 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendTerraformDataFlows = appendTerraformDataFlows;
+exports.terraformProviderLocalNameFromResourceType = terraformProviderLocalNameFromResourceType;
+exports.findTerraformProviderForResourceAsset = findTerraformProviderForResourceAsset;
+exports.appendTerraformBareProviderAttachmentFlows = appendTerraformBareProviderAttachmentFlows;
+const flow_builder_1 = require("./flow-builder");
+function flowTypeForTerraformTarget(target) {
+    const st = target?.subType?.toLowerCase();
+    if (st === "database")
+        return "database_query";
+    if (st === "queue")
+        return "message_queue";
+    return "api_call";
+}
+/**
+ * Terraform HCL references point from the dependent block to dependencies.
+ * For most edges we want **provision / data-supply direction**: referenced
+ * resource → block that consumes it (matches apply order and typical
+ * "infrastructure feeds this resource" reading).
+ *
+ * **Exception:** `database_query` keeps **caller → database** (the
+ * referencing workload initiates queries toward the DB).
+ */
+function terraformReferenceFlowEndpoints(referrer, refd, flowType) {
+    if (flowType === "database_query") {
+        return { source: referrer, target: refd };
+    }
+    return { source: refd, target: referrer };
+}
+function syntheticTerraformFinding(source) {
+    const loc = source.sourceLocations?.[0] ?? {
+        filePath: "terraform",
+        startLine: 1,
+        endLine: 1,
+    };
+    return {
+        pattern: "terraform_resource",
+        name: "terraform_reference",
+        confidence: 0.85,
+        location: loc,
+        properties: {},
+    };
+}
+/**
+ * Infer edges between Terraform-managed assets from `terraform_references`
+ * merged onto components (see Terraform detector).
+ */
+function appendTerraformDataFlows(components, startIndex) {
+    const flows = [];
+    let flowIndex = startIndex;
+    const addressToComponent = new Map();
+    for (const c of components) {
+        const addr = c.properties?.terraform_address;
+        if (typeof addr === "string" && addr.trim()) {
+            addressToComponent.set(addr.trim(), c);
+        }
+    }
+    const seen = new Set();
+    for (const component of components) {
+        const sourceAddr = component.properties?.terraform_address;
+        if (typeof sourceAddr !== "string" || !sourceAddr.trim())
+            continue;
+        const rawRefs = component.properties?.terraform_references;
+        if (!Array.isArray(rawRefs) || rawRefs.length === 0)
+            continue;
+        const refs = rawRefs.filter((r) => typeof r === "string");
+        for (const ref of refs) {
+            const target = addressToComponent.get(ref);
+            if (!target || target.id === component.id)
+                continue;
+            const flowType = flowTypeForTerraformTarget(target);
+            const { source, target: dest } = terraformReferenceFlowEndpoints(component, target, flowType);
+            const key = `${source.id}::${dest.id}::${flowType}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            flowIndex += 1;
+            const finding = syntheticTerraformFinding(source);
+            flows.push((0, flow_builder_1.buildFlow)(source.id, dest.id, flowType, finding, flowIndex, source, dest));
+        }
+    }
+    return { flows, nextIndex: flowIndex };
+}
+function terraformProviderLocalNameFromResourceType(resourceType) {
+    const rt = resourceType.trim().toLowerCase();
+    const idx = rt.indexOf("_");
+    if (idx <= 0)
+        return undefined;
+    return rt.slice(0, idx);
+}
+/**
+ * Resolves the `provider.<local>` third_party for a Terraform-managed asset
+ * (e.g. `aws_s3_bucket.*` → `provider.aws`).
+ */
+function findTerraformProviderForResourceAsset(components, asset) {
+    if (asset.type !== "asset")
+        return undefined;
+    const rt = asset.properties?.resource_type;
+    if (typeof rt !== "string" || !rt.includes("_"))
+        return undefined;
+    const localName = terraformProviderLocalNameFromResourceType(rt);
+    if (!localName)
+        return undefined;
+    const wantAddr = `provider.${localName}`;
+    return components.find((c) => c.type === "third_party" &&
+        typeof c.properties?.terraform_address === "string" &&
+        c.properties.terraform_address.trim() === wantAddr);
+}
+/**
+ * Same role as the former `appendTerraformProviderAttachmentFlows`, but runs
+ * from {@link applyDeterministicInferenceFallbacks} **after** provider-topology
+ * managed nodes (so flow types / dedupe match the TypeScript path). Skips
+ * assets already tagged with `managed_by_provider` and skips pairs already in
+ * `pairKeys` (e.g. provider → resource edges emitted as `file_transfer`).
+ *
+ * Skips resources declared under a `module.*` address so the provider node does
+ * not fan out to every nested resource (references / topology already link the
+ * graph).
+ */
+function appendTerraformBareProviderAttachmentFlows(input) {
+    const byAddr = new Map();
+    for (const c of input.components) {
+        const a = c.properties?.terraform_address;
+        if (typeof a === "string" && a.trim()) {
+            byAddr.set(a.trim(), c);
+        }
+    }
+    for (const c of input.components) {
+        if (c.type !== "asset")
+            continue;
+        if (typeof c.properties?.managed_by_provider === "string")
+            continue;
+        const rt = c.properties?.resource_type;
+        if (typeof rt !== "string" || !rt.includes("_"))
+            continue;
+        const addrRaw = c.properties?.terraform_address;
+        if (typeof addrRaw === "string" &&
+            addrRaw.startsWith("module.") &&
+            addrRaw.split(".").length > 2) {
+            continue;
+        }
+        const localName = terraformProviderLocalNameFromResourceType(rt);
+        if (!localName)
+            continue;
+        const prov = byAddr.get(`provider.${localName}`);
+        if (!prov || prov.type !== "third_party")
+            continue;
+        if (prov.id === c.id)
+            continue;
+        const pairKey = `${prov.id}::${c.id}`;
+        if (input.pairKeys.has(pairKey))
+            continue;
+        input.pairKeys.add(pairKey);
+        const loc = c.sourceLocations?.[0] ?? {
+            filePath: "terraform",
+            startLine: 1,
+            endLine: 1,
+        };
+        input.flows.push({
+            id: `flow_fallback_${input.flows.length + 1}`,
+            sourceComponentId: prov.id,
+            targetComponentId: c.id,
+            type: "api_call",
+            confidence: 0.72,
+            sourceLocation: loc,
+            description: `${prov.name} terraform provider attachment (${String(c.properties?.terraform_address ?? "")})`,
+        });
+    }
+}

package/dist/src/evals/metrics.d.ts

@@ -0,0 +1,18 @@
+import type { DetectedComponent } from "../core/types/component";
+import type { DetectedDataFlow } from "../core/types/data-flow";
+export interface EvalScores {
+    nodePropertyFillRate: number;
+    tieredPropertyCompleteness: number;
+    thirdPartyCompleteness: number;
+    directionAccuracy: number;
+    interactionRecall: number;
+    precisionGuardrail: number;
+}
+export declare function computeEvalScores(input: {
+    components: DetectedComponent[];
+    dataFlows: DetectedDataFlow[];
+    expectedCrossSectionLinks?: Array<{
+        sourceSection: string;
+        targetSection: string;
+    }>;
+}): EvalScores;

package/dist/src/evals/metrics.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeEvalScores = computeEvalScores;
+function isFilled(value) {
+    if (value == null)
+        return false;
+    if (Array.isArray(value))
+        return value.length > 0;
+    return value !== false && value !== "";
+}
+function computeEvalScores(input) {
+    const { components, dataFlows, expectedCrossSectionLinks = [] } = input;
+    const componentValues = components.flatMap((component) => Object.values(component.properties));
+    const filled = componentValues.filter(isFilled).length;
+    const nodePropertyFillRate = componentValues.length === 0 ? 0 : filled / componentValues.length;
+    const tier1Fields = ["encryption_at_rest", "authentication_methods", "gdpr_role"];
+    const weighted = components.map((component) => {
+        const values = tier1Fields.map((field) => component.properties[field]);
+        const localFilled = values.filter(isFilled).length;
+        return values.length === 0 ? 0 : localFilled / values.length;
+    });
+    const tieredPropertyCompleteness = weighted.length === 0
+        ? 0
+        : weighted.reduce((sum, value) => sum + value, 0) / weighted.length;
+    const thirdParties = components.filter((component) => component.type === "third_party");
+    const tpComplete = thirdParties.filter((component) => Boolean(component.subType) &&
+        isFilled(component.properties.vendor ?? component.properties.client)).length;
+    const thirdPartyCompleteness = thirdParties.length === 0 ? 1 : tpComplete / thirdParties.length;
+    const actorFlows = dataFlows.filter((flow) => {
+        const source = components.find((component) => component.id === flow.sourceComponentId);
+        const target = components.find((component) => component.id === flow.targetComponentId);
+        return source?.type === "actor" && target?.type === "asset";
+    });
+    const directionAccuracy = actorFlows.length === 0 ? 1 : actorFlows.length / Math.max(1, actorFlows.length);
+    let matchedCross = 0;
+    for (const expected of expectedCrossSectionLinks) {
+        const hit = dataFlows.some((flow) => {
+            const source = components.find((component) => component.id === flow.sourceComponentId);
+            const target = components.find((component) => component.id === flow.targetComponentId);
+            return (String(source?.properties.section_id ?? "") === expected.sourceSection &&
+                String(target?.properties.section_id ?? "") === expected.targetSection);
+        });
+        if (hit)
+            matchedCross += 1;
+    }
+    const interactionRecall = expectedCrossSectionLinks.length === 0
+        ? 1
+        : matchedCross / expectedCrossSectionLinks.length;
+    const precisionGuardrail = Math.max(0, 1 - dataFlows.filter((flow) => flow.confidence < 0.5).length / Math.max(1, dataFlows.length));
+    return {
+        nodePropertyFillRate,
+        tieredPropertyCompleteness,
+        thirdPartyCompleteness,
+        directionAccuracy,
+        interactionRecall,
+        precisionGuardrail,
+    };
+}

package/dist/src/evals/run-model-matrix.d.ts

@@ -0,0 +1 @@
+import "../config/load-cli-env";

package/dist/src/evals/run-model-matrix.js

@@ -0,0 +1,68 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+require("../config/load-cli-env");
+const orchestrator_1 = require("../core/pipeline/orchestrator");
+const metrics_1 = require("./metrics");
+function parseArg(flag) {
+    const idx = process.argv.indexOf(flag);
+    if (idx < 0)
+        return undefined;
+    return process.argv[idx + 1];
+}
+async function run() {
+    const fixturePath = parseArg("--fixture");
+    const modelsArg = parseArg("--models") ?? "mock:heuristic";
+    if (!fixturePath) {
+        throw new Error("Missing --fixture <path>");
+    }
+    const models = modelsArg.split(",").map((entry) => entry.trim()).filter(Boolean);
+    const report = [];
+    for (const modelEntry of models) {
+        const [provider, model] = modelEntry.includes(":")
+            ? modelEntry.split(":")
+            : ["mock", modelEntry];
+        const config = (0, orchestrator_1.createDefaultScanConfiguration)({
+            enableAiInference: provider !== "none",
+            aiProvider: provider,
+            aiModel: model,
+        });
+        const result = await (0, orchestrator_1.scan)(node_path_1.default.resolve(process.cwd(), fixturePath), config);
+        report.push({
+            model,
+            provider,
+            scores: (0, metrics_1.computeEvalScores)({
+                components: result.scanResult.components,
+                dataFlows: result.scanResult.dataFlows,
+            }),
+            warnings: result.scanResult.warnings,
+            errors: result.scanResult.errors,
+        });
+    }
+    const outputDir = node_path_1.default.resolve(process.cwd(), "outputs");
+    node_fs_1.default.mkdirSync(outputDir, { recursive: true });
+    const jsonPath = node_path_1.default.join(outputDir, "model-eval-report.json");
+    const markdownPath = node_path_1.default.join(outputDir, "model-eval-report.md");
+    node_fs_1.default.writeFileSync(jsonPath, JSON.stringify(report, null, 2), "utf8");
+    const markdown = [
+        "# Model Eval Report",
+        "",
+        ...report.map((entry) => [
+            `## ${entry.provider}:${entry.model}`,
+            `- nodePropertyFillRate: ${entry.scores.nodePropertyFillRate.toFixed(3)}`,
+            `- tieredPropertyCompleteness: ${entry.scores.tieredPropertyCompleteness.toFixed(3)}`,
+            `- thirdPartyCompleteness: ${entry.scores.thirdPartyCompleteness.toFixed(3)}`,
+            `- directionAccuracy: ${entry.scores.directionAccuracy.toFixed(3)}`,
+            `- interactionRecall: ${entry.scores.interactionRecall.toFixed(3)}`,
+            `- precisionGuardrail: ${entry.scores.precisionGuardrail.toFixed(3)}`,
+        ].join("\n")),
+    ].join("\n");
+    node_fs_1.default.writeFileSync(markdownPath, markdown, "utf8");
+    // eslint-disable-next-line no-console
+    console.log(`Model eval reports written:\n- ${jsonPath}\n- ${markdownPath}`);
+}
+void run();

package/dist/src/index.d.ts

@@ -0,0 +1,2 @@
+export { run } from "./cli";
+export * from "./ai-enrichment";

package/dist/src/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = void 0;
+var cli_1 = require("./cli");
+Object.defineProperty(exports, "run", { enumerable: true, get: function () { return cli_1.run; } });
+__exportStar(require("./ai-enrichment"), exports);

package/dist/src/ingest/file-system.d.ts

@@ -0,0 +1,17 @@
+import type { FileInfo } from "../core/types/file";
+export interface IngestOptions {
+    maxFileSizeBytes?: number;
+    maxFileCount?: number;
+    maxTotalBytes?: number;
+    excludePaths?: string[];
+    onWarning?: (warning: string) => void;
+}
+/**
+ * Resolve a user-supplied scan path to a directory (for config, Terraform cwd,
+ * section discovery) and an ingest target (directory walk or a single file).
+ */
+export declare function resolveScanFilesystemEntry(rootPath: string): Promise<{
+    scanRootDir: string;
+    ingestTarget: string;
+}>;
+export declare function ingestFileSystem(rootPath: string, options?: IngestOptions): Promise<FileInfo[]>;

package/dist/src/ingest/file-system.js

@@ -0,0 +1,239 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveScanFilesystemEntry = resolveScanFilesystemEntry;
+exports.ingestFileSystem = ingestFileSystem;
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const scan_exclusions_1 = require("../patterns/scan-exclusions");
+const gitignore_1 = require("./gitignore");
+const sensitive_paths_1 = require("./sensitive-paths");
+const DEFAULT_MAX_FILE_SIZE_BYTES = 2 * 1024 * 1024; // 2MB
+const DEFAULT_MAX_FILE_COUNT = 20_000;
+const DEFAULT_MAX_TOTAL_BYTES = 200 * 1024 * 1024; // 200MB
+function normalizeUserPattern(pattern) {
+    return (0, gitignore_1.toPosixPath)(pattern.replace(/^\.\/+/, "").trim());
+}
+function isExcludedByUserPatterns(relativePath, isDirectory, excludePaths) {
+    const rel = (0, gitignore_1.toPosixPath)(relativePath);
+    for (const rawPattern of excludePaths) {
+        const pattern = normalizeUserPattern(rawPattern);
+        if (!pattern)
+            continue;
+        // Treat "dir/**" as excluding the directory itself and all descendants.
+        if (pattern.endsWith("/**")) {
+            const prefix = pattern.slice(0, -3).replace(/\/$/, "");
+            if (rel === prefix || rel.startsWith(`${prefix}/`))
+                return true;
+        }
+        const regex = (0, gitignore_1.gitignorePatternToRegex)(pattern, isDirectory);
+        if (regex.test(rel))
+            return true;
+    }
+    return false;
+}
+function getFileLanguage(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const base = path.basename(filePath).toLowerCase();
+    if (ext === ".ts" || ext === ".tsx") {
+        return "typescript";
+    }
+    if (ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs") {
+        return "javascript";
+    }
+    if (ext === ".py") {
+        return "python";
+    }
+    if (ext === ".json") {
+        return "json";
+    }
+    if (ext === ".yaml" || ext === ".yml") {
+        return "yaml";
+    }
+    if (ext === ".tf" || ext === ".tfvars") {
+        return "terraform";
+    }
+    // Handle .env, .env.local, etc. (basename starts with .env)
+    if (base === ".env" || base.startsWith(".env.")) {
+        return "env";
+    }
+    return undefined;
+}
+/**
+ * Resolve a user-supplied scan path to a directory (for config, Terraform cwd,
+ * section discovery) and an ingest target (directory walk or a single file).
+ */
+async function resolveScanFilesystemEntry(rootPath) {
+    const abs = path.resolve(rootPath);
+    const st = await fs_1.promises.stat(abs);
+    if (st.isDirectory()) {
+        return { scanRootDir: abs, ingestTarget: abs };
+    }
+    if (st.isFile()) {
+        return { scanRootDir: path.dirname(abs), ingestTarget: abs };
+    }
+    throw new Error(`Scan path is not a file or directory: ${abs}`);
+}
+async function ingestSingleFile(fileAbs, options) {
+    const scanRootDir = path.dirname(fileAbs);
+    const relativePath = (0, gitignore_1.toPosixPath)(path.relative(scanRootDir, fileAbs));
+    const basename = path.basename(fileAbs);
+    const excludePaths = options.excludePaths ?? [];
+    if ((0, sensitive_paths_1.isSensitiveEnvPath)(relativePath) || (0, sensitive_paths_1.isSensitiveEnvPath)(basename)) {
+        options.onWarning?.(`Skipped "${basename}" (excluded for security).`);
+        return [];
+    }
+    if (relativePath &&
+        isExcludedByUserPatterns(relativePath, false, excludePaths)) {
+        options.onWarning?.(`Skipped "${relativePath}" (matched exclude pattern).`);
+        return [];
+    }
+    const language = getFileLanguage(fileAbs);
+    if (!language) {
+        options.onWarning?.(`Skipping unsupported file type: ${basename}`);
+        return [];
+    }
+    const stat = await fs_1.promises.stat(fileAbs);
+    if (stat.size > options.maxFileSizeBytes) {
+        options.onWarning?.(`Skipped "${path.basename(fileAbs)}" (${stat.size} bytes) because it exceeds max file size (${options.maxFileSizeBytes} bytes).`);
+        return [];
+    }
+    if (stat.size > options.maxTotalBytes) {
+        options.onWarning?.(`Skipped "${path.basename(fileAbs)}" because ingest byte budget (${options.maxTotalBytes} bytes) would be exceeded.`);
+        return [];
+    }
+    const content = await fs_1.promises.readFile(fileAbs, "utf8");
+    return [
+        {
+            path: relativePath,
+            name: basename,
+            content,
+            language,
+            size: stat.size,
+        },
+    ];
+}
+async function walkDirectory(currentDir, rootDir, accumulatedRules, files, options, state) {
+    if (state.fileCountLimitReached)
+        return;
+    const excludePaths = options.excludePaths ?? [];
+    const rules = await (0, gitignore_1.gitignoreRulesForDir)(currentDir, accumulatedRules);
+    const entries = await fs_1.promises.readdir(currentDir, { withFileTypes: true });
+    for (const entry of entries) {
+        if (state.fileCountLimitReached)
+            return;
+        const entryPath = path.join(currentDir, entry.name);
+        const relativePath = (0, gitignore_1.toPosixPath)(path.relative(rootDir, entryPath));
+        if (entry.isDirectory()) {
+            if ((0, scan_exclusions_1.shouldSkipDirectoryName)(entry.name)) {
+                continue;
+            }
+            if (relativePath &&
+                isExcludedByUserPatterns(relativePath, true, excludePaths)) {
+                continue;
+            }
+            if ((0, gitignore_1.isPathIgnored)(entryPath, true, rules)) {
+                continue;
+            }
+            await walkDirectory(entryPath, rootDir, rules, files, options, state);
+        }
+        else if (entry.isFile()) {
+            if (relativePath &&
+                isExcludedByUserPatterns(relativePath, false, excludePaths)) {
+                continue;
+            }
+            if ((0, gitignore_1.isPathIgnored)(entryPath, false, rules)) {
+                continue;
+            }
+            const language = getFileLanguage(entryPath);
+            if (!language)
+                continue;
+            const stat = await fs_1.promises.stat(entryPath);
+            if (stat.size > options.maxFileSizeBytes) {
+                options.onWarning?.(`Skipped "${relativePath}" (${stat.size} bytes) because it exceeds max file size (${options.maxFileSizeBytes} bytes).`);
+                continue;
+            }
+            if (state.totalBytesIncluded + stat.size >
+                options.maxTotalBytes) {
+                options.onWarning?.(`Skipped "${relativePath}" because ingest byte budget (${options.maxTotalBytes} bytes) would be exceeded.`);
+                continue;
+            }
+            if (state.filesIncluded >= options.maxFileCount) {
+                state.fileCountLimitReached = true;
+                options.onWarning?.(`Stopped ingest after ${state.filesIncluded} files because max file count (${options.maxFileCount}) was reached.`);
+                return;
+            }
+            const content = await fs_1.promises.readFile(entryPath, "utf8");
+            files.push({
+                path: relativePath,
+                name: entry.name,
+                content,
+                language,
+                size: stat.size,
+            });
+            state.filesIncluded += 1;
+            state.totalBytesIncluded += stat.size;
+        }
+    }
+}
+async function ingestFileSystem(rootPath, options = {}) {
+    const abs = path.resolve(rootPath);
+    const st = await fs_1.promises.stat(abs);
+    const resolvedOptions = {
+        maxFileSizeBytes: options.maxFileSizeBytes ?? DEFAULT_MAX_FILE_SIZE_BYTES,
+        maxFileCount: options.maxFileCount ?? DEFAULT_MAX_FILE_COUNT,
+        maxTotalBytes: options.maxTotalBytes ?? DEFAULT_MAX_TOTAL_BYTES,
+        excludePaths: [
+            ...scan_exclusions_1.DEFAULT_EXCLUDED_FILE_GLOBS,
+            ...(options.excludePaths ?? []),
+        ],
+        onWarning: options.onWarning,
+    };
+    if (st.isFile()) {
+        return ingestSingleFile(abs, resolvedOptions);
+    }
+    if (!st.isDirectory()) {
+        throw new Error(`Scan path is not a file or directory: ${abs}`);
+    }
+    const rootDir = abs;
+    const files = [];
+    const state = {
+        filesIncluded: 0,
+        totalBytesIncluded: 0,
+        fileCountLimitReached: false,
+    };
+    await walkDirectory(rootDir, rootDir, [], files, resolvedOptions, state);
+    return files;
+}

package/dist/src/ingest/gitignore.d.ts

@@ -0,0 +1,12 @@
+export type IgnoreRule = {
+    baseDir: string;
+    pattern: string;
+    negate: boolean;
+    dirOnly: boolean;
+    regex: RegExp;
+};
+export declare function toPosixPath(p: string): string;
+export declare function gitignorePatternToRegex(pattern: string, dirOnly: boolean): RegExp;
+export declare function readGitignore(dir: string): Promise<IgnoreRule[]>;
+export declare function gitignoreRulesForDir(currentDir: string, accumulatedRules: IgnoreRule[]): Promise<IgnoreRule[]>;
+export declare function isPathIgnored(absolutePath: string, isDirectory: boolean, rules: IgnoreRule[]): boolean;