fln 1.0.0

package/dist/core/scanTree.js

@@ -0,0 +1,348 @@
+import { lstat, open, readdir, readlink, realpath, stat } from "node:fs/promises";
+import { cpus } from "node:os";
+import { relative, sep } from "node:path";
+import ignore from "ignore";
+import { IgnoreMatcher } from "./ignoreMatcher";
+function normalizePathSegment(pathSegment) {
+    if (sep === "/")
+        return pathSegment;
+    return pathSegment.split(sep).join("/");
+}
+function getFileScore(fileName) {
+    const lowerName = fileName.toLowerCase();
+    if (lowerName.startsWith("readme"))
+        return 0;
+    if (lowerName === "package.json" ||
+        lowerName.startsWith("tsconfig") ||
+        lowerName === "pyproject.toml" ||
+        lowerName === "cargo.toml" ||
+        lowerName === "go.mod" ||
+        lowerName === "cmakelists.txt" ||
+        lowerName === "makefile" ||
+        lowerName === "dockerfile" ||
+        lowerName === "vcpkg.json" ||
+        lowerName.startsWith(".env") ||
+        lowerName.includes(".config.") ||
+        lowerName.startsWith(".prettier") ||
+        lowerName.startsWith(".eslintrc"))
+        return 1;
+    if (lowerName.startsWith("index.") ||
+        lowerName.startsWith("main.") ||
+        lowerName.startsWith("app.") ||
+        lowerName.startsWith("server.") ||
+        lowerName.startsWith("mod.") ||
+        lowerName.startsWith("lib."))
+        return 2;
+    if (lowerName.includes("types") ||
+        lowerName.includes("interface") ||
+        lowerName.includes("schema") ||
+        lowerName.includes("config") ||
+        lowerName.includes("constants") ||
+        lowerName.endsWith(".d.ts") ||
+        lowerName.endsWith(".h") ||
+        lowerName.endsWith(".hpp"))
+        return 3;
+    if (lowerName.startsWith("license") ||
+        lowerName.startsWith("changelog") ||
+        lowerName.startsWith("contributing") ||
+        lowerName.startsWith("code_of_conduct") ||
+        lowerName.startsWith("security"))
+        return 15;
+    if (lowerName.includes(".test.") ||
+        lowerName.includes(".spec.") ||
+        lowerName.startsWith("test_") ||
+        lowerName.endsWith("_test.go"))
+        return 20;
+    return 10;
+}
+async function isBinaryFile(filePath, fileSize) {
+    if (fileSize === 0)
+        return false;
+    const handle = await open(filePath, "r");
+    try {
+        const buffer = Buffer.alloc(Math.min(512, fileSize));
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+        for (let index = 0; index < bytesRead; index++)
+            if (buffer[index] === 0)
+                return true;
+        return false;
+    }
+    finally {
+        await handle.close();
+    }
+}
+async function isGeneratedFile(filePath, fileSize) {
+    if (fileSize === 0)
+        return false;
+    const handle = await open(filePath, "r");
+    try {
+        const buffer = Buffer.alloc(Math.min(100, fileSize));
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+        const content = buffer.toString("utf8", 0, bytesRead);
+        return content.includes("<!-- 🥞 fln");
+    }
+    catch {
+        return false;
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function scanTree(options, logger) {
+    const stats = {
+        files: 0,
+        directories: 0,
+        binary: 0,
+        skipped: 0,
+        errors: 0,
+        totalSizeBytes: 0,
+        outputSizeBytes: 0,
+        outputTokenCount: 0
+    };
+    const ignoreMatcher = new IgnoreMatcher({
+        rootDirectory: options.rootDirectory,
+        excludePatterns: options.excludePatterns,
+        useGitignore: options.useGitignore,
+        logger
+    });
+    const includeMatcher = ignore().add(options.includePatterns);
+    const concurrencyLimit = Math.max(8, Math.min(64, cpus().length * 4));
+    const excludedPathSet = new Set(options.excludedPaths.map(pathItem => normalizePathSegment(pathItem)));
+    const visitedRealPaths = new Set();
+    let processedItems = 0;
+    let totalEstimate = 0;
+    if (options.followSymlinks)
+        try {
+            const rootRealPath = await realpath(options.rootDirectory);
+            visitedRealPaths.add(rootRealPath);
+        }
+        catch {
+            logger.debug("Failed to resolve root real path.");
+        }
+    const rootNode = await scanEntry(options.rootDirectory, "");
+    if (!rootNode || rootNode.type !== "directory")
+        throw new Error("Root directory is empty or all files were excluded.");
+    return { projectName: options.projectName, root: rootNode, stats };
+    async function scanEntry(currentPath, relativePath, dirent) {
+        const normalizedRelativePath = normalizePathSegment(relativePath);
+        const name = dirent?.name ?? currentPath.split(sep).pop() ?? "";
+        if (normalizedRelativePath !== "" && excludedPathSet.has(normalizedRelativePath))
+            return undefined;
+        const isExplicitlyIncluded = normalizedRelativePath !== "" && includeMatcher.ignores(normalizedRelativePath);
+        if (!isExplicitlyIncluded && normalizedRelativePath !== "" && ignoreMatcher.ignores(normalizedRelativePath))
+            return undefined;
+        if (!options.includeHidden && name.startsWith(".") && name !== ".")
+            return undefined;
+        try {
+            let symlinkTarget;
+            const isSymlink = dirent ? dirent.isSymbolicLink() : undefined;
+            if (isSymlink) {
+                symlinkTarget = await readlink(currentPath);
+                if (!options.followSymlinks)
+                    return {
+                        name,
+                        path: normalizedRelativePath,
+                        type: "symlink",
+                        size: 0,
+                        target: symlinkTarget
+                    };
+                const resolvedPath = await realpath(currentPath);
+                if (visitedRealPaths.has(resolvedPath)) {
+                    stats.skipped++;
+                    return {
+                        name,
+                        path: normalizedRelativePath,
+                        type: "symlink",
+                        size: 0,
+                        target: symlinkTarget,
+                        skipReason: "symlinkCycle"
+                    };
+                }
+                visitedRealPaths.add(resolvedPath);
+                const statsResult = await stat(currentPath);
+                if (statsResult.isFile())
+                    return await buildFileNode({
+                        currentPath,
+                        normalizedRelativePath,
+                        name,
+                        fileSize: statsResult.size,
+                        symlinkTarget,
+                        isExplicitlyIncluded
+                    });
+                if (statsResult.isDirectory())
+                    return await buildDirectoryNode({
+                        currentPath,
+                        normalizedRelativePath,
+                        name,
+                        symlinkTarget
+                    });
+                return undefined;
+            }
+            if (dirent?.isDirectory())
+                return await buildDirectoryNode({
+                    currentPath,
+                    normalizedRelativePath,
+                    name
+                });
+            if (dirent?.isFile()) {
+                const statsResult = await stat(currentPath);
+                return await buildFileNode({
+                    currentPath,
+                    normalizedRelativePath,
+                    name,
+                    fileSize: statsResult.size,
+                    isExplicitlyIncluded
+                });
+            }
+            if (dirent)
+                return undefined;
+            const entryStats = await lstat(currentPath);
+            if (entryStats.isSymbolicLink()) {
+                symlinkTarget = await readlink(currentPath);
+                if (!options.followSymlinks)
+                    return {
+                        name,
+                        path: normalizedRelativePath,
+                        type: "symlink",
+                        size: 0,
+                        target: symlinkTarget
+                    };
+                const resolvedPath = await realpath(currentPath);
+                if (visitedRealPaths.has(resolvedPath)) {
+                    stats.skipped++;
+                    return {
+                        name,
+                        path: normalizedRelativePath,
+                        type: "symlink",
+                        size: 0,
+                        target: symlinkTarget,
+                        skipReason: "symlinkCycle"
+                    };
+                }
+                visitedRealPaths.add(resolvedPath);
+            }
+            const statsResult = options.followSymlinks ? await stat(currentPath) : entryStats;
+            if (statsResult.isFile())
+                return await buildFileNode({
+                    currentPath,
+                    normalizedRelativePath,
+                    name,
+                    fileSize: statsResult.size,
+                    symlinkTarget,
+                    isExplicitlyIncluded
+                });
+            if (statsResult.isDirectory())
+                return await buildDirectoryNode({
+                    currentPath,
+                    normalizedRelativePath,
+                    name,
+                    symlinkTarget
+                });
+        }
+        catch (error) {
+            stats.errors++;
+            logger.warn(`Failed to access ${normalizedRelativePath || "."}: ${String(error)}`);
+        }
+        return undefined;
+    }
+    async function buildFileNode(input) {
+        stats.files++;
+        stats.totalSizeBytes += input.fileSize;
+        processedItems++;
+        if (totalEstimate === 0)
+            totalEstimate = Math.max(processedItems + 50, 100);
+        if (options.onProgress)
+            options.onProgress(processedItems, Math.max(totalEstimate, processedItems));
+        let skipReason;
+        if (!input.isExplicitlyIncluded && await isGeneratedFile(input.currentPath, input.fileSize))
+            skipReason = "generated";
+        if (!skipReason && input.fileSize > options.maximumFileSizeBytes)
+            skipReason = "tooLarge";
+        let isBinary = false;
+        if (!skipReason)
+            try {
+                isBinary = await isBinaryFile(input.currentPath, input.fileSize);
+            }
+            catch (error) {
+                stats.errors++;
+                skipReason = "readError";
+                logger.warn(`Failed to read ${input.normalizedRelativePath || "."}: ${String(error)}`);
+            }
+        if (isBinary)
+            stats.binary++;
+        if (skipReason) {
+            stats.skipped++;
+            return {
+                name: input.name,
+                path: input.normalizedRelativePath,
+                type: "file",
+                size: input.fileSize,
+                isBinary,
+                target: input.symlinkTarget,
+                skipReason
+            };
+        }
+        return {
+            name: input.name,
+            path: input.normalizedRelativePath,
+            type: "file",
+            size: input.fileSize,
+            isBinary,
+            target: input.symlinkTarget
+        };
+    }
+    async function buildDirectoryNode(input) {
+        stats.directories++;
+        await ignoreMatcher.addGitignoreForDirectory(input.currentPath);
+        const entries = await readdir(input.currentPath, { withFileTypes: true });
+        totalEstimate = Math.max(totalEstimate, processedItems + entries.length);
+        const children = (await mapWithConcurrency(entries, concurrencyLimit, async (entry) => {
+            const childPath = `${input.currentPath}${sep}${entry.name}`;
+            const childRelativePath = relative(options.rootDirectory, childPath);
+            return scanEntry(childPath, childRelativePath, entry);
+        }))
+            .filter((node) => node !== undefined)
+            .sort((left, right) => {
+            if (left.type !== right.type)
+                return left.type === "directory" ? 1 : -1;
+            if (left.type === "file" && right.type === "file") {
+                const scoreA = getFileScore(left.name);
+                const scoreB = getFileScore(right.name);
+                if (scoreA !== scoreB)
+                    return scoreA - scoreB;
+            }
+            return left.name.localeCompare(right.name, undefined, { numeric: true, sensitivity: "base" });
+        });
+        return {
+            name: input.name,
+            path: input.normalizedRelativePath,
+            type: "directory",
+            size: 0,
+            children,
+            target: input.symlinkTarget
+        };
+    }
+}
+async function mapWithConcurrency(items, concurrency, mapper) {
+    if (items.length === 0)
+        return [];
+    const limit = Math.max(1, Math.min(concurrency, items.length));
+    if (limit === 1) {
+        const results = [];
+        for (const item of items)
+            results.push(await mapper(item));
+        return results;
+    }
+    const results = new Array(items.length);
+    let nextIndex = 0;
+    const workers = Array.from({ length: limit }, async () => {
+        while (true) {
+            const currentIndex = nextIndex++;
+            if (currentIndex >= items.length)
+                return;
+            results[currentIndex] = await mapper(items[currentIndex]);
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}

package/dist/core/size.d.ts

@@ -0,0 +1,4 @@
+export declare function parseByteSize(input: string): number;
+export declare function formatByteSize(sizeBytes: number): string;
+export declare function formatTokenCount(count: number): string;
+//# sourceMappingURL=size.d.ts.map

package/dist/core/size.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"size.d.ts","sourceRoot":"","sources":["../../src/core/size.ts"],"names":[],"mappings":"AAIA,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAgBnD;AAED,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CASxD;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAOtD"}

package/dist/core/size.js

@@ -0,0 +1,32 @@
+const kibibyte = 1024;
+const mebibyte = kibibyte * 1024;
+const gibibyte = mebibyte * 1024;
+export function parseByteSize(input) {
+    const normalizedInput = input.trim().toLowerCase();
+    const match = normalizedInput.match(/^(\d+(?:\.\d+)?)\s*(b|kb|mb|gb)?$/);
+    if (!match)
+        throw new Error(`Invalid size: "${input}"`);
+    const value = Number(match[1]);
+    const unit = match[2] ?? "b";
+    const multiplier = unit === "kb" ? kibibyte :
+        unit === "mb" ? mebibyte :
+            unit === "gb" ? gibibyte :
+                1;
+    return Math.floor(value * multiplier);
+}
+export function formatByteSize(sizeBytes) {
+    if (sizeBytes >= gibibyte)
+        return `${(sizeBytes / gibibyte).toFixed(2)} GB`;
+    if (sizeBytes >= mebibyte)
+        return `${(sizeBytes / mebibyte).toFixed(2)} MB`;
+    if (sizeBytes >= kibibyte)
+        return `${(sizeBytes / kibibyte).toFixed(2)} KB`;
+    return `${sizeBytes} B`;
+}
+export function formatTokenCount(count) {
+    if (count >= 1_000_000)
+        return `≈ ${(count / 1_000_000).toFixed(1)}M`;
+    if (count >= 1000)
+        return `≈ ${(count / 1000).toFixed(1)}K`;
+    return `≈ ${count}`;
+}

package/dist/core/statsCollector.d.ts

@@ -0,0 +1,4 @@
+import type { FileNode } from "./types";
+export declare function collectExtensionStats(root: FileNode): Map<string, number>;
+export declare function collectProcessedFiles(root: FileNode): string[];
+//# sourceMappingURL=statsCollector.d.ts.map

package/dist/core/statsCollector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statsCollector.d.ts","sourceRoot":"","sources":["../../src/core/statsCollector.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAGxC,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAkBzE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,EAAE,CAkB9D"}

package/dist/core/statsCollector.js

@@ -0,0 +1,28 @@
+import { extname } from "node:path";
+export function collectExtensionStats(root) {
+    const stats = new Map();
+    function walk(node) {
+        if (node.type === "file" && !node.isBinary && !node.skipReason) {
+            const ext = extname(node.name) || "(no ext)";
+            stats.set(ext, (stats.get(ext) || 0) + 1);
+        }
+        if (node.children)
+            for (const child of node.children)
+                walk(child);
+    }
+    walk(root);
+    return stats;
+}
+export function collectProcessedFiles(root) {
+    const files = [];
+    function walk(node, parentPath = "") {
+        const currentPath = parentPath ? `${parentPath}/${node.name}` : node.name;
+        if (node.type === "file" && !node.isBinary && !node.skipReason)
+            files.push(currentPath);
+        if (node.children)
+            for (const child of node.children)
+                walk(child, currentPath);
+    }
+    walk(root);
+    return files;
+}

package/dist/core/types.d.ts

@@ -0,0 +1,53 @@
+export type OutputFormat = "json" | "md";
+export type SkipReason = "generated" | "readError" | "symlinkCycle" | "tooLarge" | "totalSizeLimit";
+export type FileType = "directory" | "file" | "symlink";
+export type LogLevel = "debug" | "normal" | "silent" | "verbose";
+export type FileNode = {
+    name: string;
+    path: string;
+    type: FileType;
+    size: number;
+    children?: FileNode[];
+    target?: string;
+    isBinary?: boolean;
+    skipReason?: SkipReason;
+};
+export type ScanStats = {
+    files: number;
+    directories: number;
+    binary: number;
+    skipped: number;
+    errors: number;
+    totalSizeBytes: number;
+    outputSizeBytes: number;
+    outputTokenCount: number;
+};
+export type ScanResult = {
+    projectName: string;
+    root: FileNode;
+    stats: ScanStats;
+};
+export type ProgressCallback = (current: number, total: number) => void;
+export type ScanOptions = {
+    projectName: string;
+    rootDirectory: string;
+    excludePatterns: string[];
+    includePatterns: string[];
+    excludedPaths: string[];
+    includeHidden: boolean;
+    useGitignore: boolean;
+    maximumFileSizeBytes: number;
+    maximumTotalSizeBytes: number;
+    followSymlinks: boolean;
+    onProgress?: ProgressCallback;
+};
+export type RenderOptions = {
+    outputFile: string;
+    format: OutputFormat;
+    includeTree: boolean;
+    includeContents: boolean;
+    useAnsi: boolean;
+    banner?: string;
+    footer?: string;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/core/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,IAAI,CAAC;AAEzC,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,cAAc,GAAG,UAAU,GAAG,gBAAgB,CAAC;AAEpG,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,SAAS,CAAC;AAExD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEjE,MAAM,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,SAAS,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;AAExE,MAAM,MAAM,WAAW,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC"}

package/dist/core/types.js

@@ -0,0 +1 @@
+export {};

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./api";
+export * from "./core";
+export * from "./version";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,QAAQ,CAAC;AACvB,cAAc,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1,3 @@
+export * from "./api";
+export * from "./core";
+export * from "./version";

package/dist/infra/countTokens.d.ts

@@ -0,0 +1,2 @@
+export declare function countTokens(text: string): number;
+//# sourceMappingURL=countTokens.d.ts.map

package/dist/infra/countTokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"countTokens.d.ts","sourceRoot":"","sources":["../../src/infra/countTokens.ts"],"names":[],"mappings":"AAqEA,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuKhD"}