@feiyoug/skill-lab 0.0.0 → 0.0.2

package/src/analyzer/steps/001-discovery/discover-files.ts

@@ -0,0 +1,211 @@
+import type { SkillFile } from "../../../skillreader/mod.js";
+import { getFileRole, getFileType } from "../../../shared/mod.js";
+
+import type { AnalyzerContext } from "../../types.js";
+import { FILETYPE_CONFIGS } from "../../rules/mod.js";
+import { isHostFsPath, isUrl } from "../../rules/shared/file-refs.js";
+import type { FileRefDiscovery } from "../../types.js";
+import type { FileReference, Reference } from "../../../shared/mod.js";
+import { encodeCodeBlockPath } from "../../utils/code-block-path.js";
+
+type StartQueueItem = { path: string; depth: number; referencedBy?: Reference };
+
+/**
+ * Discovers referenced files recursively and excludes starting queue paths.
+ *
+ * Step 001 is code-block aware:
+ * - Extracts code blocks from each file
+ * - Runs per-language extractFileRefs on each block
+ * - Enqueues discovered local markdown/text files for recursive discovery
+ * - Emits virtual code-block FileReferences for step 002 scanning
+ */
+export async function discoverReferencedFiles(
+    context: AnalyzerContext,
+    input: {
+        startQueue: StartQueueItem[];
+        allFiles: SkillFile[];
+        readTextFile: (path: string) => Promise<string | null>;
+        maxScanDepth: number;
+        onDiscover?: (progress: { scannedCount: number; discoveredCount: number }) => void;
+    },
+): Promise<FileReference[]> {
+    const discovered = new Map<string, FileReference>();
+    const queue: StartQueueItem[] = [...input.startQueue];
+    let scannedCount = 0;
+    let discoveredCount = 0;
+
+    // A set of file paths which have been processed
+    const processed = new Set<string>();
+    const startPaths = new Set(input.startQueue.map((item) => normalizePath(item.path)));
+
+    while (queue.length > 0) {
+        const current = queue.shift();
+        if (!current) break;
+
+        if (processed.has(current.path) || current.depth > input.maxScanDepth) continue;
+        processed.add(current.path);
+        input.onDiscover?.({ scannedCount: scannedCount++, discoveredCount });
+
+        const content = await input.readTextFile(current.path);
+        if (!content) continue;
+
+        const currentFileType = getFileType(current.path);
+
+        const blocks = await FILETYPE_CONFIGS[currentFileType]?.extractCodeBlocks?.(
+            context,
+            content,
+        ) ?? [];
+
+        blocks.push({
+            language: currentFileType,
+            content,
+            startLine: 0,
+            endLine: Math.max(0, content.split("\n").length),
+            type: "content",
+        });
+
+        for (const block of blocks) {
+            const blockRefs =
+                await FILETYPE_CONFIGS[block.language]?.extractFileRefs?.(context, block.content) ??
+                    [];
+
+            for (const blockRef of blockRefs) {
+                const absoluteRef: FileRefDiscovery = {
+                    ...blockRef,
+                    line: blockRef.line + block.startLine - 1,
+                };
+
+                const normalizedPath = normalizePath(absoluteRef.path);
+                if (
+                    !normalizedPath || startPaths.has(normalizedPath) ||
+                    discovered.has(normalizedPath)
+                ) {
+                    continue;
+                }
+
+                const referenceFromCurrent: Reference = {
+                    file: current.path,
+                    line: absoluteRef.line,
+                    type: block.type,
+                    referencedBy: current.referencedBy,
+                };
+
+                const existsInSkill = input.allFiles.filter((file) =>
+                    file.path.includes(normalizedPath)
+                );
+                if (existsInSkill.length > 0) {
+                    for (const file of existsInSkill) {
+                        if (discovered.has(file.path)) continue;
+
+                        const resolvedFileType = getFileType(file.path);
+                        const localEntry: FileReference = {
+                            path: file.path,
+                            sourceType: "local",
+                            fileType: resolvedFileType,
+                            role: getFileRole(file.path),
+                            depth: current.depth + 1,
+                            discoveryMethod: absoluteRef.via,
+                            referencedBy: referenceFromCurrent,
+                        };
+                        discovered.set(file.path, localEntry);
+
+                        // Keep scaning for files that are referenced within the skill repo
+                        queue.push({
+                            path: file.path,
+                            depth: current.depth + 1,
+                            referencedBy: localEntry.referencedBy,
+                        });
+                        input.onDiscover?.({ scannedCount, discoveredCount: discoveredCount++ });
+                    }
+                    continue;
+                }
+
+                if (isHostFsPath(normalizedPath)) {
+                    discovered.set(normalizedPath, {
+                        path: normalizedPath,
+                        sourceType: "external",
+                        fileType: "unknown",
+                        role: "host-fs",
+                        depth: current.depth + 1,
+                        discoveryMethod: absoluteRef.via,
+                        referencedBy: referenceFromCurrent,
+                    });
+                    continue;
+                }
+
+                if (isUrl(normalizedPath) || absoluteRef.via === "markdown-link") {
+                    discovered.set(normalizedPath, {
+                        path: normalizedPath,
+                        sourceType: "external",
+                        fileType: "unknown",
+                        role: "regular",
+                        depth: current.depth + 1,
+                        discoveryMethod: absoluteRef.via,
+                        referencedBy: referenceFromCurrent,
+                    });
+                    continue;
+                }
+
+                if (absoluteRef.via === "import") {
+                    discovered.set(normalizedPath, {
+                        path: normalizedPath,
+                        sourceType: "external",
+                        fileType: block.language,
+                        role: "library",
+                        depth: current.depth + 1,
+                        discoveryMethod: absoluteRef.via,
+                        referencedBy: referenceFromCurrent,
+                    });
+                    continue;
+                }
+
+                if (absoluteRef.via === "source") {
+                    discovered.set(normalizedPath, {
+                        path: normalizedPath,
+                        sourceType: "external",
+                        fileType: block.language,
+                        role: "library",
+                        depth: current.depth + 1,
+                        discoveryMethod: absoluteRef.via,
+                        referencedBy: referenceFromCurrent,
+                    });
+                    continue;
+                }
+            }
+
+            let codeBlockPath = current.path;
+            if (block.type != "content") {
+                codeBlockPath = encodeCodeBlockPath(
+                    current.path,
+                    block.startLine,
+                    block.endLine,
+                );
+            }
+            if (!discovered.has(codeBlockPath)) {
+                discovered.set(codeBlockPath, {
+                    path: codeBlockPath,
+                    sourceType: "local",
+                    fileType: block.language,
+                    role: "script",
+                    depth: current.depth + 1,
+                    discoveryMethod: "code-block",
+                    referencedBy: {
+                        file: current.path,
+                        line: block.startLine,
+                        lineEnd: block.endLine,
+                        type: block.type,
+                        referencedBy: current.referencedBy,
+                    },
+                });
+            }
+        }
+    }
+
+    return Array.from(discovered.values());
+}
+
+function normalizePath(value: string): string {
+    return value.replace(/^\/+/, "").replace(/\\/g, "/").trim();
+}
+
+export type { FileRefDiscovery };

package/src/analyzer/steps/001-discovery/filter-files.ts

@@ -0,0 +1,72 @@
+import type { SkillFile } from "../../../skillreader/mod.js";
+import type { FileReference, FileRole } from "../../../shared/mod.js";
+
+export function filterScanQueue(input: {
+    queue: FileReference[];
+    allFiles: SkillFile[];
+    maxFileSize: number;
+    maxFileCount: number;
+}): { queue: FileReference[]; skipped: Array<{ path: string; reason: string }> } {
+    const result: FileReference[] = [];
+    const skipped: Array<{ path: string; reason: string }> = [];
+
+    const ordered = [...input.queue].sort((a, b) => {
+        const roleRank = (value: FileRole): number => {
+            switch (value) {
+                case "entrypoint":
+                    return 0;
+                case "readme":
+                    return 1;
+                case "reference":
+                    return 2;
+                case "license":
+                    return 3;
+                case "script":
+                    return 4;
+                case "library":
+                    return 5;
+                case "config":
+                    return 6;
+                default:
+                    return 7;
+            }
+        };
+
+        const fileTypeRank = (value: FileReference["fileType"]): number => {
+            if (value === "markdown" || value === "text") return 0;
+            if (
+                value === "bash" || value === "javascript" || value === "typescript" ||
+                value === "python"
+            ) return 1;
+            if (value === "json" || value === "yaml") return 2;
+            return 3;
+        };
+
+        const roleDiff = roleRank(a.role) - roleRank(b.role);
+        if (roleDiff !== 0) return roleDiff;
+        return fileTypeRank(a.fileType) - fileTypeRank(b.fileType);
+    });
+
+    for (const fileRef of ordered) {
+        const file = input.allFiles.find((item) => item.path === fileRef.path);
+
+        if (file?.contentType === "binary") {
+            skipped.push({ path: fileRef.path, reason: "binary" });
+            continue;
+        }
+
+        if ((file?.size ?? 0) > input.maxFileSize) {
+            skipped.push({ path: fileRef.path, reason: "too_large" });
+            continue;
+        }
+
+        if (result.length >= input.maxFileCount) {
+            skipped.push({ path: fileRef.path, reason: "too_many" });
+            continue;
+        }
+
+        result.push(fileRef);
+    }
+
+    return { queue: result, skipped };
+}

package/src/analyzer/steps/001-discovery/mod.ts

@@ -0,0 +1,103 @@
+import * as dntShim from "../../../_dnt.shims.js";
+import ProgressBar from "../../../deps/jsr.io/@deno-library/progress/1.5.1/mod.js";
+import {
+    DEFAULT_ANALYZER_CONFIG,
+    DEFAULT_SKILL_VERSION,
+    FRONTMATTER_SUPPORTED_FIELDS,
+} from "../../config/mod.js";
+import type { AnalyzerContext, AnalyzerState } from "../../types.js";
+import { discoverReferencedFiles } from "./discover-files.js";
+import { filterScanQueue } from "./filter-files.js";
+
+export async function run001Discovery(
+    state: AnalyzerState,
+    context: AnalyzerContext,
+): Promise<AnalyzerState> {
+    const files = await context.skillReader.listFiles();
+    const skillMdPath = await context.skillReader.getSkillMdPath();
+    const frontmatter = await context.skillReader.getSkillMdFrontmatter();
+
+    const nextState: AnalyzerState = {
+        ...state,
+        files,
+        frontmatter,
+        skillId: String(frontmatter.name ?? state.skillId),
+        skillVersionId: String(
+            frontmatter.metadata?.version ?? state.skillVersionId ?? DEFAULT_SKILL_VERSION,
+        ),
+    };
+
+    for (const field of Object.keys(frontmatter)) {
+        if (
+            !FRONTMATTER_SUPPORTED_FIELDS.includes(
+                field as (typeof FRONTMATTER_SUPPORTED_FIELDS)[number],
+            ) && ![
+                "startLineNumber",
+                "endLineNumber",
+            ].includes(field)
+        ) {
+            nextState.warnings.push(
+                `Unknown frontmatter field '${field}' - analysis not supported yet`,
+            );
+        }
+    }
+
+    const shouldLogProgress = (context.showProgressBar ?? false) && dntShim.Deno.stderr.isTerminal();
+    const maxScanDepth = state.metadata.config.maxScanDepth ??
+        DEFAULT_ANALYZER_CONFIG.scan?.maxScanDepth ?? 5;
+    const maxFileCount = state.metadata.config.maxFileCount ??
+        DEFAULT_ANALYZER_CONFIG.scan?.maxFileCount ?? 100;
+    const maxFileSize = state.metadata.config.maxFileSize ??
+        DEFAULT_ANALYZER_CONFIG.scan?.maxFileSize ?? 1_000_000;
+    const discoveryBar = shouldLogProgress
+        ? new ProgressBar({
+            total: files.length,
+            clear: true,
+            output: dntShim.Deno.stderr,
+            complete: "=",
+            incomplete: "-",
+            display: "Readings skills [:bar] :percent ETA :eta",
+        })
+        : null;
+
+    let discovered: AnalyzerState["scanQueue"] = [];
+    try {
+        discovered = await discoverReferencedFiles(context, {
+            startQueue: [{ path: skillMdPath, depth: 0 }],
+            allFiles: files,
+            readTextFile: (path) => context.skillReader.readTextFile(path),
+            maxScanDepth,
+            onDiscover: (progress) => {
+                discoveryBar?.render(progress.scannedCount, {
+                    total: progress.discoveredCount + 1,
+                });
+            },
+        });
+    } finally {
+        discoveryBar?.end();
+    }
+
+    discovered.push({
+        path: skillMdPath,
+        sourceType: "local",
+        fileType: "markdown",
+        role: "entrypoint",
+        depth: 0,
+    });
+
+    const filtered = filterScanQueue({
+        queue: discovered,
+        allFiles: files,
+        maxFileCount,
+        maxFileSize,
+    });
+
+    return {
+        ...nextState,
+        scanQueue: filtered.queue,
+        metadata: {
+            ...nextState.metadata,
+            skippedFiles: [...nextState.metadata.skippedFiles, ...filtered.skipped],
+        },
+    };
+}