@feiyoug/skill-lab 0.0.0 → 0.0.2

package/src/analyzer/treesitter/client.ts

@@ -0,0 +1,120 @@
+import { Language, Parser, Query, Tree } from "web-tree-sitter";
+import { ensureGrammar } from "./registry.js";
+import type { TreesitterGrammar } from "./registry.js";
+import type { AnalyzerLogger } from "../types.js";
+import { NO_OP_LOGGER } from "../logging.js";
+
+export class TreesitterClient {
+    private PARSER_BY_GRAMMAR: Partial<Record<TreesitterGrammar, Parser>> = {};
+    private LANG_BY_GRAMMAR: Partial<Record<TreesitterGrammar, Language>> = {};
+    private QUERY_CACHE = new Map<string, Query>();
+    private ROOT_NODE_CACHE_BY_CONTENT: Partial<
+        // cache by length then hash
+        Record<TreesitterGrammar, Map<number, Map<number, Tree>>>
+    > = {};
+    /** Parser.init() is idempotent but we avoid re-calling it. */
+    private parserInitialized: boolean = false;
+
+    constructor(
+        private readonly logger: AnalyzerLogger = NO_OP_LOGGER,
+        private readonly showProgressBar: boolean = false,
+    ) {}
+
+    private async ensureParserInit() {
+        if (this.parserInitialized) return;
+
+        await Parser.init();
+        this.parserInitialized = true;
+    }
+
+    public async getParser(grammar: TreesitterGrammar): Promise<Parser> {
+        if (this.PARSER_BY_GRAMMAR[grammar]) {
+            return this.PARSER_BY_GRAMMAR[grammar]!;
+        }
+
+        await this.ensureParserInit();
+
+        let lang = this.LANG_BY_GRAMMAR[grammar];
+        if (!lang) {
+            const wasmPath = await ensureGrammar(grammar, {
+                logger: this.logger,
+                showProgressBar: this.showProgressBar,
+            });
+            lang = await Language.load(wasmPath);
+            this.LANG_BY_GRAMMAR[grammar] = lang;
+        }
+
+        const parser = new Parser();
+        parser.setLanguage(lang);
+
+        this.PARSER_BY_GRAMMAR[grammar] = parser;
+        return parser;
+    }
+
+    /**
+     * Creates (and caches) a tree-sitter Query for the given grammar and S-expression query string.
+     */
+    public async createQuery(grammar: TreesitterGrammar, queryString: string): Promise<Query> {
+        const cacheKey = `${grammar}:${queryString}`;
+        if (this.QUERY_CACHE.has(cacheKey)) {
+            return this.QUERY_CACHE.get(cacheKey)!;
+        }
+
+        await this.getParser(grammar); // ensures LANG_BY_GRAMMAR[grammar] is populated
+        const lang = this.LANG_BY_GRAMMAR[grammar]!;
+        const query = new Query(lang, queryString);
+        this.QUERY_CACHE.set(cacheKey, query);
+        return query;
+    }
+
+    /**
+     * Parses content into a tree-sitter Tree with memoization.
+     *
+     * Cache key strategy:
+     * - first level: content length
+     * - second level: fast non-cryptographic hash of content
+     */
+    public async parse(grammar: TreesitterGrammar, content: string): Promise<Tree> {
+        const treeCacheByLen = this.getTreeCache(grammar);
+        const len = content.length;
+        const treeCacheByHash = treeCacheByLen.get(len);
+        if (treeCacheByHash) {
+            const hash = this.hashContent(content);
+            const cached = treeCacheByHash.get(hash);
+            if (cached) return cached;
+        }
+
+        const parser = await this.getParser(grammar);
+        const tree = parser.parse(content);
+        if (!tree) {
+            throw new Error(`Failed to parse ${grammar} content`);
+        }
+
+        const hash = this.hashContent(content);
+        const bucket = treeCacheByLen.get(len) ?? new Map<number, Tree>();
+        bucket.set(hash, tree);
+        treeCacheByLen.set(len, bucket);
+
+        return tree;
+    }
+
+    private getTreeCache(grammar: TreesitterGrammar): Map<number, Map<number, Tree>> {
+        if (!this.ROOT_NODE_CACHE_BY_CONTENT[grammar]) {
+            this.ROOT_NODE_CACHE_BY_CONTENT[grammar] = new Map<
+                number,
+                Map<number, Tree>
+            >();
+        }
+        return this.ROOT_NODE_CACHE_BY_CONTENT[grammar]!;
+    }
+
+    private hashContent(content: string): number {
+        // FNV-1a 32-bit (fast, non-cryptographic)
+        let hash = 0x811c9dc5;
+        for (let i = 0; i < content.length; i++) {
+            hash ^= content.charCodeAt(i);
+            hash = Math.imul(hash, 0x01000193);
+        }
+        return hash >>> 0;
+    }
+}

package/src/analyzer/treesitter/registry.ts

@@ -0,0 +1,198 @@
+/**
+ * Shared grammar registry for web-tree-sitter WASM grammars.
+ *
+ * Used by both AstGrepClient and TreesitterClient. Grammar .wasm files are
+ * downloaded from GitHub Releases on first use and cached locally following
+ * the XDG cache directory convention.
+ *
+ * Cache location (in priority order):
+ *   1. $SKILL_LAB_CACHE_DIR
+ *   2. $XDG_CACHE_HOME/skill-lab
+ *   3. ~/.cache/skill-lab  (Linux/macOS)
+ *      %LOCALAPPDATA%\skill-lab\Cache  (Windows)
+ */
+import * as dntShim from "../../_dnt.shims.js";
+
+
+import { join } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import type { AnalyzerLogger } from "../types.js";
+import ProgressBar from "../../deps/jsr.io/@deno-library/progress/1.5.1/mod.js";
+
+const TREESITTER_GRAMMER_SUBDIR = "treesitter/grammars";
+const ANSI_CLEAR_LINE = "\r\x1b[K";
+const ENCODER = new TextEncoder();
+
+export type GrammarSpec = {
+    /** Local filename used in the cache directory. */
+    filename: string;
+    /** Exact, version-pinned URL to download the grammar .wasm from. */
+    url: string;
+};
+
+/**
+ * Grammar specifications. All URLs are exact version-pinned to ensure
+ * ABI compatibility with web-tree-sitter@0.25.4.
+ *
+ * Grammar .wasm files are compiled by the tree-sitter CLI against a specific
+ * WASM ABI. Mixing grammar versions with a different web-tree-sitter version
+ * causes silent parse failures, so both must be pinned together.
+ */
+export const GRAMMAR_SPECS = {
+    bash: {
+        filename: "bash.wasm",
+        url: "https://github.com/tree-sitter/tree-sitter-bash/releases/download/v0.25.1/tree-sitter-bash.wasm",
+    },
+    javascript: {
+        filename: "javascript.wasm",
+        url: "https://github.com/tree-sitter/tree-sitter-javascript/releases/download/v0.25.0/tree-sitter-javascript.wasm",
+    },
+    python: {
+        filename: "python.wasm",
+        url: "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.25.0/tree-sitter-python.wasm",
+    },
+    typescript: {
+        filename: "typescript.wasm",
+        url: "https://github.com/tree-sitter/tree-sitter-typescript/releases/download/v0.23.2/tree-sitter-typescript.wasm",
+    },
+    tsx: {
+        filename: "tsx.wasm",
+        url: "https://github.com/tree-sitter/tree-sitter-typescript/releases/download/v0.23.2/tree-sitter-tsx.wasm",
+    },
+    markdown: {
+        filename: "markdown.wasm",
+        url: "https://github.com/tree-sitter-grammars/tree-sitter-markdown/releases/download/v0.5.2/tree-sitter-markdown.wasm",
+    },
+    "markdown-inline": {
+        filename: "markdown_inline.wasm",
+        url: "https://github.com/tree-sitter-grammars/tree-sitter-markdown/releases/download/v0.5.2/tree-sitter-markdown_inline.wasm",
+    },
+} satisfies Record<string, GrammarSpec>;
+
+/** Available languages for tree-sitter grammars. */
+export type TreesitterGrammar = keyof typeof GRAMMAR_SPECS;
+
+/**
+ * Returns the skill-lab cache directory, following XDG conventions.
+ *
+ * Priority:
+ *   1. $SKILL_LAB_CACHE_DIR — explicit override
+ *   2. $XDG_CACHE_HOME/skill-lab — if XDG_CACHE_HOME is set
+ *   3. Platform default:
+ *        Linux/macOS: ~/.cache/skill-lab
+ *        Windows:     %LOCALAPPDATA%\skill-lab\Cache
+ */
+export function getCacheDir(): string {
+    const explicit = dntShim.Deno.env.get("SKILL_LAB_CACHE_DIR");
+    if (explicit) return explicit;
+
+    const xdgCache = dntShim.Deno.env.get("XDG_CACHE_HOME");
+    if (xdgCache) return join(xdgCache, "skill-lab");
+
+    if (dntShim.Deno.build.os === "windows") {
+        const localAppData = dntShim.Deno.env.get("LOCALAPPDATA");
+        if (localAppData) return join(localAppData, "skill-lab", "Cache");
+        const userProfile = dntShim.Deno.env.get("USERPROFILE") ?? "~";
+        return join(userProfile, "AppData", "Local", "skill-lab", "Cache");
+    }
+
+    const home = dntShim.Deno.env.get("HOME") ?? "~";
+    return join(home, ".cache", "skill-lab");
+}
+
+/**
+ * Ensures a grammar .wasm file is present in the local cache.
+ *
+ * On first call for a given language, downloads the file from the pinned URL
+ * in GRAMMAR_SPECS. Subsequent calls return immediately from cache.
+ *
+ * @param lang - Language key (e.g. "bash", "typescript", "markdown-inline")
+ * @param opts.logger - Optional logger for download events
+ * @returns Absolute path to the cached .wasm file
+ */
+export async function ensureGrammar(
+    lang: string,
+    opts?: {
+        logger?: AnalyzerLogger;
+        showProgressBar?: boolean;
+    },
+): Promise<string> {
+    if (!(lang in GRAMMAR_SPECS)) {
+        throw new Error(
+            `No grammar spec for language: "${lang}". Available: ${
+                Object.keys(GRAMMAR_SPECS).join(", ")
+            }`,
+        );
+    }
+    const spec = GRAMMAR_SPECS[lang as TreesitterGrammar];
+
+    const cacheDir = join(getCacheDir(), TREESITTER_GRAMMER_SUBDIR);
+    const cachedPath = join(cacheDir, spec.filename);
+
+    // Return immediately if already cached
+    try {
+        const stat = await dntShim.Deno.stat(cachedPath);
+        if (stat.isFile) return cachedPath;
+    } catch {
+        // Not cached yet — fall through to download
+    }
+
+    // Download and write to cache
+    await dntShim.Deno.mkdir(cacheDir, { recursive: true });
+    if (dntShim.Deno.stderr.isTerminal()) {
+        dntShim.Deno.stderr.writeSync(ENCODER.encode(ANSI_CLEAR_LINE));
+    }
+
+    const resp = await fetch(spec.url);
+    if (!resp.ok) {
+        throw new Error(
+            `Failed to download grammar ${spec.filename}: HTTP ${resp.status} from ${spec.url}`,
+        );
+    }
+
+    const shouldRenderProgress = (opts?.showProgressBar ?? false) && dntShim.Deno.stdout.isTerminal();
+    const contentLenHeader = resp.headers.get("content-length")!;
+    const contentLen = parseInt(contentLenHeader);
+    const scanBar = shouldRenderProgress
+        ? new ProgressBar({
+            total: contentLen,
+            clear: true,
+            output: dntShim.Deno.stderr,
+            complete: "=",
+            incomplete: "-",
+            display: `Installing ${lang} grammar [:bar] :percent ETA :eta`,
+        })
+        : null;
+
+    let bytes: Uint8Array;
+    try {
+        if (!resp.body) {
+            bytes = new Uint8Array(await resp.arrayBuffer());
+        } else {
+            const reader = resp.body.getReader();
+            const chunks: Uint8Array[] = [];
+            let received = 0;
+
+            while (true) {
+                const { value, done } = await reader.read();
+                if (done) break;
+                if (!value) continue;
+
+                chunks.push(value);
+                received += value.byteLength;
+                scanBar?.render(received);
+            }
+
+            bytes = new Uint8Array(received);
+            let offset = 0;
+            for (const chunk of chunks) {
+                bytes.set(chunk, offset);
+                offset += chunk.byteLength;
+            }
+        }
+    } finally {
+        scanBar?.end();
+    }
+    await dntShim.Deno.writeFile(cachedPath, bytes);
+
+    return cachedPath;
+}

package/src/analyzer/types.ts

@@ -0,0 +1,78 @@
+import type { SkillFile, SkillReader } from "../skillreader/mod.js";
+import type {
+    FileRefDiscoveryMethod,
+    FileReference,
+    FileType,
+    Finding,
+    Frontmatter,
+    Permission,
+    Reference,
+    ReferenceType,
+    Risk,
+} from "../shared/mod.js";
+import { AstGrepClient } from "./astgrep/client.js";
+import { TreesitterClient } from "./treesitter/client.js";
+import type { AnalyzerConfig, ScanConfig } from "./config/mod.js";
+export type { AnalyzerConfig, ScanConfig };
+
+export type AnalyzerLogger = {
+    debug: (template: string, props?: Record<string, unknown>) => void;
+    info: (template: string, props?: Record<string, unknown>) => void;
+    warn: (template: string, props?: Record<string, unknown>) => void;
+    error: (template: string, props?: Record<string, unknown>) => void;
+};
+
+export type AnalyzerState = {
+    skillId: string;
+    skillVersionId: string;
+    files: SkillFile[];
+    frontmatter: Frontmatter;
+    scanQueue: FileReference[];
+    permissions: Permission[];
+    findings: Finding[];
+    risks: Risk[];
+    warnings: string[];
+    metadata: {
+        scannedFiles: Set<string>;
+        skippedFiles: Array<{ path: string; reason: string; referenceBy?: Reference }>;
+        rulesUsed: string[];
+        config: ScanConfig;
+    };
+};
+
+export type AnalyzerContext = {
+    skillReader: SkillReader;
+    treesitterClient: TreesitterClient;
+    astgrepClient: AstGrepClient;
+    logger?: AnalyzerLogger;
+    showProgressBar?: boolean;
+    /** Resolved (defaults-merged) user config. */
+    config: AnalyzerConfig;
+};
+
+/** A file path or package reference discovered in source content. */
+export type FileRefDiscovery = {
+    path: string;
+    line: number;
+    via: FileRefDiscoveryMethod;
+};
+
+export type CodeBlock = {
+    language: FileType;
+    content: string;
+    startLine: number;
+    endLine: number;
+    type: ReferenceType;
+};
+
+export type FileTypeConfig = {
+    extractCodeBlocks?: (
+        context: AnalyzerContext,
+        content: string,
+    ) => Promise<CodeBlock[]>;
+    extractFileRefs?: (
+        context: AnalyzerContext,
+        content: string,
+    ) => FileRefDiscovery[] | Promise<FileRefDiscovery[]>;
+    defaultLanguage: FileType | null;
+};

package/src/analyzer/utils/code-block-path.ts

@@ -0,0 +1,33 @@
+export type DecodedCodeBlockPath = {
+    parentPath: string;
+    startLine: number;
+    endLine: number;
+};
+
+export function encodeCodeBlockPath(
+    parentPath: string,
+    startLine: number,
+    endLine: number,
+): string {
+    return `${parentPath}:${startLine}-${endLine}`;
+}
+
+export function decodeCodeBlockPath(path: string): DecodedCodeBlockPath | null {
+    const match = path.match(/^(.+):(\d+)-(\d+)$/);
+    if (!match) return null;
+
+    const startLine = Number(match[2]);
+    const endLine = Number(match[3]);
+    if (!Number.isFinite(startLine) || !Number.isFinite(endLine)) return null;
+    if (startLine <= 0 || endLine < startLine) return null;
+
+    return {
+        parentPath: match[1],
+        startLine,
+        endLine,
+    };
+}
+
+export function isCodeBlockPath(path: string): boolean {
+    return decodeCodeBlockPath(path) !== null;
+}

package/src/analyzer/utils/id-generator.ts

@@ -0,0 +1,59 @@
+export function generatePermissionId(
+    tool: string,
+    args: string[] = ["*"],
+    metadata?: Record<string, unknown>,
+): string {
+    const normalizedArgs = args.length > 0 ? args : ["*"];
+    const base = sanitize(`${tool}-${normalizedArgs.join("-")}`);
+    const normalizedMetadata = filterMetadataForHash(metadata ?? {});
+
+    const shouldHash = normalizedArgs.length > 1 || Object.keys(normalizedMetadata).length > 0;
+    if (!shouldHash) return base;
+
+    const hashInput = stableStringify({ args: normalizedArgs, metadata: normalizedMetadata });
+    return `${base}-${shortHash(hashInput)}`;
+}
+
+export function generateRiskId(type: string, index: number): string {
+    return sanitize(`risk-${type}-${index + 1}`);
+}
+
+function sanitize(value: string): string {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9.*-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "");
+}
+
+function filterMetadataForHash(metadata: Record<string, unknown>): Record<string, unknown> {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(metadata)) {
+        if (key === "pattern") continue;
+        if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+
+function stableStringify(value: unknown): string {
+    if (Array.isArray(value)) {
+        return `[${value.map((item) => stableStringify(item)).join(",")}]`;
+    }
+    if (value && typeof value === "object") {
+        const entries = Object.entries(value as Record<string, unknown>)
+            .sort(([a], [b]) => a.localeCompare(b))
+            .map(([key, item]) => `${key}:${stableStringify(item)}`);
+        return `{${entries.join(",")}}`;
+    }
+    return JSON.stringify(value);
+}
+
+function shortHash(input: string): string {
+    let hash = 0;
+    for (let i = 0; i < input.length; i += 1) {
+        hash = (hash * 31 + input.charCodeAt(i)) | 0;
+    }
+    return Math.abs(hash).toString(36).padStart(6, "0").slice(0, 6);
+}

package/src/analyzer/utils/secret-validator.ts

@@ -0,0 +1,29 @@
+const SECRET_KEYWORDS = [
+    "token",
+    "key",
+    "secret",
+    "password",
+    "pwd",
+    "pass",
+    "credential",
+    "cred",
+    "auth",
+    "api",
+    "apikey",
+    "bearer",
+    "oauth",
+    "jwt",
+    "session",
+    "private",
+];
+
+export function isSecretLikeName(value: unknown): boolean {
+    if (typeof value !== "string") return false;
+    const normalized = value.toLowerCase();
+    return SECRET_KEYWORDS.some((keyword) => normalized.includes(keyword));
+}
+
+export function normalizeKeyCandidate(value: unknown): string {
+    if (typeof value !== "string") return "";
+    return value.replace(/[;,)]+$/, "").trim();
+}

package/src/analyzer/utils/url-parser.ts

@@ -0,0 +1,25 @@
+export function parseUrlFromUnknown(
+    input: string,
+): { host: string; protocol: string; raw: string } | null {
+    if (!input) return null;
+    const cleaned = input.trim().replace(/^['"`]|['"`]$/g, "");
+    const withProtocol = /^https?:\/\//i.test(cleaned) ? cleaned : `https://${cleaned}`;
+
+    try {
+        const url = new URL(withProtocol);
+        return {
+            host: url.hostname,
+            protocol: url.protocol.replace(":", ""),
+            raw: cleaned,
+        };
+    } catch {
+        return null;
+    }
+}
+
+export function classifyDestination(host: string): "loopback" | "internal" | "external" {
+    if (["localhost", "127.0.0.1", "::1"].includes(host)) return "loopback";
+    if (host.endsWith(".local")) return "internal";
+    if (/^10\.|^192\.168\.|^172\.(1[6-9]|2[0-9]|3[01])\./.test(host)) return "internal";
+    return "external";
+}

package/src/deps/jsr.io/@deno-library/progress/1.5.1/deps.ts

@@ -0,0 +1,3 @@
+export { bgGreen, bgWhite, stripAnsiCode } from "../../../@std/fmt/1.0.3/colors.js";
+export { writeAll } from "../../../@std/io/0.225.0/write_all.js";
+// export type { Writer } from "jsr:@std/io@0.225.0/types";