@feiyoug/skill-lab 0.0.0 → 0.0.2

package/src/shared/deep_merge.ts

@@ -0,0 +1,73 @@
+export type JsonValue =
+    | null
+    | boolean
+    | number
+    | string
+    | JsonObject
+    | JsonArray;
+
+export type JsonObject = { [key: string]: JsonValue };
+export type JsonArray = JsonValue[];
+
+export type ArrayMerge = (base: JsonArray, override: JsonArray) => JsonArray;
+
+export type DeepMergeOptions = {
+    arrayMerge?: ArrayMerge;
+};
+
+const DEFAULT_OPTIONS: DeepMergeOptions = {
+    arrayMerge: undefined,
+};
+
+function isPlainObject(value: unknown): value is JsonObject {
+    if (!value || typeof value !== "object") return false;
+    const proto = Object.getPrototypeOf(value);
+    return proto === Object.prototype || proto === null;
+}
+
+function mergeInternal(base: JsonValue, override: JsonValue, options: DeepMergeOptions): JsonValue {
+    if (override === undefined) return base;
+
+    if (Array.isArray(base) && Array.isArray(override)) {
+        return options.arrayMerge ? options.arrayMerge(base, override) : override;
+    }
+
+    if (isPlainObject(base) && isPlainObject(override)) {
+        const result: JsonObject = {};
+        const keys = new Set([...Object.keys(base), ...Object.keys(override)]);
+
+        for (const key of keys) {
+            if (key in override) {
+                const merged = mergeInternal(base[key], override[key], options);
+                if (merged !== undefined) {
+                    result[key] = merged;
+                }
+            } else {
+                result[key] = base[key];
+            }
+        }
+
+        return result;
+    }
+
+    return override;
+}
+
+export function deepMergeJson<T>(base: T, override: unknown, options?: DeepMergeOptions): T {
+    const resolvedOptions = options ? { ...DEFAULT_OPTIONS, ...options } : DEFAULT_OPTIONS;
+    return mergeInternal(base as JsonValue, override as JsonValue, resolvedOptions) as T;
+}
+
+export function mergeArrayUnion(base: JsonArray, override: JsonArray): JsonArray {
+    const merged = [...base, ...override];
+    const seen = new Set<unknown>();
+    const result: JsonArray = [];
+
+    for (const item of merged) {
+        if (seen.has(item)) continue;
+        seen.add(item);
+        result.push(item);
+    }
+
+    return result;
+}

package/src/shared/mod.ts

@@ -0,0 +1,2 @@
+export * from "./types/mod.js";
+export * from "./deep_merge.js";

package/src/shared/types/filetypes.ts

@@ -0,0 +1,101 @@
+import { ReferenceType } from "./references.js";
+
+export type Frontmatter = {
+    startLineNumber?: number;
+    endLineNumber?: number;
+    name?: string;
+    description?: string;
+    [key: string]: unknown;
+};
+
+export type FileType =
+    | "markdown"
+    | "text"
+    | "bash"
+    | "javascript"
+    | "typescript"
+    | "python"
+    | "json"
+    | "yaml"
+    | "toml"
+    | "config"
+    | "sql"
+    | "csv"
+    | "xml"
+    | "binary"
+    | "unknown";
+
+export type CodeBlock = {
+    language: FileType;
+    content: string;
+    startLine: number;
+    endLine: number;
+    type: ReferenceType;
+};
+
+export const FILETYPE_BY_LANGUAGE: Record<string, FileType> = {
+    bash: "bash",
+    sh: "bash",
+    shell: "bash",
+    zsh: "bash",
+    js: "javascript",
+    javascript: "javascript",
+    mjs: "javascript",
+    cjs: "javascript",
+    ts: "typescript",
+    typescript: "typescript",
+    py: "python",
+    python: "python",
+    md: "markdown",
+    markdown: "markdown",
+    text: "text",
+
+    json: "json",
+    yaml: "yaml",
+    yml: "yaml",
+    toml: "toml",
+    sql: "sql",
+    csv: "csv",
+    xml: "xml",
+};
+
+export const FILE_TYPE_BY_EXTENSION: Record<string, FileType> = {
+    md: "markdown",
+    txt: "text",
+    sh: "bash",
+    bash: "bash",
+    js: "javascript",
+    mjs: "javascript",
+    cjs: "javascript",
+    ts: "typescript",
+    tsx: "typescript",
+    py: "python",
+    json: "json",
+    yaml: "yaml",
+    yml: "yaml",
+    toml: "toml",
+    ini: "config",
+    sql: "sql",
+    csv: "csv",
+    xml: "xml",
+};
+
+export const BINARY_EXTENSIONS = new Set(["zip", "tar", "gz", "png", "jpg", "jpeg", "gif", "pdf"]);
+
+// Classifies files by extension with a safe unknown fallback.
+export function getFileType(path: string): FileType {
+    const lower = path.toLowerCase();
+
+    const ext = lower.split(".").pop();
+    if (!ext) return "unknown";
+
+    if (FILE_TYPE_BY_EXTENSION[ext]) return FILE_TYPE_BY_EXTENSION[ext];
+    if (BINARY_EXTENSIONS.has(ext)) return "binary";
+
+    return "unknown";
+}
+
+export function isFile(path: string): boolean {
+    const ft = getFileType(path);
+    return ft !== "unknown" && ft != "binary";
+}

package/src/shared/types/findings.ts

@@ -0,0 +1,7 @@
+import { Reference } from "./references.js";
+
+export type Finding = {
+    ruleId: string;
+    reference: Reference;
+    extracted: Record<string, unknown>;
+};

package/src/shared/types/mod.ts

@@ -0,0 +1,6 @@
+export * from "./filetypes.js";
+export * from "./findings.js";
+export * from "./permissions.js";
+export * from "./references.js";
+export * from "./risks.js";
+export * from "./syntaxNode.js";

package/src/shared/types/permissions.ts

@@ -0,0 +1,17 @@
+import { FileRole, Reference } from "./references.js";
+
+export type PermissionScope = "fs" | "sys" | "net" | "env" | "hooks" | "data" | "dep";
+
+export type Permission = {
+    id: string;
+    tool: string;
+    scope: PermissionScope;
+    permission: string;
+    args?: string[];
+    fileRole?: FileRole;
+    metadata?: Record<string, unknown>;
+    references: Reference[];
+    source: "frontmatter" | "detected" | "inferred";
+    comment?: string;
+    risks: string[];
+};

package/src/shared/types/references.ts

@@ -0,0 +1,62 @@
+import { FileType } from "./filetypes.js";
+
+export type ReferenceType = "frontmatter" | "content" | "script" | "inline";
+
+export type Reference = {
+    file: string;
+    line: number;
+    lineEnd?: number;
+    type: ReferenceType;
+    referencedBy?: Reference;
+};
+
+export type SourceType = "local" | "external";
+
+export type FileRole =
+    | "entrypoint"
+    | "license"
+    | "readme"
+    | "reference"
+    | "config"
+    | "script"
+    | "library" // unresolved imported dependency (package/module not in skill files)
+    | "regular"
+    | "host-fs"; // path targeting the host filesystem (not a skill-local file)
+
+export type FileRefDiscoveryMethod =
+    | "markdown-link" // [text](path) explicit markdown link
+    | "inline-code" // `command path` inline code token
+    | "bare-path" // bare path-like string in prose
+    | "import" // import/require/from package statement
+    | "url" // URL referenced by a network command (curl, wget, fetch)
+    | "source" // shell source / . file.sh include
+    | "code-block"; // virtual code block reference within parent file
+
+export type FileReference = {
+    /** The relative path of file to the root of the skill */
+    path: string;
+    sourceType: SourceType;
+    fileType: FileType;
+    role: FileRole;
+    depth: number;
+    referencedBy?: Reference;
+    /** How this reference was discovered during file extraction. */
+    discoveryMethod?: FileRefDiscoveryMethod;
+};
+
+// Adds semantic meaning to paths for priority and reporting.
+export function getFileRole(path: string): FileRole {
+    const base = path.split("/").pop()?.toLowerCase() ?? "";
+    const lower = path.toLowerCase();
+    if (base === "skill.md") return "entrypoint";
+    if (base === "reference.md" || lower.includes("/references/")) return "reference";
+    if (base.startsWith("license")) return "license";
+    if (base.startsWith("readme")) return "readme";
+    if (lower.includes("/scripts/")) return "script";
+    if (/\.(env\.example|sample\.|template|example\.)/i.test(base)) return "config";
+    return "regular";
+}
+
+export function getSourceType(existsInSkill: boolean): SourceType {
+    return existsInSkill ? "local" : "external";
+}

package/src/shared/types/risks.ts

@@ -0,0 +1,72 @@
+import { Finding } from "./findings.js";
+import { Permission } from "./permissions.js";
+import { Reference } from "./references.js";
+
+export type Severity = "info" | "warning" | "critical";
+
+type NETWORK_RISK_CODES =
+    | "NETWORK:data_exfiltration"
+    | "NETWORK:external_network_access"
+    | "NETWORK:remote_code_execution"
+    | "NETWORK:credential_leak"
+    | "NETWORK:localhost_secret_exposure";
+
+type INJECTION_RISK_CODES = "INJECTION:command_injection";
+
+type PROMPT_RISK_CODES = "PROMPT:prompt_override";
+
+type DESTRUCTIVE_RISK_CODES =
+    | "DESTRUCTIVE:destructive_behavior"
+    | "DESTRUCTIVE:permission_weakening";
+
+type PRIVILEGE_RISK_CODES = "PRIVILEGE:privilege_escalation";
+
+type PERSISTENCE_RISK_CODES = "PERSISTENCE:persistence";
+
+type SECRETS_RISK_CODES = "SECRETS:secret_access";
+
+type DEPENDENCY_RISK_CODES = "DEPENDENCY:external_import";
+
+type REFERENCE_RISK_CODES = "REFERENCE:external_file";
+
+export type RiskCode =
+    | NETWORK_RISK_CODES
+    | INJECTION_RISK_CODES
+    | PROMPT_RISK_CODES
+    | DESTRUCTIVE_RISK_CODES
+    | PRIVILEGE_RISK_CODES
+    | PERSISTENCE_RISK_CODES
+    | SECRETS_RISK_CODES
+    | DEPENDENCY_RISK_CODES
+    | REFERENCE_RISK_CODES;
+
+export type RuleRiskResult = {
+    code: RiskCode;
+    severity: Severity;
+    message: string;
+    metadata?: Record<string, unknown>;
+};
+
+export type RuleRiskInput = {
+    permission?: Permission;
+    finding: Finding;
+};
+
+export type Risk = {
+    id: string;
+    type: RiskCode;
+    groupKey?: string;
+    severity: Severity;
+    message: string;
+    reference: Reference;
+    permissions: string[];
+    metadata?: Record<string, unknown>;
+};
+
+export type StaticRuleRiskMapping = RuleRiskResult;
+
+export type DynamicRuleRiskMapping = (
+    input: RuleRiskInput,
+) => RuleRiskResult | RuleRiskResult[] | null;
+
+export type RuleRiskMapping = StaticRuleRiskMapping | DynamicRuleRiskMapping;

package/src/shared/types/syntaxNode.ts

@@ -0,0 +1,7 @@
+export type SyntaxNodeLike = {
+    type: string;
+    text: string;
+    startPosition: { row: number; column: number };
+    endPosition: { row: number; column: number };
+    children: SyntaxNodeLike[];
+};

package/src/skillreader/cloudStorage/mod.ts

@@ -0,0 +1,170 @@
+import { isProbablyText } from "../utils/mod.js";
+import type { SkillFile, SkillManifest, SkillZipManifest, ZipManifestFile } from "../types.js";
+import { SkillReader } from "../types.js";
+import { isZipManifest, parseSkillManifest } from "../manifest.js";
+import { decompressDeflateStream, fetchRangeStream } from "../utils/http-range.js";
+import { getFileType } from "../../shared/mod.js";
+
+/**
+ * ZIP local file header signature (magic number: 0x04034b50).
+ */
+const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50;
+
+/**
+ * ZIP local file header fixed section length in bytes.
+ */
+const LOCAL_FILE_HEADER_LENGTH = 30;
+
+export type CloudStorageSkillReaderOptions = {
+    /**
+     * Base URL for skill storage (CloudFront/CDN).
+     * Example: https://d1234abcd.cloudfront.net/skills/my-skill/v1.0.0
+     */
+    baseUrl: string;
+};
+
+/**
+ * Reader for skill-lab backend cloud storage (public CDN).
+ */
+export class CloudStorageSkillReader extends SkillReader {
+    private baseUrl: string;
+    private cachedManifest: SkillManifest | null = null;
+    private dataOffsetCache = new Map<string, number>();
+
+    constructor(options: CloudStorageSkillReaderOptions) {
+        super();
+        this.baseUrl = options.baseUrl.replace(/\/+$/, "");
+    }
+
+    async retrieveFiles(dir?: string): Promise<SkillFile[]> {
+        const manifest = await this.readManifest();
+        if (!manifest) {
+            throw new Error("Manifest not found at storage URL");
+        }
+
+        const targetPrefix = dir ? normalizePrefix(dir) : "";
+
+        return manifest.files
+            .filter((file) => {
+                if (!targetPrefix) return true;
+                return file.path.startsWith(`${targetPrefix}/`);
+            })
+            .map((file) => ({
+                path: file.path,
+                size: file.size,
+                contentType: getFileType(file.path),
+            }));
+    }
+
+    async readTextFile(path: string): Promise<string | null> {
+        const contentType = getFileType(path);
+        if (contentType === "binary") return null;
+
+        const stream = await this.readFile(path);
+        if (!stream) return null;
+
+        const buffer = new Uint8Array(await new Response(stream).arrayBuffer());
+        if (!isProbablyText(buffer)) return null;
+
+        return new TextDecoder().decode(buffer);
+    }
+
+    async readFile(path: string): Promise<ReadableStream<Uint8Array> | null> {
+        const manifest = await this.readManifest();
+        if (!manifest) return null;
+
+        if (isZipManifest(manifest)) {
+            return await this.readFileFromZip(path, manifest);
+        }
+
+        const fileUrl = `${this.baseUrl}/${path.replace(/^\/+/, "")}`;
+        try {
+            const response = await fetch(fileUrl);
+            if (!response.ok) return null;
+            return response.body;
+        } catch {
+            return null;
+        }
+    }
+
+    async readManifest(): Promise<SkillManifest | null> {
+        if (this.cachedManifest) return this.cachedManifest;
+
+        const manifestUrl = `${this.baseUrl}/manifest.json`;
+        try {
+            const response = await fetch(manifestUrl);
+            if (!response.ok) return null;
+
+            const text = await response.text();
+            const manifest = parseSkillManifest(text);
+            this.cachedManifest = manifest;
+            return manifest;
+        } catch {
+            return null;
+        }
+    }
+
+    private async readFileFromZip(
+        path: string,
+        manifest: SkillZipManifest,
+    ): Promise<ReadableStream<Uint8Array> | null> {
+        const normalizedPath = path.replace(/^\/+/, "");
+        const fileEntry = manifest.files.find((file) => file.path === normalizedPath);
+        if (!fileEntry) return null;
+
+        const zipUrl = `${this.baseUrl}/skill.zip`;
+        const dataOffset = await this.getDataOffset(zipUrl, fileEntry);
+        const rangeStream = await fetchRangeStream(zipUrl, dataOffset, fileEntry.compressedSize);
+
+        if (fileEntry.compressionMethod === 0) {
+            return rangeStream;
+        }
+        if (fileEntry.compressionMethod === 8) {
+            return decompressDeflateStream(rangeStream);
+        }
+
+        throw new Error(`Unsupported compression: ${fileEntry.compressionMethod}`);
+    }
+
+    private async getDataOffset(
+        zipUrl: string,
+        fileEntry: Pick<ZipManifestFile, "path" | "offset">,
+    ): Promise<number> {
+        const cached = this.dataOffsetCache.get(fileEntry.path);
+        if (cached !== undefined) return cached;
+
+        const headerStream = await fetchRangeStream(
+            zipUrl,
+            fileEntry.offset,
+            LOCAL_FILE_HEADER_LENGTH,
+        );
+        const headerBuffer = new Uint8Array(await new Response(headerStream).arrayBuffer());
+
+        if (headerBuffer.byteLength < LOCAL_FILE_HEADER_LENGTH) {
+            throw new Error("Failed to read ZIP header");
+        }
+
+        const view = new DataView(
+            headerBuffer.buffer,
+            headerBuffer.byteOffset,
+            headerBuffer.byteLength,
+        );
+
+        const signature = view.getUint32(0, true);
+        if (signature !== LOCAL_FILE_HEADER_SIGNATURE) {
+            throw new Error(`Invalid ZIP signature at offset ${fileEntry.offset}`);
+        }
+
+        const fileNameLength = view.getUint16(26, true);
+        const extraFieldLength = view.getUint16(28, true);
+        const dataOffset = fileEntry.offset + LOCAL_FILE_HEADER_LENGTH +
+            fileNameLength + extraFieldLength;
+
+        this.dataOffsetCache.set(fileEntry.path, dataOffset);
+        return dataOffset;
+    }
+}
+
+function normalizePrefix(prefix: string): string {
+    return prefix.replace(/^\/+|\/+$/g, "");
+}

package/src/skillreader/factory.ts

@@ -0,0 +1,71 @@
+import * as dntShim from "../_dnt.shims.js";
+import { join } from "../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { LocalFsSkillReader } from "./fs/mod.js";
+import { ensureGitRepoRoot, LocalGitSkillReader } from "./fs/git.js";
+import { GitHubSkillReader } from "./github/mod.js";
+import { parseGitHubRepo } from "./github/utils.js";
+import type { SkillReader } from "./types.js";
+
+export type SkillReaderFactoryOptions = {
+    source: string;
+    subDir?: string;
+    gitRef?: string;
+    githubToken?: string;
+};
+
+export class SkillReaderFactory {
+    static async create(options: SkillReaderFactoryOptions): Promise<SkillReader> {
+        const source = options.source.trim();
+        if (!source) {
+            throw new Error("Source is required");
+        }
+
+        const githubParsed = parseGitHubRepo(source);
+        if (githubParsed) {
+            return new GitHubSkillReader({
+                repoUrl: source,
+                gitRef: options.gitRef,
+                dir: options.subDir,
+                token: options.githubToken ?? dntShim.Deno.env.get("GITHUB_TOKEN") ?? undefined,
+            });
+        }
+
+        if (isHttpUrl(source)) {
+            throw new Error(
+                "Invalid source: URL must be a GitHub repository URL or a local directory path",
+            );
+        }
+
+        const rootPath = options.subDir ? join(source, options.subDir) : source;
+        if (options.gitRef) {
+            await ensureDirectory(source);
+            await ensureGitRepoRoot(source);
+            const reader = new LocalGitSkillReader({
+                repoRoot: source,
+                gitRef: options.gitRef,
+                subDir: options.subDir,
+            });
+            await reader.validateRepositoryRef();
+            return reader;
+        }
+
+        await ensureDirectory(rootPath);
+        return new LocalFsSkillReader({ root: rootPath });
+    }
+}
+
+function isHttpUrl(value: string): boolean {
+    return /^https?:\/\//i.test(value);
+}
+
+async function ensureDirectory(path: string): Promise<void> {
+    try {
+        const stat = await dntShim.Deno.stat(path);
+        if (!stat.isDirectory) {
+            throw new Error("Path must be a directory");
+        }
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Local path not found: ${message}`);
+    }
+}