@feiyoug/skill-lab 0.0.0 → 0.0.2

package/src/skillreader/fs/git.ts

@@ -0,0 +1,153 @@
+import * as dntShim from "../../_dnt.shims.js";
+import { getFileType } from "../../shared/mod.js";
+import { parseSkillManifest } from "../manifest.js";
+import type { SkillFile } from "../types.js";
+import { SkillReader } from "../types.js";
+import { isProbablyText } from "../utils/mod.js";
+
+type LocalGitSkillReaderOptions = {
+    repoRoot: string;
+    gitRef: string;
+    subDir?: string;
+};
+
+type GitEntry = {
+    fullPath: string;
+    relativePath: string;
+};
+
+export class LocalGitSkillReader extends SkillReader {
+    private readonly repoRoot: string;
+    private readonly gitRef: string;
+    private readonly subDir: string;
+
+    constructor(options: LocalGitSkillReaderOptions) {
+        super();
+        this.repoRoot = options.repoRoot;
+        this.gitRef = options.gitRef;
+        this.subDir = options.subDir?.replace(/^\/+|\/+$/g, "") ?? "";
+    }
+
+    async retrieveFiles(dir?: string): Promise<SkillFile[]> {
+        const entries = await this.listEntries(dir);
+        return entries.map((entry) => ({
+            path: entry.relativePath,
+            contentType: getFileType(entry.relativePath),
+        }));
+    }
+
+    async readTextFile(path: string): Promise<string | null> {
+        if (getFileType(path) === "binary") return null;
+        const stream = await this.readFile(path);
+        if (!stream) return null;
+        const bytes = new Uint8Array(await new Response(stream).arrayBuffer());
+        if (!isProbablyText(bytes)) return null;
+        return new TextDecoder().decode(bytes);
+    }
+
+    async readFile(path: string): Promise<ReadableStream<Uint8Array> | null> {
+        const fullPath = this.joinWithSubDir(path);
+        const result = await runGit(this.repoRoot, ["show", `${this.gitRef}:${fullPath}`]);
+        if (result.code !== 0) return null;
+        return new Response(new Uint8Array(result.stdout)).body;
+    }
+
+    async readManifest() {
+        try {
+            const text = await this.readTextFile("manifest.json");
+            if (!text) return null;
+            return parseSkillManifest(text);
+        } catch {
+            return null;
+        }
+    }
+
+    async validateRepositoryRef(): Promise<void> {
+        const result = await runGit(this.repoRoot, ["rev-parse", "--verify", this.gitRef]);
+        if (result.code !== 0) {
+            throw new Error(`Git reference not found: ${this.gitRef}`);
+        }
+    }
+
+    private async listEntries(dir?: string): Promise<GitEntry[]> {
+        const scoped = this.joinWithSubDir(dir);
+        const args = ["ls-tree", "-r", "--name-only", this.gitRef];
+        if (scoped) {
+            args.push("--", scoped);
+        }
+        const result = await runGit(this.repoRoot, args);
+        if (result.code !== 0) {
+            throw new Error(`Failed to list files for git reference: ${this.gitRef}`);
+        }
+
+        const files = new TextDecoder().decode(result.stdout)
+            .split("\n")
+            .map((line) => line.trim())
+            .filter(Boolean);
+
+        const rootPrefix = this.joinWithSubDir(dir);
+        return files
+            .map((fullPath) => {
+                const relativePath = rootPrefix
+                    ? fullPath.startsWith(`${rootPrefix}/`)
+                        ? fullPath.slice(rootPrefix.length + 1)
+                        : ""
+                    : fullPath;
+                return { fullPath, relativePath };
+            })
+            .filter((entry) => entry.relativePath.length > 0);
+    }
+
+    private joinWithSubDir(path?: string): string {
+        const cleaned = path?.replace(/^\/+|\/+$/g, "") ?? "";
+        return [this.subDir, cleaned].filter(Boolean).join("/");
+    }
+}
+
+async function runGit(repoRoot: string, args: string[]) {
+    const denoRuntime = dntShim.Deno as unknown as {
+        Command?: new (
+            command: string,
+            options: {
+                args: string[];
+                stdout: "piped";
+                stderr: "piped";
+            },
+        ) => { output: () => Promise<{ code: number; stdout: Uint8Array; stderr: Uint8Array }> };
+    };
+
+    if (denoRuntime.Command) {
+        return await new denoRuntime.Command("git", {
+            args: ["-C", repoRoot, ...args],
+            stdout: "piped",
+            stderr: "piped",
+        }).output();
+    }
+
+    const { spawnSync } = await import("node:child_process");
+    const output = spawnSync("git", ["-C", repoRoot, ...args], {
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    return {
+        code: output.status ?? 1,
+        stdout: output.stdout ? new Uint8Array(output.stdout) : new Uint8Array(),
+        stderr: output.stderr ? new Uint8Array(output.stderr) : new Uint8Array(),
+    };
+}
+
+export async function ensureGitRepoRoot(path: string): Promise<void> {
+    const result = await runGit(path, ["rev-parse", "--show-toplevel"]);
+    if (result.code !== 0) {
+        throw new Error("Local path is not a git repository root");
+    }
+
+    const topLevel = new TextDecoder().decode(result.stdout).trim();
+    const canonicalPath = await dntShim.Deno.realPath(path);
+    const canonicalTopLevel = await dntShim.Deno.realPath(topLevel);
+    if (canonicalPath !== canonicalTopLevel) {
+        throw new Error(
+            "When --gitRef is used with local sources, path must point to the git repository root",
+        );
+    }
+}

package/src/skillreader/fs/mod.ts

@@ -0,0 +1,84 @@
+import * as dntShim from "../../_dnt.shims.js";
+import { join, relative } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { isProbablyText } from "../utils/mod.js";
+import type { SkillFile } from "../types.js";
+import { SkillReader } from "../types.js";
+import { parseSkillManifest } from "../manifest.js";
+import { getFileType } from "../../shared/mod.js";
+
+export type LocalFsSkillReaderOptions = {
+    root: string;
+};
+
+export class LocalFsSkillReader extends SkillReader {
+    private root: string;
+
+    constructor(options: LocalFsSkillReaderOptions) {
+        super();
+        this.root = options.root;
+    }
+
+    async retrieveFiles(dir?: string): Promise<SkillFile[]> {
+        const base = dir ? join(this.root, dir) : this.root;
+        const files: SkillFile[] = [];
+
+        for await (const entry of this.walkDir(base)) {
+            if (!entry.isFile) continue;
+            const relPath = relative(this.root, entry.path);
+            files.push({
+                path: relPath.replace(/\\/g, "/"),
+                size: entry.size,
+                contentType: getFileType(relPath),
+            });
+        }
+
+        return files;
+    }
+
+    async readTextFile(path: string): Promise<string | null> {
+        const fullPath = join(this.root, path);
+        const contentType = getFileType(path);
+        if (contentType === "binary") return null;
+        try {
+            const data = await dntShim.Deno.readFile(fullPath);
+            if (!isProbablyText(data)) return null;
+            return new TextDecoder().decode(data);
+        } catch {
+            return null;
+        }
+    }
+
+    async readFile(path: string): Promise<ReadableStream<Uint8Array> | null> {
+        const fullPath = join(this.root, path);
+        try {
+            const file = await dntShim.Deno.open(fullPath, { read: true });
+            return file.readable;
+        } catch {
+            return null;
+        }
+    }
+
+    async readManifest() {
+        const manifestPath = join(this.root, "manifest.json");
+        try {
+            const text = await dntShim.Deno.readTextFile(manifestPath);
+            return parseSkillManifest(text);
+        } catch {
+            return null;
+        }
+    }
+
+    private async *walkDir(
+        dir: string,
+    ): AsyncGenerator<{ path: string; size: number; isFile: boolean }> {
+        for await (const entry of dntShim.Deno.readDir(dir)) {
+            const fullPath = join(dir, entry.name);
+            if (entry.isDirectory) {
+                yield* this.walkDir(fullPath);
+            } else if (entry.isFile) {
+                const stat = await dntShim.Deno.stat(fullPath);
+                yield { path: fullPath, size: stat.size, isFile: true };
+            }
+        }
+    }
+}

package/src/skillreader/github/base.ts

@@ -0,0 +1,162 @@
+import { isProbablyText } from "../utils/mod.js";
+import type { SkillFile } from "../types.js";
+import { SkillReader } from "../types.js";
+import { parseSkillManifest } from "../manifest.js";
+import {
+    buildGitHubHeaders,
+    buildPrefix,
+    normalizeRelativePath,
+    parseGitHubRepo,
+} from "./utils.js";
+import { getFileType } from "../../shared/mod.js";
+
+export type GitHubTreeEntry = {
+    path: string;
+    type: string;
+    sha: string;
+    size?: number;
+};
+
+export type AbstractGitHubSkillReaderOptions = {
+    repoUrl: string;
+    gitRef?: string;
+    dir?: string;
+    token?: string;
+};
+
+export abstract class AbstractGitHubSkillReader extends SkillReader {
+    protected owner: string;
+    protected repo: string;
+    protected dir: string;
+    protected token?: string;
+
+    private requestedGitRef?: string;
+    private resolvedGitRef?: string;
+    private treeEntries: Map<string, GitHubTreeEntry> | null = null;
+
+    constructor(options: AbstractGitHubSkillReaderOptions) {
+        super();
+        const parsed = parseGitHubRepo(options.repoUrl);
+        if (!parsed) throw new Error("Unsupported repo URL");
+        this.owner = parsed.owner;
+        this.repo = parsed.repo;
+        this.requestedGitRef = options.gitRef;
+        this.dir = options.dir?.replace(/^\/+|\/+$/g, "") ?? "";
+        this.token = options.token;
+    }
+
+    async retrieveFiles(dir?: string): Promise<SkillFile[]> {
+        const entries = await this.getTreeEntries();
+        const prefix = buildPrefix(this.dir, dir);
+        const files: SkillFile[] = [];
+
+        for (const entry of entries.values()) {
+            if (entry.type !== "blob") continue;
+            if (prefix && !entry.path.startsWith(prefix)) continue;
+            const relativePath = normalizeRelativePath(prefix, entry.path);
+            if (!relativePath) continue;
+            files.push({
+                path: relativePath,
+                size: entry.size,
+                contentType: getFileType(relativePath),
+            });
+        }
+
+        return files;
+    }
+
+    async readTextFile(path: string): Promise<string | null> {
+        if (getFileType(path) === "binary") return null;
+        const stream = await this.readFile(path);
+        if (!stream) return null;
+        const buffer = new Uint8Array(await new Response(stream).arrayBuffer());
+        if (!isProbablyText(buffer)) return null;
+        return new TextDecoder().decode(buffer);
+    }
+
+    async readFile(path: string): Promise<ReadableStream<Uint8Array> | null> {
+        const entry = await this.findEntry(path);
+        if (!entry) return null;
+        return this.readFileFromEntry(entry);
+    }
+
+    async readManifest() {
+        try {
+            const text = await this.readTextFile("manifest.json");
+            if (!text) return null;
+            return parseSkillManifest(text);
+        } catch {
+            return null;
+        }
+    }
+
+    protected abstract readFileFromEntry(
+        entry: GitHubTreeEntry,
+    ): Promise<ReadableStream<Uint8Array> | null>;
+
+    protected buildHeaders(): Record<string, string> {
+        return buildGitHubHeaders(this.token);
+    }
+
+    protected async resolveGitRef(): Promise<string> {
+        if (this.resolvedGitRef) return this.resolvedGitRef;
+        if (this.requestedGitRef) {
+            this.resolvedGitRef = this.requestedGitRef;
+            return this.resolvedGitRef;
+        }
+
+        const response = await fetch(
+            `https://api.github.com/repos/${this.owner}/${this.repo}`,
+            { headers: this.buildHeaders() },
+        );
+        if (!response.ok) {
+            throw new Error(`GitHub repository fetch failed: ${response.status}`);
+        }
+        const repoData = await response.json() as { default_branch?: string };
+        if (!repoData.default_branch) {
+            throw new Error("GitHub repository metadata missing default branch");
+        }
+        this.resolvedGitRef = repoData.default_branch;
+        return this.resolvedGitRef;
+    }
+
+    protected async getTreeEntries(): Promise<Map<string, GitHubTreeEntry>> {
+        if (this.treeEntries) return this.treeEntries;
+
+        const gitRef = await this.resolveGitRef();
+        const headers = this.buildHeaders();
+        const commitResponse = await fetch(
+            `https://api.github.com/repos/${this.owner}/${this.repo}/commits/${gitRef}`,
+            { headers },
+        );
+        if (!commitResponse.ok) {
+            throw new Error(`GitHub commit fetch failed: ${commitResponse.status}`);
+        }
+        const commitData = await commitResponse.json() as {
+            commit?: { tree?: { sha?: string } };
+        };
+        const treeSha = commitData?.commit?.tree?.sha;
+        if (!treeSha) throw new Error("Missing tree SHA");
+
+        const treeResponse = await fetch(
+            `https://api.github.com/repos/${this.owner}/${this.repo}/git/trees/${treeSha}?recursive=1`,
+            { headers },
+        );
+        if (!treeResponse.ok) {
+            throw new Error(`GitHub tree fetch failed: ${treeResponse.status}`);
+        }
+        const treeData = await treeResponse.json() as { tree?: GitHubTreeEntry[] };
+        const entries = new Map<string, GitHubTreeEntry>();
+        for (const entry of treeData?.tree ?? []) {
+            entries.set(entry.path, entry);
+        }
+        this.treeEntries = entries;
+        return entries;
+    }
+
+    private async findEntry(path: string): Promise<GitHubTreeEntry | null> {
+        const entries = await this.getTreeEntries();
+        const fullPath = this.dir ? `${this.dir}/${path}` : path;
+        return entries.get(fullPath) ?? null;
+    }
+}

package/src/skillreader/github/githubApi.ts

@@ -0,0 +1,40 @@
+import {
+    AbstractGitHubSkillReader,
+    type AbstractGitHubSkillReaderOptions,
+    type GitHubTreeEntry,
+} from "./base.js";
+
+export type GitHubApiSkillReaderOptions = AbstractGitHubSkillReaderOptions & {
+    token: string;
+};
+
+export class GitHubApiSkillReader extends AbstractGitHubSkillReader {
+    constructor(options: GitHubApiSkillReaderOptions) {
+        super(options);
+    }
+
+    protected async readFileFromEntry(
+        entry: GitHubTreeEntry,
+    ): Promise<ReadableStream<Uint8Array> | null> {
+        const blobResponse = await fetch(
+            `https://api.github.com/repos/${this.owner}/${this.repo}/git/blobs/${entry.sha}`,
+            { headers: this.buildHeaders() },
+        );
+        if (!blobResponse.ok) return null;
+        const blobData = await blobResponse.json() as { content?: string; encoding?: string };
+        if (!blobData.content || blobData.encoding !== "base64") return null;
+        const buffer = decodeBase64ToBuffer(blobData.content);
+        return new Response(buffer).body;
+    }
+}
+
+function decodeBase64ToBuffer(input: string): ArrayBuffer {
+    const cleaned = input.replace(/\n/g, "");
+    const binary = atob(cleaned);
+    const buffer = new ArrayBuffer(binary.length);
+    const bytes = new Uint8Array(buffer);
+    for (let i = 0; i < binary.length; i += 1) {
+        bytes[i] = binary.charCodeAt(i);
+    }
+    return buffer;
+}

package/src/skillreader/github/githubRaw.ts

@@ -0,0 +1,24 @@
+import {
+    AbstractGitHubSkillReader,
+    type AbstractGitHubSkillReaderOptions,
+    type GitHubTreeEntry,
+} from "./base.js";
+
+export type GitHubRawSkillReaderOptions = AbstractGitHubSkillReaderOptions;
+
+export class GitHubRawSkillReader extends AbstractGitHubSkillReader {
+    constructor(options: GitHubRawSkillReaderOptions) {
+        super(options);
+    }
+
+    protected async readFileFromEntry(
+        entry: GitHubTreeEntry,
+    ): Promise<ReadableStream<Uint8Array> | null> {
+        const gitRef = await this.resolveGitRef();
+        const rawUrl =
+            `https://raw.githubusercontent.com/${this.owner}/${this.repo}/${gitRef}/${entry.path}`;
+        const response = await fetch(rawUrl);
+        if (!response.ok) return null;
+        return response.body;
+    }
+}

package/src/skillreader/github/mod.ts

@@ -0,0 +1,45 @@
+import { SkillReader } from "../types.js";
+import { GitHubApiSkillReader } from "./githubApi.js";
+import { GitHubRawSkillReader } from "./githubRaw.js";
+
+export type GitHubSkillReaderOptions = {
+    repoUrl: string;
+    gitRef?: string;
+    dir?: string;
+    token?: string;
+};
+
+export class GitHubSkillReader extends SkillReader {
+    private reader: SkillReader;
+
+    constructor(options: GitHubSkillReaderOptions) {
+        super();
+        if (options.token) {
+            this.reader = new GitHubApiSkillReader({
+                ...options,
+                token: options.token,
+            });
+        } else {
+            this.reader = new GitHubRawSkillReader(options);
+        }
+    }
+
+    retrieveFiles(dir?: string) {
+        return this.reader.retrieveFiles(dir);
+    }
+
+    readTextFile(path: string) {
+        return this.reader.readTextFile(path);
+    }
+
+    readFile(path: string) {
+        return this.reader.readFile(path);
+    }
+
+    readManifest() {
+        return this.reader.readManifest();
+    }
+}
+
+export { GitHubApiSkillReader } from "./githubApi.js";
+export { GitHubRawSkillReader } from "./githubRaw.js";

package/src/skillreader/github/utils.ts

@@ -0,0 +1,40 @@
+export type ParsedGitHubRepo = {
+    owner: string;
+    repo: string;
+};
+
+export function parseGitHubRepo(repoUrl: string): ParsedGitHubRepo | null {
+    try {
+        const parsed = new URL(repoUrl);
+        if (!(parsed.hostname === "github.com" || parsed.hostname === "www.github.com")) {
+            return null;
+        }
+        const match = parsed.pathname.match(/^\/([^\/]+)\/([^\/]+?)(\.git)?$/);
+        if (!match) return null;
+        return { owner: match[1], repo: match[2] };
+    } catch {
+        return null;
+    }
+}
+
+export function buildPrefix(baseDir?: string, extraDir?: string): string {
+    const cleanedBase = baseDir?.replace(/^\/+|\/+$/g, "") ?? "";
+    const cleanedExtra = extraDir?.replace(/^\/+|\/+$/g, "") ?? "";
+    const combined = [cleanedBase, cleanedExtra].filter(Boolean).join("/");
+    return combined ? `${combined}/` : "";
+}
+
+export function normalizeRelativePath(prefix: string, fullPath: string): string {
+    if (!prefix) return fullPath;
+    if (!fullPath.startsWith(prefix)) return "";
+    return fullPath.slice(prefix.length);
+}
+
+export function buildGitHubHeaders(token?: string): Record<string, string> {
+    const headers: Record<string, string> = {
+        Accept: "application/vnd.github.v3+json",
+        "User-Agent": "goskilla-api",
+    };
+    if (token) headers.Authorization = `Bearer ${token}`;
+    return headers;
+}

package/src/skillreader/manifest.ts

@@ -0,0 +1,67 @@
+import type {
+    SkillFile,
+    SkillFileManifest,
+    SkillManifest,
+    SkillZipManifest,
+    ZipManifestFile,
+} from "./types.js";
+
+export const SKILL_MANIFEST_VERSION = 1 as const;
+export const SKILL_ZIP_MANIFEST_VERSION = 2 as const;
+
+export function createSkillManifest(files: SkillFile[]): SkillFileManifest {
+    return {
+        version: SKILL_MANIFEST_VERSION,
+        generatedAt: new Date().toISOString(),
+        files,
+    };
+}
+
+export function createZipManifest(
+    files: ZipManifestFile[],
+    metadata: {
+        skillVersionId: string;
+        repoUrl: string;
+        commitHash: string;
+        dir?: string | null;
+        checksum: string;
+        compression: "deflate" | "store";
+    },
+): SkillZipManifest {
+    return {
+        version: SKILL_ZIP_MANIFEST_VERSION,
+        storage_method: "zip",
+        compression: metadata.compression,
+        skill_version_id: metadata.skillVersionId,
+        repo_url: metadata.repoUrl,
+        commit_hash: metadata.commitHash,
+        dir: metadata.dir ?? null,
+        created_at: new Date().toISOString(),
+        checksum: metadata.checksum,
+        files,
+    };
+}
+
+export function parseSkillManifest(text: string): SkillManifest | null {
+    try {
+        const parsed = JSON.parse(text) as SkillManifest;
+        if (!parsed || typeof parsed !== "object") return null;
+        if (!Array.isArray((parsed as { files?: unknown }).files)) return null;
+        if ((parsed as { version?: number }).version === SKILL_MANIFEST_VERSION) {
+            return parsed as SkillFileManifest;
+        }
+        if ((parsed as { version?: number }).version === SKILL_ZIP_MANIFEST_VERSION) {
+            const zipManifest = parsed as SkillZipManifest;
+            if (zipManifest.storage_method !== "zip") return null;
+            if (!zipManifest.compression) return null;
+            return zipManifest;
+        }
+        return null;
+    } catch {
+        return null;
+    }
+}
+
+export function isZipManifest(manifest: SkillManifest): manifest is SkillZipManifest {
+    return (manifest as SkillZipManifest).storage_method === "zip";
+}

package/src/skillreader/mod.ts

@@ -0,0 +1,26 @@
+import "../_dnt.polyfills.js";
+export type {
+    SkillFile,
+    SkillFileManifest,
+    SkillFrontmatter,
+    SkillManifest,
+    SkillReader,
+    SkillZipManifest,
+    ZipManifestFile,
+} from "./types.js";
+export {
+    createSkillManifest,
+    createZipManifest,
+    isZipManifest,
+    parseSkillManifest,
+    SKILL_MANIFEST_VERSION,
+    SKILL_ZIP_MANIFEST_VERSION,
+} from "./manifest.js";
+export { LocalFsSkillReader } from "./fs/mod.js";
+export { SkillReaderFactory } from "./factory.js";
+export type { SkillReaderFactoryOptions } from "./factory.js";
+export { Analyzer } from "../analyzer/mod.js";
+export type { AnalyzerAnalyzeInput } from "../analyzer/mod.js";
+export { GitHubApiSkillReader, GitHubRawSkillReader, GitHubSkillReader } from "./github/mod.js";
+export { CloudStorageSkillReader } from "./cloudStorage/mod.js";
+export type { CloudStorageSkillReaderOptions } from "./cloudStorage/mod.js";