@nanmicoder/dsh-skills-hub 0.0.1

package/lib/market/skillhub-provider.js

@@ -0,0 +1,277 @@
+/**
+ * SkillHub provider (https://api.skillhub.cn)
+ *
+ * Endpoints (verified against the live API):
+ *  - GET /api/skills?page=&pageSize=&keyword=      → {code, data:{skills[], total}, message}
+ *      NOTE: pagination param MUST be `pageSize` (a `limit` param is silently ignored)
+ *      NOTE: search param MUST be `keyword` (a `q` param is silently ignored)
+ *  - GET /api/v1/skills/{slug}                     → {skill, owner, latestVersion, securityReports}
+ *  - GET /api/v1/skills/{slug}/files               → {count, files:[{path, sha256, size}]}
+ *  - GET /api/v1/skills/{slug}/file?path=          → 302 redirect to Tencent COS (follow)
+ */
+import { getProviderBase, providerFetch, providerFetchJson, readResponseTextWithLimit, MarketHttpError, } from "./provider-fetch.js";
+import { detectMarketLanguage, MARKET_ERROR_CODES, MARKET_LIMITS, MarketUpstreamError, skillId, } from "./types.js";
+// The frontmatter parser is shared with the ClawHub provider: both sources hand
+// us a raw SKILL.md and the detail contract promises a frontmatter-free body.
+import { parseFrontmatter } from "./clawhub-provider.js";
+import { markSourceHealth } from "./cache.js";
+/**
+ * Vendor statuses that mean "no findings". Anything else (suspicious,
+ * malicious, …) flags the skill: an unrecognised vendor status must never be
+ * reported as safe.
+ */
+const BENIGN_STATUSES = new Set(['benign', 'safe', 'clean']);
+// ─── Untrusted-payload helpers ───────────────────────────────────────────────
+/**
+ * Upstream JSON is untrusted input: a field declared `string` in the ported
+ * types can arrive as an object, an array or a number. These helpers keep every
+ * `NormalizedSkill` field at its declared runtime type.
+ */
+function asString(value) {
+    return typeof value === 'string' ? value : undefined;
+}
+function asNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+}
+/** A usable slug is a non-empty string — anything else is not a skill entry. */
+function hasUsableSlug(item) {
+    if (typeof item !== 'object' || item === null)
+        return false;
+    const slug = item.slug;
+    return typeof slug === 'string' && slug.trim() !== '';
+}
+/** File entries are only useful with a string path; sizes default to 0. */
+function usableFileEntries(files) {
+    if (!Array.isArray(files))
+        return [];
+    const out = [];
+    for (const file of files) {
+        if (typeof file !== 'object' || file === null)
+            continue;
+        const record = file;
+        if (typeof record.path !== 'string' || record.path === '')
+            continue;
+        out.push({
+            path: record.path,
+            size: asNumber(record.size) ?? 0,
+            sha256: asString(record.sha256),
+            contentType: asString(record.contentType),
+        });
+    }
+    return out;
+}
+function mapSecurity(reports, verified) {
+    const normalized = [];
+    for (const [vendor, report] of Object.entries(reports ?? {})) {
+        const status = asString(report?.status);
+        if (status === undefined)
+            continue;
+        normalized.push({
+            vendor,
+            status,
+            statusText: asString(report?.statusText) || status,
+            reportUrl: asString(report?.reportUrl),
+        });
+    }
+    if (normalized.length === 0)
+        return { status: 'unknown', reports: normalized };
+    const anyFlagged = normalized.some((report) => !BENIGN_STATUSES.has(report.status.toLowerCase()));
+    if (anyFlagged)
+        return { status: 'flagged', reports: normalized };
+    return { status: verified ? 'verified' : 'benign', reports: normalized };
+}
+function parseUpstream(item) {
+    const upstreamUrl = asString(item.upstream_url);
+    if (item.source !== 'clawhub' || !upstreamUrl)
+        return undefined;
+    // upstream_url looks like https://clawhub.ai/{owner}/{slug} — the trailing segment is the slug.
+    try {
+        const segments = new URL(upstreamUrl).pathname.split('/').filter(Boolean);
+        const slug = segments[segments.length - 1];
+        if (slug)
+            return { source: 'clawhub', slug };
+    }
+    catch {
+        // Malformed upstream URL — treat as a native entry.
+    }
+    return undefined;
+}
+function normalizeListItem(item) {
+    const tags = [];
+    for (const sub of item.subCategories ?? []) {
+        const name = asString(sub?.name);
+        if (name)
+            tags.push(name);
+    }
+    const verified = item.verified === undefined ? undefined : Boolean(item.verified);
+    return {
+        id: skillId('skillhub', item.slug),
+        source: 'skillhub',
+        slug: item.slug,
+        name: asString(item.name) || asString(item.displayName) || item.slug,
+        // Prefer the Chinese copy: this market's primary audience reads zh-CN.
+        summary: asString(item.description_zh)
+            || asString(item.description)
+            || asString(item.summary_zh)
+            || asString(item.summary)
+            || '',
+        author: { handle: asString(item.ownerName) || '' },
+        stats: {
+            downloads: asNumber(item.downloads) ?? 0,
+            installs: asNumber(item.installs),
+            stars: asNumber(item.stars),
+        },
+        tags,
+        category: asString(item.category),
+        version: asString(item.version),
+        updatedAt: asNumber(item.updated_at) ?? asNumber(item.updatedAt),
+        iconUrl: asString(item.iconUrl) || undefined,
+        // List responses carry no security reports — `verified` is the only signal;
+        // the detail endpoint refines this to benign/flagged via securityReports.
+        securityStatus: verified ? 'verified' : 'unknown',
+        requiresApiKey: item.labels?.requires_api_key === 'true',
+        verified,
+        upstream: parseUpstream(item),
+        installState: 'installable',
+    };
+}
+/**
+ * Fetch one page of the `{code, data, message}` envelope.
+ *
+ * SkillHub paginates by page number, so the "cursor" is just the next page
+ * number as a string; `total` (not the page length) decides exhaustion.
+ */
+async function fetchPage(params) {
+    const base = getProviderBase('skillhub');
+    const url = new URL('/api/skills', base);
+    url.searchParams.set('page', String(params.page));
+    url.searchParams.set('pageSize', String(params.pageSize));
+    if (params.keyword)
+        url.searchParams.set('keyword', params.keyword);
+    const envelope = await providerFetchJson('skillhub', url.toString());
+    if (envelope.code !== 0 || !Array.isArray(envelope.data?.skills)) {
+        throw new MarketUpstreamError('skillhub', MARKET_ERROR_CODES.upstreamBadResponse, `skillhub responded code=${envelope.code}: ${envelope.message || 'bad payload'}`);
+    }
+    const rawItems = envelope.data.skills;
+    const usable = rawItems.filter(hasUsableSlug);
+    if (usable.length === 0 && rawItems.length > 0) {
+        // Every entry was malformed: report the payload instead of silently serving
+        // an empty page while `/status` still claims the source is healthy.
+        const error = new MarketUpstreamError('skillhub', MARKET_ERROR_CODES.upstreamBadResponse, 'skillhub list contained no usable items');
+        markSourceHealth('skillhub', 'degraded', error.message);
+        throw error;
+    }
+    if (usable.length !== rawItems.length) {
+        markSourceHealth('skillhub', 'degraded', 'skillhub list contained malformed items');
+    }
+    const items = usable.map(normalizeListItem);
+    const total = envelope.data.total ?? 0;
+    const hasMore = params.page * params.pageSize < total;
+    return {
+        items,
+        nextCursor: hasMore ? String(params.page + 1) : undefined,
+        total,
+    };
+}
+function cursorToPage(cursor) {
+    return cursor ? Math.max(1, Number.parseInt(cursor, 10) || 1) : 1;
+}
+export const skillhubProvider = {
+    source: 'skillhub',
+    async list({ cursor, limit }) {
+        return fetchPage({ page: cursorToPage(cursor), pageSize: limit });
+    },
+    async search({ q, cursor, limit }) {
+        return fetchPage({ keyword: q, page: cursorToPage(cursor), pageSize: limit });
+    },
+    async detail(slug) {
+        const base = getProviderBase('skillhub');
+        const data = await providerFetchJson('skillhub', new URL(`/api/v1/skills/${encodeURIComponent(slug)}`, base).toString());
+        if (typeof data.skill?.slug !== 'string' || data.skill.slug.trim() === '') {
+            throw new MarketUpstreamError('skillhub', MARKET_ERROR_CODES.upstreamBadResponse, 'skillhub detail missing skill');
+        }
+        const item = normalizeListItem(data.skill);
+        if (data.skill.stats) {
+            item.stats = {
+                downloads: asNumber(data.skill.stats.downloads) ?? item.stats.downloads,
+                installs: asNumber(data.skill.stats.installs) ?? item.stats.installs,
+                stars: asNumber(data.skill.stats.stars) ?? item.stats.stars,
+            };
+        }
+        const security = mapSecurity(data.securityReports, data.skill.verified === undefined ? undefined : Boolean(data.skill.verified));
+        const version = asString(data.latestVersion?.version) || item.version;
+        let files = [];
+        try {
+            files = await skillhubProvider.listFiles(slug);
+        }
+        catch {
+            // File list is best-effort at detail time; install re-fetches it.
+        }
+        // SkillHub detail has no full SKILL.md body — fetch it for the overview tab.
+        let description = '';
+        let descriptionFrontmatter;
+        const skillMd = files.find((file) => file.path === 'SKILL.md');
+        if (skillMd) {
+            try {
+                const fetched = await skillhubProvider.fetchFile(slug, 'SKILL.md');
+                description = fetched.content;
+            }
+            catch {
+                description = item.summary;
+            }
+        }
+        else {
+            description = item.summary;
+        }
+        // This endpoint serves SKILL.md verbatim, frontmatter included. Strip it and
+        // expose the parsed map like the ClawHub provider does: the overview tab
+        // would otherwise render the raw `---` block as body text, and the metadata
+        // panel would never appear for SkillHub skills.
+        if (description.startsWith('---')) {
+            const parsed = parseFrontmatter(description);
+            description = parsed.content;
+            descriptionFrontmatter = parsed.frontmatter;
+        }
+        return {
+            ...item,
+            version,
+            author: {
+                handle: asString(data.owner?.handle) || item.author.handle,
+                displayName: asString(data.owner?.displayName),
+                avatarUrl: asString(data.owner?.image) || undefined,
+            },
+            securityStatus: security.status,
+            securityReports: security.reports.length ? security.reports : undefined,
+            description,
+            ...(descriptionFrontmatter ? { descriptionFrontmatter } : {}),
+            files: files.map((file) => ({
+                path: file.path,
+                size: file.size,
+                sha256: file.sha256,
+                contentType: file.contentType,
+                language: detectMarketLanguage(file.path),
+                tooBig: false,
+            })),
+            totalSize: files.reduce((sum, file) => sum + file.size, 0),
+        };
+    },
+    async listFiles(slug) {
+        const base = getProviderBase('skillhub');
+        const data = await providerFetchJson('skillhub', new URL(`/api/v1/skills/${encodeURIComponent(slug)}/files`, base).toString());
+        if (!Array.isArray(data.files)) {
+            throw new MarketUpstreamError('skillhub', MARKET_ERROR_CODES.upstreamBadResponse, 'skillhub files missing list');
+        }
+        return usableFileEntries(data.files);
+    },
+    async fetchFile(slug, filePath) {
+        const base = getProviderBase('skillhub');
+        const url = new URL(`/api/v1/skills/${encodeURIComponent(slug)}/file`, base);
+        url.searchParams.set('path', filePath);
+        // 302 → Tencent COS; providerFetch follows redirects.
+        const res = await providerFetch('skillhub', url.toString());
+        if (!res.ok) {
+            throw new MarketHttpError('skillhub', res.status, res.status === 404 ? MARKET_ERROR_CODES.upstreamBadResponse : MARKET_ERROR_CODES.upstreamError, `skillhub file fetch failed (${res.status})`);
+        }
+        return await readResponseTextWithLimit('skillhub', res, MARKET_LIMITS.maxFileSize, `file ${filePath}`);
+    },
+};

package/lib/market/types.js

@@ -0,0 +1,82 @@
+/**
+ * Skills Market — shared types for the market aggregation layer.
+ *
+ * Upstream sources:
+ *  - ClawHub  (https://clawhub.ai)      — cursor pagination, no security audits
+ *  - SkillHub (https://api.skillhub.cn) — page/pageSize pagination, security reports
+ */
+export const MARKET_SOURCES = ['clawhub', 'skillhub'];
+// ─── Error codes ─────────────────────────────────────────────────────────────
+export const MARKET_ERROR_CODES = {
+    upstreamError: 'MARKET_UPSTREAM_ERROR',
+    upstreamTimeout: 'MARKET_UPSTREAM_TIMEOUT',
+    upstreamBadResponse: 'MARKET_UPSTREAM_BAD_RESPONSE',
+    installInProgress: 'MARKET_INSTALL_IN_PROGRESS',
+    alreadyInstalled: 'MARKET_ALREADY_INSTALLED',
+    notInstallable: 'MARKET_NOT_INSTALLABLE',
+    checksumMismatch: 'MARKET_CHECKSUM_MISMATCH',
+    diskError: 'MARKET_DISK_ERROR',
+    notInstalled: 'MARKET_NOT_INSTALLED',
+    notManaged: 'MARKET_NOT_MANAGED',
+};
+export class MarketUpstreamError extends Error {
+    source;
+    code;
+    constructor(source, code, message) {
+        super(message);
+        this.source = source;
+        this.code = code;
+        this.name = 'MarketUpstreamError';
+    }
+}
+// ─── Limits ──────────────────────────────────────────────────────────────────
+export const MARKET_LIMITS = {
+    /** Max bytes for a single skill file (install + preview) */
+    maxFileSize: 5 * 1024 * 1024,
+    /** Max total bytes for an installable skill */
+    maxTotalSize: 20 * 1024 * 1024,
+    /** Max file count for an installable skill */
+    maxFileCount: 200,
+    /** File preview content is truncated beyond this many bytes */
+    previewTruncateBytes: 300 * 1024,
+    /** ClawHub search has no pagination — cap merged search results */
+    searchResultCap: 50,
+};
+export function skillId(source, slug) {
+    return `${source}:${slug}`;
+}
+export function parseSkillId(id) {
+    const idx = id.indexOf(':');
+    if (idx <= 0)
+        return null;
+    const source = id.slice(0, idx);
+    const slug = id.slice(idx + 1);
+    if (!MARKET_SOURCES.includes(source) || !slug)
+        return null;
+    return { source: source, slug };
+}
+/** Directory-name whitelist: lowercase alnum, dash, underscore, dot (no leading dot). */
+export function sanitizeDirName(slug) {
+    // Upstream payloads are untrusted: a non-string slug is simply not a usable
+    // directory name, and must degrade to "not installable" instead of throwing
+    // from `slug.toLowerCase()` deep inside a list request.
+    if (typeof slug !== 'string')
+        return null;
+    const name = slug.toLowerCase();
+    if (!/^[a-z0-9][a-z0-9._-]*$/.test(name))
+        return null;
+    if (name.includes('..'))
+        return null;
+    return name;
+}
+const LANG_MAP = {
+    md: 'markdown', ts: 'typescript', tsx: 'typescript',
+    js: 'javascript', jsx: 'javascript', mjs: 'javascript', cjs: 'javascript',
+    json: 'json', yaml: 'yaml', yml: 'yaml', sh: 'bash', bash: 'bash', zsh: 'bash',
+    py: 'python', toml: 'toml', css: 'css', html: 'html',
+    txt: 'text', xml: 'xml', sql: 'sql', rs: 'rust', go: 'go', rb: 'ruby',
+};
+export function detectMarketLanguage(filename) {
+    const ext = filename.split('.').pop()?.toLowerCase() || '';
+    return LANG_MAP[ext] || 'text';
+}