@nanmicoder/dsh-skills-hub 0.0.1

package/lib/market/market-service.js

@@ -0,0 +1,312 @@
+/**
+ * Skills Market — aggregation service (host half).
+ *
+ * Merges the two upstream providers into a single paginated feed with
+ * cross-source dedupe, per-source health/degradation reporting, TTL caching
+ * (stale-while-error) and locally-computed install state.
+ *
+ * The module is deliberately free of DSH services and of the filesystem: the
+ * "what is installed locally" question is answered by an injected
+ * `InstalledLookup` (see `setInstalledLookup`), so this file can be unit-tested
+ * by pointing the providers at a stub server.
+ */
+import { getSourceHealth, markSourceHealth, MARKET_TTL, marketCache } from "./cache.js";
+import { clawhubProvider } from "./clawhub-provider.js";
+import { skillhubProvider } from "./skillhub-provider.js";
+import { MARKET_LIMITS, MARKET_SOURCES, detectMarketLanguage, sanitizeDirName, skillId, } from "./types.js";
+const providers = {
+    clawhub: clawhubProvider,
+    skillhub: skillhubProvider,
+};
+/** Nothing is installed until the host installs its real lookup. */
+const EMPTY_INSTALLED_LOOKUP = {
+    has: () => false,
+    info: () => undefined,
+};
+let installedLookup = EMPTY_INSTALLED_LOOKUP;
+export function setInstalledLookup(lookup) {
+    installedLookup = lookup;
+}
+/** Test hook: fall back to "nothing installed" (also used when a plugin unloads). */
+export function resetInstalledLookup() {
+    installedLookup = EMPTY_INSTALLED_LOOKUP;
+}
+export function encodeCursor(cursor) {
+    const keys = Object.keys(cursor);
+    if (keys.length === 0)
+        return null;
+    return Buffer.from(JSON.stringify(cursor), 'utf-8').toString('base64url');
+}
+export function decodeCursor(raw) {
+    if (!raw)
+        return undefined;
+    try {
+        const parsed = JSON.parse(Buffer.from(raw, 'base64url').toString('utf-8'));
+        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
+            return undefined;
+        const cursor = {};
+        const record = parsed;
+        for (const source of MARKET_SOURCES) {
+            const value = record[source];
+            if (typeof value === 'string' && value)
+                cursor[source] = value;
+        }
+        return cursor;
+    }
+    catch {
+        return undefined;
+    }
+}
+// ─── Install-state annotation ────────────────────────────────────────────────
+/**
+ * Stamp a market item with its local install state.
+ *
+ * Computed on every request (never cached): a skill installed a second ago must
+ * show up as installed on the next list call. The reference version also probed
+ * the skills directory for a name conflict; that check needs a filesystem, so it
+ * now belongs to the install path, which refuses to overwrite a directory it did
+ * not create.
+ */
+export function annotateInstallState(skill) {
+    // A non-string slug is untrusted upstream data: degrade it to "not
+    // installable" instead of throwing the whole list request away.
+    const dirName = typeof skill.slug === 'string' ? sanitizeDirName(skill.slug) : null;
+    if (!dirName) {
+        return { ...skill, installState: 'not-installable', notInstallableReason: 'invalid-name' };
+    }
+    const id = skillId(skill.source, skill.slug);
+    if (!installedLookup.has(id)) {
+        return { ...skill, installState: 'installable', notInstallableReason: undefined, installedInfo: undefined };
+    }
+    const info = installedLookup.info(id);
+    return {
+        ...skill,
+        installState: 'installed',
+        notInstallableReason: undefined,
+        installedInfo: {
+            dirName: info?.dirName ?? dirName,
+            version: info?.version,
+            installedAt: info?.installedAt,
+        },
+    };
+}
+/** File-level installability checks — only possible once the file list is known. */
+export function applyFileLimits(detail) {
+    const files = detail.files.map((file) => ({ ...file, tooBig: file.size > MARKET_LIMITS.maxFileSize }));
+    const result = { ...detail, files };
+    if (result.installState !== 'installable')
+        return result;
+    if (files.length === 0 || !files.some((file) => file.path === 'SKILL.md')) {
+        return { ...result, installState: 'not-installable', notInstallableReason: 'empty-file-list' };
+    }
+    if (files.length > MARKET_LIMITS.maxFileCount) {
+        return { ...result, installState: 'not-installable', notInstallableReason: 'too-many-files' };
+    }
+    if (files.some((file) => file.tooBig) || result.totalSize > MARKET_LIMITS.maxTotalSize) {
+        return { ...result, installState: 'not-installable', notInstallableReason: 'file-too-large' };
+    }
+    return result;
+}
+// ─── Cross-source dedupe ─────────────────────────────────────────────────────
+/**
+ * SkillHub mirrors ClawHub skills (source='clawhub' + upstream_url). When a
+ * page contains both the mirror and the ClawHub original, merge them: the
+ * ClawHub entry wins (fresher data), enriched with SkillHub-only fields.
+ *
+ * Unlike the reference this does not mutate its input: those items come from
+ * the shared response cache, and mutating them made `mirrors` grow on every
+ * repeat request.
+ */
+export function dedupeSkills(items) {
+    const copies = items.map((item) => ({ ...item }));
+    const byClawhubSlug = new Map();
+    for (const item of copies) {
+        if (item.source === 'clawhub')
+            byClawhubSlug.set(item.slug, item);
+    }
+    const result = [];
+    for (const item of copies) {
+        if (item.source === 'skillhub' && item.upstream?.slug) {
+            const original = byClawhubSlug.get(item.upstream.slug);
+            if (original) {
+                original.mirrors = [...(original.mirrors ?? []), item.id];
+                // Enrich the original with SkillHub-only data.
+                if (!original.iconUrl && item.iconUrl)
+                    original.iconUrl = item.iconUrl;
+                if (original.securityStatus === 'unknown' && item.securityStatus !== 'unknown') {
+                    original.securityStatus = item.securityStatus;
+                }
+                if (item.tags.length && original.tags.length === 0)
+                    original.tags = item.tags;
+                continue;
+            }
+        }
+        result.push(item);
+    }
+    return result;
+}
+/**
+ * Cache key for one provider page.
+ *
+ * The fields are JSON-encoded rather than `:`-joined: with a joined key,
+ * `q="alpha"` + `cursor="x:y"` and `q="alpha:x"` + `cursor="y"` produced the same
+ * string, so one request was served the other's page. `source` stays a leading
+ * segment so every key is obviously source-scoped.
+ */
+function providerPageCacheKey(source, kind, params) {
+    return `${kind}:${source}:${JSON.stringify([params.q ?? null, params.cursor ?? null, params.limit])}`;
+}
+async function fetchProviderPage(source, params) {
+    const isSearch = Boolean(params.q);
+    const cacheKey = providerPageCacheKey(source, isSearch ? 'search' : 'list', params);
+    const ttl = isSearch ? MARKET_TTL.search : MARKET_TTL.list;
+    const cached = marketCache.get(cacheKey);
+    if (cached) {
+        return { page: cached, status: { status: 'ok', fetchedAt: Date.now(), fromCache: true } };
+    }
+    try {
+        const q = params.q;
+        const page = isSearch && q !== undefined
+            ? await providers[source].search({ q, cursor: params.cursor, limit: params.limit })
+            : await providers[source].list({ cursor: params.cursor, limit: params.limit });
+        marketCache.set(cacheKey, page, ttl);
+        return { page, status: { status: 'ok', fetchedAt: Date.now(), fromCache: false } };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        // A provider that rejects a well-formed HTTP 200 (missing `items`, a
+        // non-zero SkillHub envelope code) never reaches the fetch layer's failure
+        // bookkeeping, so record it here — otherwise the source bar keeps claiming
+        // the registry is healthy while every request fails.
+        markSourceHealth(source, 'failed', message);
+        // Stale-while-error: a degraded source still serves its last good page, so a
+        // temporary upstream outage does not empty the panel.
+        const stale = marketCache.getStale(cacheKey);
+        if (stale) {
+            return {
+                page: stale.value,
+                status: { status: 'cached', fetchedAt: stale.storedAt, fromCache: true, error: message },
+            };
+        }
+        return { page: null, status: { ...getSourceHealth(source), fromCache: false, error: message } };
+    }
+}
+export async function listMarketSkills(params) {
+    const cursor = decodeCursor(params.cursor);
+    const isFirstPage = !params.cursor;
+    const activeSources = params.source === 'all' ? MARKET_SOURCES : [params.source];
+    const q = params.q;
+    const outcomes = new Map();
+    await Promise.all(activeSources.map(async (source) => {
+        // A source absent from a non-first-page cursor is exhausted.
+        const providerCursor = cursor?.[source];
+        if (!isFirstPage && !providerCursor) {
+            outcomes.set(source, { page: { items: [] }, status: { status: 'ok', fromCache: true } });
+            return;
+        }
+        // ClawHub search has no pagination: ask it for one big page and let the
+        // aggregate cap apply, instead of pretending a second page exists.
+        const limit = q && source === 'clawhub' ? MARKET_LIMITS.searchResultCap : params.limit;
+        outcomes.set(source, await fetchProviderPage(source, { q, cursor: providerCursor, limit }));
+    }));
+    let merged = [];
+    const nextCursor = {};
+    const sources = {};
+    for (const source of MARKET_SOURCES) {
+        const outcome = outcomes.get(source);
+        if (!outcome) {
+            sources[source] = { status: 'ok', fromCache: false };
+            continue;
+        }
+        sources[source] = outcome.status;
+        if (outcome.page) {
+            merged.push(...outcome.page.items);
+            if (outcome.page.nextCursor)
+                nextCursor[source] = outcome.page.nextCursor;
+        }
+    }
+    merged = dedupeSkills(merged);
+    merged.sort((a, b) => b.stats.downloads - a.stats.downloads);
+    merged = merged.map((item) => annotateInstallState(item));
+    if (params.security !== 'all') {
+        merged = merged.filter((item) => item.securityStatus === params.security);
+    }
+    if (params.installed !== 'all') {
+        merged = merged.filter((item) => params.installed === 'installed'
+            ? item.installState === 'installed'
+            : item.installState !== 'installed');
+    }
+    return { items: merged, nextCursor: encodeCursor(nextCursor), sources };
+}
+// ─── Detail / file content ───────────────────────────────────────────────────
+export async function getMarketSkillDetail(source, slug) {
+    const cacheKey = `detail:${source}:${slug}`;
+    let detail = marketCache.get(cacheKey);
+    let sourceStatus = { status: 'ok', fetchedAt: Date.now(), fromCache: true };
+    if (!detail) {
+        try {
+            detail = await providers[source].detail(slug);
+            marketCache.set(cacheKey, detail, MARKET_TTL.detail);
+            sourceStatus = { status: 'ok', fetchedAt: Date.now(), fromCache: false };
+        }
+        catch (error) {
+            const stale = marketCache.getStale(cacheKey);
+            if (!stale)
+                throw error;
+            detail = stale.value;
+            sourceStatus = {
+                status: 'cached',
+                fetchedAt: stale.storedAt,
+                fromCache: true,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+    const annotated = applyFileLimits(annotateInstallState(detail));
+    return { skill: annotated, sourceStatus };
+}
+export function isValidMarketFilePath(filePath) {
+    if (!filePath || filePath.length > 512)
+        return false;
+    if (filePath.startsWith('/') || filePath.startsWith('\\'))
+        return false;
+    if (filePath.includes('..') || filePath.includes('\0'))
+        return false;
+    return true;
+}
+export async function getMarketFileContent(source, slug, filePath) {
+    // JSON-encoded for the same reason as the list keys: slug and filePath are
+    // both caller-controlled, so a `:`-joined key is ambiguous.
+    const cacheKey = `file:${source}:${JSON.stringify([slug, filePath])}`;
+    const cached = marketCache.get(cacheKey);
+    if (cached)
+        return cached;
+    const fetched = await providers[source].fetchFile(slug, filePath);
+    let content = fetched.content;
+    let truncated = false;
+    if (Buffer.byteLength(content, 'utf-8') > MARKET_LIMITS.previewTruncateBytes) {
+        content = Buffer.from(content, 'utf-8').subarray(0, MARKET_LIMITS.previewTruncateBytes).toString('utf-8');
+        truncated = true;
+    }
+    const result = {
+        path: filePath,
+        content,
+        language: detectMarketLanguage(filePath),
+        size: fetched.size,
+        truncated,
+    };
+    marketCache.set(cacheKey, result, MARKET_TTL.fileContent);
+    return result;
+}
+// ─── Status ──────────────────────────────────────────────────────────────────
+export function getMarketStatus() {
+    return {
+        clawhub: getSourceHealth('clawhub'),
+        skillhub: getSourceHealth('skillhub'),
+    };
+}
+/** Look up a single skill (used by install) — the detail path, bypassing list. */
+export async function resolveMarketSkill(source, slug) {
+    const { skill } = await getMarketSkillDetail(source, slug);
+    return skill;
+}

package/lib/market/provider-fetch.js

@@ -0,0 +1,216 @@
+/**
+ * Skills Market — shared upstream fetch helper.
+ *
+ * Every upstream request goes through `providerFetch` so timeouts, retries,
+ * redirect following (SkillHub file downloads 302 to COS) and per-source health
+ * bookkeeping stay in one place.
+ *
+ * Base URLs, timeout and retry budget are injected with `configureProviderFetch`
+ * — the plugin passes its resolved Config in `apply()`. Nothing is read from
+ * `process.env` at module scope, so the module (and every provider built on it)
+ * is directly unit-testable by pointing the bases at a local stub.
+ */
+import { markSourceHealth } from "./cache.js";
+import { MARKET_ERROR_CODES, MARKET_LIMITS, MarketUpstreamError } from "./types.js";
+const DEFAULT_CONFIG = {
+    clawhubBaseUrl: 'https://clawhub.ai',
+    skillhubBaseUrl: 'https://api.skillhub.cn',
+    timeoutMs: 15_000,
+    retries: 1,
+    userAgent: 'dsh-skills-hub',
+};
+let currentConfig = { ...DEFAULT_CONFIG };
+/** Strip trailing slashes so `new URL('/api/...', base)` never doubles a separator. */
+function normalizeBaseUrl(value) {
+    return value.replace(/\/+$/, '');
+}
+/**
+ * Point the providers at different hosts / limits. Called by the plugin with the
+ * resolved Config; tests call it to aim the providers at a local stub server.
+ * Unspecified fields keep their current value, so a partial override can never
+ * silently blank the other source.
+ */
+export function configureProviderFetch(config) {
+    const next = { ...currentConfig };
+    if (typeof config.clawhubBaseUrl === 'string' && config.clawhubBaseUrl) {
+        next.clawhubBaseUrl = normalizeBaseUrl(config.clawhubBaseUrl);
+    }
+    if (typeof config.skillhubBaseUrl === 'string' && config.skillhubBaseUrl) {
+        next.skillhubBaseUrl = normalizeBaseUrl(config.skillhubBaseUrl);
+    }
+    if (typeof config.timeoutMs === 'number' && Number.isFinite(config.timeoutMs) && config.timeoutMs > 0) {
+        next.timeoutMs = config.timeoutMs;
+    }
+    if (typeof config.retries === 'number' && Number.isFinite(config.retries) && config.retries >= 0) {
+        next.retries = Math.floor(config.retries);
+    }
+    if (typeof config.userAgent === 'string' && config.userAgent) {
+        next.userAgent = config.userAgent;
+    }
+    currentConfig = next;
+}
+/** Base URL of one upstream, as resolved from the active configuration. */
+export function getProviderBase(source) {
+    return source === 'clawhub' ? currentConfig.clawhubBaseUrl : currentConfig.skillhubBaseUrl;
+}
+/**
+ * An upstream answered with a non-2xx status (or a retryable 429/5xx).
+ *
+ * Extends `MarketUpstreamError` so the existing `instanceof MarketUpstreamError`
+ * handling (and its `code`) keeps working, and adds the HTTP `status` so the
+ * route layer can tell "skill not found" apart from "upstream is broken".
+ */
+export class MarketHttpError extends MarketUpstreamError {
+    status;
+    constructor(source, status, code, message) {
+        super(source, code, message);
+        this.name = 'MarketHttpError';
+        this.status = status;
+    }
+}
+function errorFrom(source, message) {
+    return new MarketUpstreamError(source, MARKET_ERROR_CODES.upstreamError, message);
+}
+/** Merge the caller's headers over our defaults without losing either. */
+function requestHeaders(init, userAgent) {
+    const headers = {
+        Accept: 'application/json, text/plain, */*',
+        'User-Agent': userAgent,
+    };
+    if (!init?.headers)
+        return headers;
+    new Headers(init.headers).forEach((value, key) => {
+        headers[key] = value;
+    });
+    return headers;
+}
+/**
+ * Fetch an upstream URL with timeout and retries.
+ *
+ * `retries` is the number of *extra* attempts (the contract's default of 1 means
+ * two attempts), matching the reference server. A 429 or 5xx counts as a failed
+ * attempt; every other status — including 404 — is returned to the caller so the
+ * provider can map it to a meaningful error.
+ */
+export async function providerFetch(source, url, init) {
+    const config = currentConfig;
+    const attempts = Math.max(1, config.retries + 1);
+    let lastError;
+    for (let attempt = 0; attempt < attempts; attempt++) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), config.timeoutMs);
+        try {
+            // Respect a caller-supplied signal (request cancellation) alongside ours.
+            const signal = init?.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal;
+            const res = await fetch(url, {
+                ...init,
+                headers: requestHeaders(init, config.userAgent),
+                redirect: 'follow',
+                signal,
+            });
+            if (res.status === 429 || res.status >= 500) {
+                lastError = new MarketHttpError(source, res.status, MARKET_ERROR_CODES.upstreamError, `${source} responded ${res.status}`);
+                continue;
+            }
+            markSourceHealth(source, 'ok');
+            return res;
+        }
+        catch (error) {
+            if (error instanceof MarketUpstreamError) {
+                lastError = error;
+                continue;
+            }
+            const isAbort = error instanceof Error && error.name === 'AbortError';
+            lastError = new MarketUpstreamError(source, isAbort ? MARKET_ERROR_CODES.upstreamTimeout : MARKET_ERROR_CODES.upstreamError, isAbort
+                ? `${source} request timed out`
+                : `${source} request failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    const finalError = lastError ?? errorFrom(source, `${source} request failed`);
+    markSourceHealth(source, 'failed', finalError.message);
+    throw finalError;
+}
+/** `providerFetch` plus JSON decoding and non-2xx/parse-failure classification. */
+export async function providerFetchJson(source, url, init) {
+    const res = await providerFetch(source, url, init);
+    if (!res.ok) {
+        const error = new MarketHttpError(source, res.status, MARKET_ERROR_CODES.upstreamError, `${source} responded ${res.status} for ${safePathname(url)}`);
+        markSourceHealth(source, 'failed', error.message);
+        throw error;
+    }
+    // A JSON body is capped exactly like a file body: `res.text()` would buffer
+    // whatever the upstream (or an attacker in front of it) sends, and a chunked
+    // response without `content-length` can be arbitrarily large.
+    let text;
+    try {
+        ({ content: text } = await readResponseTextWithLimit(source, res, MARKET_LIMITS.maxTotalSize, 'JSON response body'));
+    }
+    catch (error) {
+        const failure = error instanceof MarketUpstreamError
+            ? error
+            : new MarketUpstreamError(source, MARKET_ERROR_CODES.upstreamBadResponse, `${source} response could not be read`);
+        markSourceHealth(source, 'failed', failure.message);
+        throw failure;
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        const error = new MarketUpstreamError(source, MARKET_ERROR_CODES.upstreamBadResponse, `${source} returned invalid JSON`);
+        markSourceHealth(source, 'failed', error.message);
+        throw error;
+    }
+}
+/** `new URL()` throws on relative input; error messages must never mask the real cause. */
+function safePathname(url) {
+    try {
+        return new URL(url).pathname;
+    }
+    catch {
+        return url;
+    }
+}
+function responseTooLarge(source, label, maxBytes) {
+    return new MarketUpstreamError(source, MARKET_ERROR_CODES.upstreamBadResponse, `${source} ${label} exceeds the actual size limit (${maxBytes} bytes)`);
+}
+/**
+ * Read a response body as text, aborting as soon as it exceeds `maxBytes`.
+ *
+ * `content-length` is only a hint (redirects and chunked responses omit it), so
+ * the body is streamed and counted rather than trusted.
+ */
+export async function readResponseTextWithLimit(source, response, maxBytes, label) {
+    const contentLength = Number(response.headers.get('content-length'));
+    if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+        throw responseTooLarge(source, label, maxBytes);
+    }
+    if (!response.body)
+        return { content: '', size: 0 };
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    const chunks = [];
+    let total = 0;
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (value) {
+                total += value.byteLength;
+                if (total > maxBytes) {
+                    await reader.cancel().catch(() => { });
+                    throw responseTooLarge(source, label, maxBytes);
+                }
+                chunks.push(decoder.decode(value, { stream: true }));
+            }
+        }
+        chunks.push(decoder.decode());
+        return { content: chunks.join(''), size: total };
+    }
+    finally {
+        reader.releaseLock();
+    }
+}