miladyai 2.0.0-alpha.27

package/dist/services/skill-marketplace.js

@@ -0,0 +1,521 @@
+import { createIntegrationTelemetrySpan } from "../diagnostics/integration-observability.js";
+import os from "node:os";
+import path from "node:path";
+import { logger } from "@elizaos/core";
+import { execFile } from "node:child_process";
+import fs from "node:fs/promises";
+import { promisify } from "node:util";
+
+//#region src/services/skill-marketplace.ts
+const execFileAsync = promisify(execFile);
+const DEFAULT_SKILLS_MARKETPLACE_URL = "https://clawhub.ai";
+const LEGACY_SKILLSMP_HOST = "skillsmp.com";
+const VALID_NAME = /^[a-zA-Z0-9._-]+$/;
+const VALID_GIT_REF = /^[a-zA-Z0-9][\w./-]*$/;
+/** Timeout for git clone/sparse-checkout (shallow + sparse should be fast). */
+const GIT_TIMEOUT_MS = 15e3;
+/** Timeout for marketplace API fetch calls. */
+const FETCH_TIMEOUT_MS = 3e4;
+/**
+* Run a security scan on a skill directory.
+*
+* Checks for binary files, symlink escapes, and missing SKILL.md.
+* This is a self-contained manifest check — the full content-level scan
+* (code + markdown patterns) is handled by the AgentSkillsService when
+* it loads the skill. This layer catches the most dangerous structural
+* attacks at the marketplace install boundary.
+*/
+async function runSkillSecurityScan(skillDir) {
+	const fsPromises = await import("node:fs/promises");
+	const pathMod = await import("node:path");
+	const findings = [];
+	const manifestFindings = [];
+	let scannedFiles = 0;
+	const BINARY_EXTENSIONS = new Set([
+		".exe",
+		".dll",
+		".so",
+		".dylib",
+		".wasm",
+		".bin",
+		".com",
+		".bat",
+		".cmd"
+	]);
+	async function walk(dir) {
+		const entries = await fsPromises.readdir(dir, { withFileTypes: true });
+		for (const entry of entries) {
+			if (entry.name === "node_modules") continue;
+			const fullPath = pathMod.join(dir, entry.name);
+			const relPath = pathMod.relative(skillDir, fullPath);
+			if (entry.isDirectory()) await walk(fullPath);
+			else if (entry.isFile()) {
+				scannedFiles++;
+				const ext = pathMod.extname(entry.name).toLowerCase();
+				if (BINARY_EXTENSIONS.has(ext)) manifestFindings.push({
+					ruleId: "binary-file",
+					severity: "critical",
+					file: relPath,
+					message: `Binary executable file detected (${ext})`
+				});
+			} else if (entry.isSymbolicLink()) {
+				const resolved = await fsPromises.realpath(fullPath).catch(() => "");
+				if (resolved && !resolved.startsWith(skillDir + pathMod.sep)) manifestFindings.push({
+					ruleId: "symlink-escape",
+					severity: "critical",
+					file: relPath,
+					message: `Symbolic link points outside skill directory`
+				});
+			}
+		}
+	}
+	await walk(skillDir);
+	const skillMdPath = pathMod.join(skillDir, "SKILL.md");
+	if (!await fsPromises.stat(skillMdPath).then((s) => s.isFile()).catch(() => false)) manifestFindings.push({
+		ruleId: "missing-skill-md",
+		severity: "critical",
+		file: "SKILL.md",
+		message: "No SKILL.md file found — invalid skill package"
+	});
+	const hasBlocking = manifestFindings.some((f) => f.ruleId === "binary-file" || f.ruleId === "symlink-escape" || f.ruleId === "missing-skill-md");
+	const critical = manifestFindings.filter((f) => f.severity === "critical").length;
+	const warn = manifestFindings.filter((f) => f.severity === "warn").length;
+	let status = "clean";
+	if (hasBlocking) status = "blocked";
+	else if (critical > 0) status = "critical";
+	else if (warn > 0) status = "warning";
+	const report = {
+		scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		status,
+		summary: {
+			scannedFiles,
+			critical,
+			warn,
+			info: 0
+		},
+		findings,
+		manifestFindings,
+		skillPath: skillDir
+	};
+	await fsPromises.writeFile(pathMod.join(skillDir, ".scan-results.json"), JSON.stringify(report, null, 2), "utf-8");
+	return report;
+}
+function stateDirBase() {
+	return process.env.MILADY_STATE_DIR?.trim() || path.join(os.homedir(), ".milady");
+}
+function safeName(raw) {
+	const slug = raw.trim().replace(/[^a-zA-Z0-9._-]/g, "-").replace(/-+/g, "-").replace(/^-+|-+$/g, "");
+	if (!slug) throw new Error("Invalid skill name");
+	if (!VALID_NAME.test(slug)) throw new Error(`Invalid skill name: ${raw}`);
+	return slug;
+}
+function validateGitRef(ref) {
+	if (!ref || !VALID_GIT_REF.test(ref)) throw new Error("Invalid git ref");
+}
+function sanitizeSkillPath(raw) {
+	const trimmed = raw.trim();
+	if (!trimmed) throw new Error("Invalid skill path");
+	if (trimmed.startsWith("~")) throw new Error("Invalid skill path");
+	if (path.posix.isAbsolute(trimmed) || path.win32.isAbsolute(trimmed)) throw new Error("Invalid skill path");
+	if (trimmed.includes("\\")) throw new Error("Invalid skill path");
+	const cleaned = trimmed.replace(/^\/+/, "");
+	if (!cleaned) throw new Error("Invalid skill path");
+	if (path.posix.isAbsolute(cleaned) || path.win32.isAbsolute(cleaned)) throw new Error("Invalid skill path");
+	if (cleaned === ".") return ".";
+	const parts = cleaned.split("/").filter(Boolean);
+	if (parts.length === 0) throw new Error("Invalid skill path");
+	if (parts.some((p) => p === "." || p === "..")) throw new Error("Invalid skill path");
+	return parts.join("/");
+}
+function assertPathWithinRoot(rootDir, targetPath) {
+	const root = path.resolve(rootDir);
+	const target = path.resolve(targetPath);
+	if (target === root) return;
+	if (!target.startsWith(`${root}${path.sep}`)) throw new Error("Skill path escapes repository root");
+}
+function normalizeRepo(raw) {
+	const repo = raw.replace(/^https:\/\/github\.com\//i, "").replace(/\.git$/i, "").replace(/^github:/i, "").trim();
+	if (!/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/.test(repo)) throw new Error(`Invalid repository: ${raw}`);
+	return repo;
+}
+function parseGithubUrl(rawUrl) {
+	let url;
+	try {
+		url = new URL(rawUrl);
+	} catch (err) {
+		throw new Error(`Invalid GitHub URL: ${err instanceof Error ? err.message : String(err)}`);
+	}
+	if (url.hostname !== "github.com") throw new Error("Only github.com URLs are supported for skill install");
+	const rawIndex = rawUrl.toLowerCase().indexOf("/tree/");
+	if (rawIndex !== -1) {
+		const rawPath = rawUrl.slice(rawIndex + 6).split(/[?#]/)[0];
+		let decoded = rawPath;
+		try {
+			decoded = decodeURIComponent(rawPath);
+		} catch {}
+		if (/(^|\/)\.\.(\/|$)/.test(decoded)) throw new Error("Invalid skill path");
+	}
+	const parts = url.pathname.split("/").filter(Boolean);
+	if (parts.length < 2) throw new Error("GitHub URL must include owner/repo");
+	const repository = normalizeRepo(`${parts[0]}/${parts[1]}`);
+	if (parts[2] === "tree" && parts.length >= 5) {
+		const ref = parts[3];
+		validateGitRef(ref);
+		const treePath = parts.slice(4).join("/");
+		return {
+			repository,
+			path: treePath ? sanitizeSkillPath(treePath) : null,
+			ref: ref || null
+		};
+	}
+	return {
+		repository,
+		path: null,
+		ref: null
+	};
+}
+function installationRoot(workspaceDir) {
+	return path.join(workspaceDir, "skills", ".marketplace");
+}
+function installsRecordPath(workspaceDir) {
+	return path.join(workspaceDir, "skills", ".cache", "marketplace-installs.json");
+}
+async function ensureInstallDirs(workspaceDir) {
+	await fs.mkdir(installationRoot(workspaceDir), { recursive: true });
+	await fs.mkdir(path.dirname(installsRecordPath(workspaceDir)), { recursive: true });
+}
+async function readInstallRecords(workspaceDir) {
+	try {
+		const raw = await fs.readFile(installsRecordPath(workspaceDir), "utf-8");
+		const parsed = JSON.parse(raw);
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+		return parsed;
+	} catch {
+		return {};
+	}
+}
+async function writeInstallRecords(workspaceDir, records) {
+	await ensureInstallDirs(workspaceDir);
+	await fs.writeFile(installsRecordPath(workspaceDir), JSON.stringify(records, null, 2), "utf-8");
+}
+function normalizeTags(raw) {
+	if (Array.isArray(raw)) return raw.map((t) => String(t ?? "").trim()).filter((t) => t.length > 0).slice(0, 10);
+	if (raw && typeof raw === "object") return Object.keys(raw).map((t) => t.trim()).filter((t) => t.length > 0).slice(0, 10);
+	return [];
+}
+function inferRepository(skill) {
+	const candidates = [
+		skill.repository,
+		skill.repo,
+		skill.gitRepo,
+		skill.github,
+		skill.githubRepo,
+		skill.git?.repo
+	];
+	for (const value of candidates) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		try {
+			return normalizeRepo(value);
+		} catch (err) {
+			logger.debug(`[skill-marketplace] Failed to normalize repo: ${err instanceof Error ? err.message : err}`);
+		}
+	}
+	const githubUrl = skill.githubUrl;
+	if (typeof githubUrl === "string" && githubUrl.includes("github.com")) try {
+		const parts = new URL(githubUrl).pathname.split("/").filter(Boolean);
+		if (parts.length >= 2) return normalizeRepo(`${parts[0]}/${parts[1]}`);
+	} catch (err) {
+		logger.debug(`[skill-marketplace] Failed to normalize repo: ${err instanceof Error ? err.message : err}`);
+	}
+	return null;
+}
+function inferPath(skill) {
+	const candidates = [
+		skill.path,
+		skill.skillPath,
+		skill.installPath,
+		skill.directory
+	];
+	for (const value of candidates) {
+		if (typeof value !== "string") continue;
+		const cleaned = value.replace(/^\/+/, "").trim();
+		if (cleaned && !cleaned.startsWith("..") && !cleaned.includes("/..")) return cleaned;
+	}
+	const githubUrl = skill.githubUrl;
+	if (typeof githubUrl === "string" && githubUrl.includes("/tree/")) {
+		const treeIndex = githubUrl.indexOf("/tree/");
+		const afterTree = githubUrl.slice(treeIndex + 6);
+		const slashIndex = afterTree.indexOf("/");
+		if (slashIndex !== -1) {
+			const pathPart = afterTree.slice(slashIndex + 1);
+			if (pathPart && !pathPart.startsWith("..") && !pathPart.includes("/..")) return pathPart;
+		}
+	}
+	return null;
+}
+function inferName(skill, fallbackId) {
+	const candidates = [
+		skill.displayName,
+		skill.slug,
+		skill.name,
+		skill.id,
+		skill.title
+	];
+	for (const value of candidates) {
+		if (typeof value !== "string") continue;
+		const cleaned = value.trim();
+		if (cleaned) return cleaned;
+	}
+	if (fallbackId.includes("/")) return fallbackId.split("/").pop() || fallbackId;
+	return fallbackId;
+}
+function inferDescription(skill) {
+	const candidates = [
+		skill.description,
+		skill.summary,
+		skill.shortDescription
+	];
+	for (const value of candidates) if (typeof value === "string" && value.trim()) return value.trim();
+	return "";
+}
+function resolveMarketplaceBaseUrl() {
+	return process.env.SKILLS_REGISTRY?.trim() || process.env.CLAWHUB_REGISTRY?.trim() || process.env.SKILLS_MARKETPLACE_URL?.trim() || DEFAULT_SKILLS_MARKETPLACE_URL;
+}
+function isLegacySkillsmp(baseUrl) {
+	try {
+		const hostname = new URL(baseUrl).hostname.toLowerCase();
+		return hostname === LEGACY_SKILLSMP_HOST || hostname.endsWith(".skillsmp.com");
+	} catch {
+		return baseUrl.toLowerCase().includes(LEGACY_SKILLSMP_HOST);
+	}
+}
+async function searchSkillsMarketplace(query, opts) {
+	const baseUrl = resolveMarketplaceBaseUrl();
+	const legacySkillsmp = isLegacySkillsmp(baseUrl);
+	const endpoint = legacySkillsmp ? opts?.aiSearch ? "/api/v1/skills/ai-search" : "/api/v1/skills/search" : "/api/v1/search";
+	const url = new URL(`${baseUrl}${endpoint}`);
+	if (query.trim()) url.searchParams.set("q", query.trim());
+	url.searchParams.set("limit", String(Math.max(1, Math.min(opts?.limit ?? 20, 50))));
+	const headers = { Accept: "application/json" };
+	if (legacySkillsmp) {
+		const apiKey = process.env.SKILLSMP_API_KEY?.trim();
+		if (!apiKey) throw new Error("SKILLSMP_API_KEY is not set. Add it to enable Skills marketplace search.");
+		headers.Authorization = `Bearer ${apiKey}`;
+	}
+	const searchSpan = createIntegrationTelemetrySpan({
+		boundary: "marketplace",
+		operation: "search_skills_marketplace",
+		timeoutMs: FETCH_TIMEOUT_MS
+	});
+	let resp;
+	try {
+		resp = await fetch(url, {
+			headers,
+			signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
+		});
+	} catch (err) {
+		searchSpan.failure({ error: err });
+		const msg = err instanceof Error ? err.message : String(err);
+		throw new Error(msg.includes("aborted") || msg.includes("timeout") ? `Skills marketplace request timed out after ${FETCH_TIMEOUT_MS / 1e3}s` : `Skills marketplace network error: ${msg}`);
+	}
+	const payload = await resp.json().catch(() => ({}));
+	if (!resp.ok) {
+		searchSpan.failure({
+			statusCode: resp.status,
+			errorKind: "http_error"
+		});
+		const msg = payload.error?.message;
+		throw new Error(typeof msg === "string" && msg ? msg : `Skills marketplace request failed (${resp.status})`);
+	}
+	const buckets = [
+		payload.results,
+		payload.skills,
+		payload.data
+	];
+	let list = [];
+	for (const bucket of buckets) {
+		if (Array.isArray(bucket)) {
+			list = bucket;
+			break;
+		}
+		if (bucket && typeof bucket === "object" && Array.isArray(bucket.results)) {
+			list = bucket.results;
+			break;
+		}
+		if (bucket && typeof bucket === "object" && Array.isArray(bucket.skills)) {
+			list = bucket.skills;
+			break;
+		}
+	}
+	const out = [];
+	for (const entry of list) {
+		if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
+		const skill = entry;
+		const slug = typeof skill.slug === "string" ? skill.slug.trim() : "";
+		const repository = inferRepository(skill);
+		if (!repository && !slug) continue;
+		const name = inferName(skill, repository || slug);
+		const description = inferDescription(skill);
+		const skillPath = inferPath(skill);
+		const scoreValue = skill.score;
+		const score = typeof scoreValue === "number" && Number.isFinite(scoreValue) ? scoreValue : null;
+		const githubUrl = typeof skill.githubUrl === "string" && skill.githubUrl.trim() ? skill.githubUrl.trim() : repository ? `https://github.com/${repository}` : void 0;
+		out.push({
+			id: String(skill.id ?? slug ?? name),
+			slug: slug || void 0,
+			name,
+			description,
+			repository: repository || void 0,
+			githubUrl,
+			path: skillPath,
+			tags: normalizeTags(skill.tags ?? skill.topics),
+			score,
+			source: legacySkillsmp ? "skillsmp" : "clawhub"
+		});
+	}
+	searchSpan.success({ statusCode: resp.status });
+	return out;
+}
+async function runGitCloneSubset(repository, ref, skillPath, targetDir) {
+	validateGitRef(ref);
+	if (skillPath !== ".") sanitizeSkillPath(skillPath);
+	await withTemporarySparseCheckout(repository, ref, skillPath, async (cloneDir) => {
+		const sourceDir = path.join(cloneDir, skillPath);
+		assertPathWithinRoot(cloneDir, sourceDir);
+		const stat = await fs.stat(sourceDir).catch(() => null);
+		if (!stat || !stat.isDirectory()) throw new Error(`Skill path not found in repository: ${skillPath}`);
+		await fs.mkdir(path.dirname(targetDir), { recursive: true });
+		await fs.cp(sourceDir, targetDir, {
+			recursive: true,
+			errorOnExist: true,
+			force: false
+		});
+	});
+}
+async function resolveSkillPathInRepo(repository, ref, requestedPath) {
+	validateGitRef(ref);
+	if (requestedPath) return sanitizeSkillPath(requestedPath);
+	return withTemporarySparseCheckout(repository, ref, ".", async (cloneDir) => {
+		const rootSkill = path.join(cloneDir, "SKILL.md");
+		if (await fs.stat(rootSkill).then((s) => s.isFile()).catch(() => false)) return ".";
+		const skillsDir = path.join(cloneDir, "skills");
+		const entries = await fs.readdir(skillsDir, { withFileTypes: true }).catch(() => []);
+		for (const entry of entries) {
+			if (!entry.isDirectory()) continue;
+			const candidate = path.join(skillsDir, entry.name, "SKILL.md");
+			if (await fs.stat(candidate).then((s) => s.isFile()).catch(() => false)) return path.posix.join("skills", entry.name);
+		}
+		throw new Error("Could not determine skill path automatically. Provide an explicit GitHub tree URL or path.");
+	});
+}
+async function withTemporarySparseCheckout(repository, ref, checkoutPath, task) {
+	const repoUrl = `https://github.com/${repository}.git`;
+	const tmpBase = await fs.mkdtemp(path.join(stateDirBase(), "skill-probe-"));
+	const cloneDir = path.join(tmpBase, "repo");
+	try {
+		await execFileAsync("git", [
+			"clone",
+			"--depth",
+			"1",
+			"--filter=blob:none",
+			"--sparse",
+			"--branch",
+			ref,
+			repoUrl,
+			cloneDir
+		], { timeout: GIT_TIMEOUT_MS });
+		await execFileAsync("git", [
+			"-C",
+			cloneDir,
+			"sparse-checkout",
+			"set",
+			checkoutPath
+		], { timeout: GIT_TIMEOUT_MS });
+		return await task(cloneDir);
+	} finally {
+		await fs.rm(tmpBase, {
+			recursive: true,
+			force: true
+		}).catch(() => void 0);
+	}
+}
+async function installMarketplaceSkill(workspaceDir, input) {
+	await ensureInstallDirs(workspaceDir);
+	let repository = input.repository?.trim() ? normalizeRepo(input.repository) : null;
+	let requestedPath = input.path?.trim() ? sanitizeSkillPath(input.path) : null;
+	let gitRef = "main";
+	if (input.githubUrl?.trim()) {
+		const parsed = parseGithubUrl(input.githubUrl.trim());
+		repository = parsed.repository;
+		if (!requestedPath && parsed.path) requestedPath = parsed.path;
+		if (parsed.ref) gitRef = parsed.ref;
+	}
+	if (!repository) throw new Error("Install requires a repository or GitHub URL");
+	const skillPath = await resolveSkillPathInRepo(repository, gitRef, requestedPath);
+	const id = safeName(input.name?.trim() || path.posix.basename(skillPath === "." ? repository.split("/")[1] : skillPath));
+	const targetDir = path.join(installationRoot(workspaceDir), id);
+	if (await fs.stat(targetDir).then(() => true).catch(() => false)) throw new Error(`Skill "${id}" is already installed`);
+	await runGitCloneSubset(repository, gitRef, skillPath, targetDir);
+	const skillDoc = path.join(targetDir, "SKILL.md");
+	if (!await fs.stat(skillDoc).then((s) => s.isFile()).catch(() => false)) {
+		await fs.rm(targetDir, {
+			recursive: true,
+			force: true
+		}).catch(() => void 0);
+		throw new Error("Installed path does not contain SKILL.md");
+	}
+	const scanReport = await runSkillSecurityScan(targetDir);
+	const scanStatus = scanReport.status;
+	if (scanReport.status === "blocked") {
+		await fs.rm(targetDir, {
+			recursive: true,
+			force: true
+		}).catch(() => void 0);
+		const reasons = [...scanReport.findings.map((f) => f.message), ...scanReport.manifestFindings.map((f) => f.message)];
+		throw new Error(`Skill "${id}" blocked by security scan: ${reasons.join("; ")}`);
+	}
+	if (scanReport.status === "critical" || scanReport.status === "warning") logger.warn(`[skills-marketplace] Security scan for "${id}": ${scanReport.status} (${scanReport.summary.critical} critical, ${scanReport.summary.warn} warnings)`);
+	const record = {
+		id,
+		name: input.name?.trim() || id,
+		description: input.description?.trim() || "",
+		repository,
+		githubUrl: `https://github.com/${repository}`,
+		path: skillPath,
+		installPath: targetDir,
+		installedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		source: input.source ?? "manual",
+		scanStatus
+	};
+	const records = await readInstallRecords(workspaceDir);
+	records[id] = record;
+	await writeInstallRecords(workspaceDir, records);
+	logger.info(`[skills-marketplace] Installed ${record.id} from ${record.repository}:${record.path} (scan: ${scanStatus ?? "skipped"})`);
+	return record;
+}
+async function listInstalledMarketplaceSkills(workspaceDir) {
+	const records = await readInstallRecords(workspaceDir);
+	const values = Object.values(records);
+	values.sort((a, b) => b.installedAt.localeCompare(a.installedAt));
+	return values;
+}
+async function uninstallMarketplaceSkill(workspaceDir, skillId) {
+	const id = safeName(skillId);
+	const records = await readInstallRecords(workspaceDir);
+	const existing = records[id];
+	if (!existing) throw new Error(`Installed marketplace skill "${id}" not found`);
+	const expectedRoot = path.resolve(installationRoot(workspaceDir));
+	const resolvedPath = path.resolve(existing.installPath);
+	if (!resolvedPath.startsWith(`${expectedRoot}${path.sep}`) || resolvedPath === expectedRoot) throw new Error(`Refusing to remove skill outside ${expectedRoot}`);
+	await fs.rm(existing.installPath, {
+		recursive: true,
+		force: true
+	});
+	delete records[id];
+	await writeInstallRecords(workspaceDir, records);
+	logger.info(`[skills-marketplace] Uninstalled ${id}`);
+	return existing;
+}
+
+//#endregion
+export { installMarketplaceSkill, listInstalledMarketplaceSkills, searchSkillsMarketplace, uninstallMarketplaceSkill };