@thevinci/web 1.0.0

package/server/lib/skills-catalog/clawdhub/api.js

@@ -0,0 +1,158 @@
+/**
+ * ClawdHub API client
+ * 
+ * ClawdHub is a public skill registry at https://clawdhub.com
+ * This client provides methods to fetch skills list and download skill packages.
+ */
+
+const CLAWDHUB_API_BASE = 'https://clawdhub.com/api/v1';
+const CLAWDHUB_PAGE_LIMIT = 25;
+
+// Rate limiting: ClawdHub allows 120 requests/minute
+const RATE_LIMIT_DELAY_MS = 100;
+let lastRequestTime = 0;
+
+async function rateLimitedFetch(url, options = {}) {
+  const maxAttempts = 10;
+
+  let lastResponse = null;
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const now = Date.now();
+    const elapsed = now - lastRequestTime;
+    if (elapsed < RATE_LIMIT_DELAY_MS) {
+      await new Promise((resolve) => setTimeout(resolve, RATE_LIMIT_DELAY_MS - elapsed));
+    }
+    lastRequestTime = Date.now();
+
+    const response = await fetch(url, {
+      ...options,
+      headers: {
+        Accept: 'application/json',
+        'User-Agent': 'Vinci/1.0',
+        ...options.headers,
+      },
+    });
+
+    lastResponse = response;
+
+    if (response.status === 429 || response.status >= 500) {
+      if (attempt < maxAttempts - 1) {
+        const waitMs = 50 * (attempt + 1);
+        await new Promise((resolve) => setTimeout(resolve, waitMs));
+        continue;
+      }
+    }
+
+    return response;
+  }
+
+  return lastResponse;
+}
+
+/**
+ * Fetch paginated list of skills from ClawdHub
+ * @param {Object} options
+ * @param {string} [options.cursor] - Pagination cursor from previous response
+ * @returns {Promise<{ items: Array, nextCursor?: string }>}
+ */
+export async function fetchClawdHubSkills({ cursor } = {}) {
+  const url = cursor
+    ? `${CLAWDHUB_API_BASE}/skills?cursor=${encodeURIComponent(cursor)}&limit=${CLAWDHUB_PAGE_LIMIT}`
+    : `${CLAWDHUB_API_BASE}/skills?limit=${CLAWDHUB_PAGE_LIMIT}`;
+
+  const response = await rateLimitedFetch(url);
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => '');
+    throw new Error(`ClawdHub API error (${response.status}): ${text || response.statusText}`);
+  }
+
+  const data = await response.json();
+  const nextCursor =
+    (typeof data.nextCursor === 'string' && data.nextCursor) ||
+    (typeof data.next_cursor === 'string' && data.next_cursor) ||
+    (typeof data.next === 'string' && data.next) ||
+    (typeof data.cursor === 'string' && data.cursor) ||
+    null;
+
+  return {
+    items: data.items || [],
+    nextCursor,
+  };
+}
+
+/**
+ * Fetch details for a specific skill version
+ * @param {string} slug - Skill slug/identifier
+ * @param {string} [version='latest'] - Version string or 'latest'
+ * @returns {Promise<{ skill: Object, version: Object }>}
+ */
+export async function fetchClawdHubSkillVersion(slug, version = 'latest') {
+  // For 'latest', we need to first get the skill metadata to find the latest version
+  if (version === 'latest') {
+    const skillResponse = await rateLimitedFetch(`${CLAWDHUB_API_BASE}/skills/${encodeURIComponent(slug)}`);
+    if (!skillResponse.ok) {
+      throw new Error(`ClawdHub skill not found: ${slug}`);
+    }
+    const skillData = await skillResponse.json();
+    const latestVersion = skillData.skill?.tags?.latest || skillData.latestVersion?.version;
+    if (!latestVersion) {
+      throw new Error(`No latest version found for skill: ${slug}`);
+    }
+    version = latestVersion;
+  }
+
+  const url = `${CLAWDHUB_API_BASE}/skills/${encodeURIComponent(slug)}/versions/${encodeURIComponent(version)}`;
+  const response = await rateLimitedFetch(url);
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => '');
+    throw new Error(`ClawdHub version error (${response.status}): ${text || response.statusText}`);
+  }
+
+  return response.json();
+}
+
+/**
+ * Download a skill package as a ZIP buffer
+ * @param {string} slug - Skill slug/identifier
+ * @param {string} version - Specific version string
+ * @returns {Promise<ArrayBuffer>} - ZIP file contents
+ */
+export async function downloadClawdHubSkill(slug, version) {
+  const versionParam = typeof version === 'string' && version !== 'latest'
+    ? `&version=${encodeURIComponent(version)}`
+    : '&tag=latest';
+  const url = `${CLAWDHUB_API_BASE}/download?slug=${encodeURIComponent(slug)}${versionParam}`;
+
+  const response = await rateLimitedFetch(url, {
+    headers: {
+      Accept: 'application/zip',
+    },
+  });
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => '');
+    throw new Error(`ClawdHub download error (${response.status}): ${text || response.statusText}`);
+  }
+
+  return response.arrayBuffer();
+}
+
+/**
+ * Get skill metadata without version details
+ * @param {string} slug - Skill slug/identifier
+ * @returns {Promise<Object>}
+ */
+export async function fetchClawdHubSkillInfo(slug) {
+  const url = `${CLAWDHUB_API_BASE}/skills/${encodeURIComponent(slug)}`;
+  const response = await rateLimitedFetch(url);
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => '');
+    throw new Error(`ClawdHub skill error (${response.status}): ${text || response.statusText}`);
+  }
+
+  return response.json();
+}

package/server/lib/skills-catalog/clawdhub/index.js

@@ -0,0 +1,30 @@
+/**
+ * ClawdHub integration module
+ * 
+ * Provides skill browsing and installation from the ClawdHub registry.
+ * https://clawdhub.com
+ */
+
+export { scanClawdHub, scanClawdHubPage } from './scan.js';
+export { installSkillsFromClawdHub } from './install.js';
+export {
+  fetchClawdHubSkills,
+  fetchClawdHubSkillVersion,
+  fetchClawdHubSkillInfo,
+  downloadClawdHubSkill,
+} from './api.js';
+
+/**
+ * Check if a source string refers to ClawdHub
+ * @param {string} source
+ * @returns {boolean}
+ */
+export function isClawdHubSource(source) {
+  return typeof source === 'string' && source.startsWith('clawdhub:');
+}
+
+/**
+ * ClawdHub source identifier used in curated sources
+ */
+export const CLAWDHUB_SOURCE_ID = 'clawdhub';
+export const CLAWDHUB_SOURCE_STRING = 'clawdhub:registry';

package/server/lib/skills-catalog/clawdhub/install.js

@@ -0,0 +1,238 @@
+/**
+ * ClawdHub skill installation
+ * 
+ * Downloads skills from ClawdHub as ZIP files and extracts them
+ * to the appropriate skill directory.
+ */
+
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import AdmZip from 'adm-zip';
+
+import { downloadClawdHubSkill, fetchClawdHubSkillInfo } from './api.js';
+
+const SKILL_NAME_PATTERN = /^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$/;
+
+function normalizeUserSkillDir(userSkillDir) {
+  if (!userSkillDir) return null;
+  const legacySkillDir = path.join(os.homedir(), '.vinci', '.opencode', 'skill');
+  const pluralSkillDir = path.join(os.homedir(), '.vinci', '.opencode', 'skills');
+  if (userSkillDir === legacySkillDir) {
+    if (fs.existsSync(legacySkillDir) && !fs.existsSync(pluralSkillDir)) return legacySkillDir;
+    return pluralSkillDir;
+  }
+  return userSkillDir;
+}
+
+function validateSkillName(skillName) {
+  if (typeof skillName !== 'string') return false;
+  if (skillName.length < 1 || skillName.length > 64) return false;
+  return SKILL_NAME_PATTERN.test(skillName);
+}
+
+async function safeRm(dir) {
+  try {
+    await fs.promises.rm(dir, { recursive: true, force: true });
+  } catch {
+    // ignore
+  }
+}
+
+async function ensureDir(dirPath) {
+  await fs.promises.mkdir(dirPath, { recursive: true });
+}
+
+function getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName }) {
+  const source = targetSource === 'agents' ? 'agents' : 'opencode';
+
+  if (scope === 'user') {
+    if (source === 'agents') {
+      return path.join(os.homedir(), '.agents', 'skills', skillName);
+    }
+    return path.join(userSkillDir, skillName);
+  }
+
+  if (!workingDirectory) {
+    throw new Error('workingDirectory is required for project installs');
+  }
+
+  if (source === 'agents') {
+    return path.join(workingDirectory, '.agents', 'skills', skillName);
+  }
+
+  return path.join(workingDirectory, '.opencode', 'skills', skillName);
+}
+
+/**
+ * Install skills from ClawdHub registry
+ * @param {Object} options
+ * @param {string} options.scope - 'user' or 'project'
+ * @param {string} [options.targetSource] - 'opencode' or 'agents'
+ * @param {string} [options.workingDirectory] - Required for project scope
+ * @param {string} options.userSkillDir - User skills directory
+ * @param {Array} options.selections - Array of { skillDir, clawdhub: { slug, version } }
+ * @param {string} [options.conflictPolicy] - 'prompt', 'skipAll', or 'overwriteAll'
+ * @param {Object} [options.conflictDecisions] - Per-skill conflict decisions
+ * @returns {Promise<{ ok: boolean, installed?: Array, skipped?: Array, error?: Object }>}
+ */
+export async function installSkillsFromClawdHub({
+  scope,
+  targetSource,
+  workingDirectory,
+  userSkillDir,
+  selections,
+  conflictPolicy,
+  conflictDecisions,
+} = {}) {
+  if (scope !== 'user' && scope !== 'project') {
+    return { ok: false, error: { kind: 'invalidSource', message: 'Invalid scope' } };
+  }
+
+  if (targetSource !== undefined && targetSource !== 'opencode' && targetSource !== 'agents') {
+    return { ok: false, error: { kind: 'invalidSource', message: 'Invalid target source' } };
+  }
+
+  if (!userSkillDir) {
+    return { ok: false, error: { kind: 'unknown', message: 'userSkillDir is required' } };
+  }
+
+  const normalizedUserSkillDir = normalizeUserSkillDir(userSkillDir);
+  if (normalizedUserSkillDir) {
+    userSkillDir = normalizedUserSkillDir;
+  }
+
+  if (scope === 'project' && !workingDirectory) {
+    return { ok: false, error: { kind: 'invalidSource', message: 'Project installs require a directory parameter' } };
+  }
+
+  const requestedSkills = Array.isArray(selections) ? selections : [];
+  if (requestedSkills.length === 0) {
+    return { ok: false, error: { kind: 'invalidSource', message: 'No skills selected for installation' } };
+  }
+
+  // Build installation plans
+  const skillPlans = requestedSkills.map((sel) => {
+    const slug = sel.clawdhub?.slug || sel.skillDir;
+    const version = sel.clawdhub?.version || 'latest';
+    return {
+      slug,
+      version,
+      installable: validateSkillName(slug),
+    };
+  });
+
+  // Check for conflicts before downloading
+  const conflicts = [];
+  for (const plan of skillPlans) {
+    if (!plan.installable) {
+      continue;
+    }
+
+    const targetDir = getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName: plan.slug });
+    if (fs.existsSync(targetDir)) {
+      const decision = conflictDecisions?.[plan.slug];
+      const hasAutoPolicy = conflictPolicy === 'skipAll' || conflictPolicy === 'overwriteAll';
+      if (!decision && !hasAutoPolicy) {
+        conflicts.push({ skillName: plan.slug, scope, source: targetSource === 'agents' ? 'agents' : 'opencode' });
+      }
+    }
+  }
+
+  if (conflicts.length > 0) {
+    return {
+      ok: false,
+      error: {
+        kind: 'conflicts',
+        message: 'Some skills already exist in the selected scope',
+        conflicts,
+      },
+    };
+  }
+
+  const installed = [];
+  const skipped = [];
+
+  for (const plan of skillPlans) {
+    if (!plan.installable) {
+      skipped.push({ skillName: plan.slug, reason: 'Invalid skill name' });
+      continue;
+    }
+
+    try {
+      // Resolve 'latest' version if needed
+      let resolvedVersion = plan.version;
+      if (resolvedVersion === 'latest') {
+        try {
+          const info = await fetchClawdHubSkillInfo(plan.slug);
+          const latest = info.skill?.tags?.latest || info.latestVersion?.version || null;
+          if (latest) {
+            resolvedVersion = latest;
+          }
+        } catch {
+          // ignore
+        }
+
+        if (resolvedVersion === 'latest') {
+          skipped.push({ skillName: plan.slug, reason: 'Unable to resolve latest version' });
+          continue;
+        }
+      }
+
+      const targetDir = getTargetSkillDir({ scope, targetSource, workingDirectory, userSkillDir, skillName: plan.slug });
+      const exists = fs.existsSync(targetDir);
+
+      // Determine conflict resolution
+      let decision = conflictDecisions?.[plan.slug] || null;
+      if (!decision) {
+        if (exists && conflictPolicy === 'skipAll') decision = 'skip';
+        if (exists && conflictPolicy === 'overwriteAll') decision = 'overwrite';
+        if (!exists) decision = 'overwrite'; // No conflict, proceed
+      }
+
+      if (exists && decision === 'skip') {
+        skipped.push({ skillName: plan.slug, reason: 'Already installed (skipped)' });
+        continue;
+      }
+
+      if (exists && decision === 'overwrite') {
+        await safeRm(targetDir);
+      }
+
+      // Download the skill ZIP
+      const zipBuffer = await downloadClawdHubSkill(plan.slug, resolvedVersion);
+
+      // Extract to a temp directory first for validation
+      const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), `clawdhub-${plan.slug}-`));
+      
+      try {
+        const zip = new AdmZip(Buffer.from(zipBuffer));
+        zip.extractAllTo(tempDir, true);
+
+        // Verify SKILL.md exists
+        const skillMdPath = path.join(tempDir, 'SKILL.md');
+        if (!fs.existsSync(skillMdPath)) {
+          skipped.push({ skillName: plan.slug, reason: 'SKILL.md not found in downloaded package' });
+          continue;
+        }
+
+        // Move to target directory
+        await ensureDir(path.dirname(targetDir));
+        await fs.promises.rename(tempDir, targetDir);
+
+        installed.push({ skillName: plan.slug, scope, source: targetSource === 'agents' ? 'agents' : 'opencode' });
+      } catch (extractError) {
+        await safeRm(tempDir);
+        throw extractError;
+      }
+    } catch (error) {
+      console.error(`Failed to install ClawdHub skill "${plan.slug}":`, error);
+      skipped.push({
+        skillName: plan.slug,
+        reason: error instanceof Error ? error.message : 'Failed to download or extract skill',
+      });
+    }
+  }
+
+  return { ok: true, installed, skipped };
+}

package/server/lib/skills-catalog/clawdhub/scan.js

@@ -0,0 +1,113 @@
+/**
+ * ClawdHub skill scanning
+ * 
+ * Fetches all available skills from the ClawdHub registry
+ * and transforms them into SkillsCatalogItem format.
+ */
+
+import { fetchClawdHubSkills } from './api.js';
+
+const MAX_PAGES = 20; // Safety limit to prevent infinite loops
+const CLAWDHUB_PAGE_LIMIT = 25;
+
+const mapClawdHubItem = (item) => {
+  const latestVersion = item.tags?.latest || item.latestVersion?.version || '1.0.0';
+
+  return {
+    sourceId: 'clawdhub',
+    repoSource: 'clawdhub:registry',
+    repoSubpath: null,
+    gitIdentityId: null,
+    skillDir: item.slug,
+    skillName: item.slug,
+    frontmatterName: item.displayName || item.slug,
+    description: item.summary || null,
+    installable: true,
+    warnings: [],
+    // ClawdHub-specific metadata
+    clawdhub: {
+      slug: item.slug,
+      version: latestVersion,
+      displayName: item.displayName,
+      owner: item.owner?.handle || null,
+      downloads: item.stats?.downloads || 0,
+      stars: item.stats?.stars || 0,
+      versionsCount: item.stats?.versions || 1,
+      createdAt: item.createdAt,
+      updatedAt: item.updatedAt,
+    },
+  };
+};
+
+/**
+ * Scan ClawdHub registry for all available skills
+ * @returns {Promise<{ ok: boolean, items?: Array, error?: Object }>}
+ */
+export async function scanClawdHub() {
+  try {
+    const allItems = [];
+    let cursor = null;
+
+    for (let page = 0; page < MAX_PAGES; page++) {
+      let items = [];
+      let nextCursor = null;
+
+      try {
+        const pageResult = await fetchClawdHubSkills({ cursor });
+        items = pageResult.items || [];
+        nextCursor = pageResult.nextCursor || null;
+      } catch (error) {
+        if (page > 0 && allItems.length > 0) {
+          console.warn('ClawdHub pagination failed; returning partial results.');
+          break;
+        }
+        throw error;
+      }
+
+      for (const item of items) {
+        allItems.push(mapClawdHubItem(item));
+      }
+
+      if (!nextCursor) {
+        break;
+      }
+      cursor = nextCursor;
+    }
+
+    // Sort by downloads (most popular first)
+    allItems.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
+
+    return { ok: true, items: allItems };
+  } catch (error) {
+    console.error('ClawdHub scan error:', error);
+    return {
+      ok: false,
+      error: {
+        kind: 'networkError',
+        message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
+      },
+    };
+  }
+}
+
+/**
+ * Scan a single ClawdHub page (cursor-based)
+ * @returns {Promise<{ ok: boolean, items?: Array, nextCursor?: string | null, error?: Object }>}
+ */
+export async function scanClawdHubPage({ cursor } = {}) {
+  try {
+    const { items, nextCursor } = await fetchClawdHubSkills({ cursor });
+    const mapped = (items || []).map(mapClawdHubItem).slice(0, CLAWDHUB_PAGE_LIMIT);
+    mapped.sort((a, b) => (b.clawdhub?.downloads || 0) - (a.clawdhub?.downloads || 0));
+    return { ok: true, items: mapped, nextCursor: nextCursor || null };
+  } catch (error) {
+    console.error('ClawdHub page scan error:', error);
+    return {
+      ok: false,
+      error: {
+        kind: 'networkError',
+        message: error instanceof Error ? error.message : 'Failed to fetch skills from ClawdHub',
+      },
+    };
+  }
+}

package/server/lib/skills-catalog/curated-sources.js

@@ -0,0 +1,21 @@
+export const CURATED_SKILLS_SOURCES = [
+  {
+    id: 'anthropic',
+    label: 'Anthropic',
+    description: "Anthropic's public skills repository",
+    source: 'anthropics/skills',
+    defaultSubpath: 'skills',
+    sourceType: 'github',
+  },
+  {
+    id: 'clawdhub',
+    label: 'ClawdHub',
+    description: 'Community skill registry with vector search',
+    source: 'clawdhub:registry',
+    sourceType: 'clawdhub',
+  },
+];
+
+export function getCuratedSkillsSources() {
+  return CURATED_SKILLS_SOURCES.slice();
+}

package/server/lib/skills-catalog/git.js

@@ -0,0 +1,77 @@
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+
+const execFileAsync = promisify(execFile);
+
+const DEFAULT_TIMEOUT_MS = 60_000;
+const DEFAULT_MAX_BUFFER = 4 * 1024 * 1024;
+
+export function looksLikeAuthError(message) {
+  const text = String(message || '');
+  return (
+    /permission denied/i.test(text) ||
+    /publickey/i.test(text) ||
+    /could not read from remote repository/i.test(text) ||
+    /authentication failed/i.test(text) ||
+    /fatal: could not/i.test(text)
+  );
+}
+
+export async function runGit(args, options = {}) {
+  const cwd = options.cwd;
+  const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
+  const maxBuffer = Number.isFinite(options.maxBuffer) ? options.maxBuffer : DEFAULT_MAX_BUFFER;
+
+  const identity = options.identity || null;
+  const normalizedArgs = Array.isArray(args) ? args.slice() : [];
+
+  // Non-interactive git (avoid prompts / hangs)
+  const env = {
+    ...process.env,
+    GIT_TERMINAL_PROMPT: '0',
+  };
+
+  if (identity?.sshKey) {
+    const sshKeyPath = String(identity.sshKey).trim();
+    if (sshKeyPath) {
+      // Avoid interactive host key prompts; still safe against changed keys.
+      const sshCommand = `ssh -i ${sshKeyPath} -o BatchMode=yes -o StrictHostKeyChecking=accept-new`;
+      normalizedArgs.unshift(`core.sshCommand=${sshCommand}`);
+      normalizedArgs.unshift('-c');
+    }
+  }
+
+  try {
+    const { stdout, stderr } = await execFileAsync('git', normalizedArgs, {
+      cwd,
+      env,
+      windowsHide: true,
+      timeout: timeoutMs,
+      maxBuffer,
+    });
+
+    return { ok: true, stdout: stdout || '', stderr: stderr || '' };
+  } catch (error) {
+    const err = error;
+    const stdout = typeof err?.stdout === 'string' ? err.stdout : '';
+    const stderr = typeof err?.stderr === 'string' ? err.stderr : '';
+    const message = err instanceof Error ? err.message : String(err);
+
+    return {
+      ok: false,
+      stdout,
+      stderr,
+      message,
+      code: typeof err?.code === 'number' ? err.code : null,
+      signal: typeof err?.signal === 'string' ? err.signal : null,
+    };
+  }
+}
+
+export async function assertGitAvailable() {
+  const result = await runGit(['--version'], { timeoutMs: 5_000 });
+  if (!result.ok) {
+    return { ok: false, error: { kind: 'gitUnavailable', message: 'Git is not available in PATH' } };
+  }
+  return { ok: true };
+}

package/server/lib/skills-catalog/index.js

@@ -0,0 +1,42 @@
+/**
+ * Skills catalog module
+ *
+ * Provides skill scanning, installation, and caching from GitHub repositories and ClawdHub.
+ */
+
+export {
+  CURATED_SKILLS_SOURCES,
+  getCuratedSkillsSources,
+} from './curated-sources.js';
+
+export {
+  getCacheKey,
+  getCachedScan,
+  setCachedScan,
+  clearCache,
+} from './cache.js';
+
+export {
+  parseSkillRepoSource,
+} from './source.js';
+
+export {
+  scanSkillsRepository,
+} from './scan.js';
+
+export {
+  installSkillsFromRepository,
+} from './install.js';
+
+export {
+  scanClawdHub,
+  scanClawdHubPage,
+  installSkillsFromClawdHub,
+  fetchClawdHubSkills,
+  fetchClawdHubSkillVersion,
+  fetchClawdHubSkillInfo,
+  downloadClawdHubSkill,
+  isClawdHubSource,
+  CLAWDHUB_SOURCE_ID,
+  CLAWDHUB_SOURCE_STRING,
+} from './clawdhub/index.js';