huhaa-myskills 0.1.5

package/service/packages/scanner/src/index.mjs

@@ -0,0 +1,234 @@
+// @huhaa/scanner — multi-source skill aggregator.
+//
+// P1 wires hermes + claude-code adapters via the shared markdown-skill
+// scanner. Other adapters land in P4. The orchestrator loads sources.yaml
+// from ~/.config/huhaa-myskills/, dispatches to enabled adapters, and
+// returns a flat IR list.
+
+import fs from 'node:fs';
+import YAML from 'yaml';
+import { configFile } from '../../../bin/lib/paths.mjs';
+import { scanMarkdownSkills } from './adapters/markdown-skill.mjs';
+import { scanFileDocs } from './adapters/file-docs.mjs';
+import { scanMcpConfigs } from './adapters/mcp-config.mjs';
+import { scanHermesPlugins } from './adapters/hermes-plugin.mjs';
+import { expandRoots, expandTilde } from './utils.mjs';
+
+const ADAPTERS = {
+  hermes: async (cfg, limits) => scanMarkdownSkills({
+    source: 'hermes',
+    roots: cfg.roots || [],
+    fileGlob: '**/SKILL.md',
+    limits,
+  }),
+  'claude-code': async (cfg, limits) => scanMarkdownSkills({
+    source: 'claude-code',
+    roots: cfg.roots || [],
+    fileGlob: '**/SKILL.md',
+    limits,
+  }),
+  codex: async (cfg, limits) => scanFileDocs({
+    source: 'codex',
+    editor: 'Codex',
+    kind: 'instruction',
+    files: cfg.files || [],
+    roots: cfg.roots || [],
+    globs: normalizeGlobs(cfg, ['AGENTS.md']),
+    limits,
+  }),
+  cursor: async (cfg, limits) => scanFileDocs({
+    source: 'cursor',
+    editor: 'Cursor',
+    kind: 'instruction',
+    files: cfg.files || [],
+    roots: cfg.roots || [],
+    globs: normalizeGlobs(cfg, ['.cursorrules', '.cursor/rules/**/*.{md,mdc}']),
+    limits,
+  }),
+  'mcp-config': async (cfg, limits) => scanMcpConfigs({
+    source: 'mcp-config',
+    editor: 'MCP',
+    files: cfg.files || [],
+    limits,
+  }),
+  'hermes-plugin': async (cfg, limits) => scanHermesPlugins({
+    source: 'hermes-plugin',
+    editor: 'Hermes Agent',
+    roots: cfg.roots || [],
+    limits,
+  }),
+  'project-runbook': async (cfg, limits) => scanFileDocs({
+    source: 'project-runbook',
+    editor: 'Project Docs',
+    kind: 'runbook',
+    files: cfg.files || [],
+    roots: cfg.roots || [],
+    globs: normalizeGlobs(cfg, ['docs/RUNBOOK-*.md', 'AGENTS.md', 'CLAUDE.md', '.cursorrules', '.cursor/rules/**/*.{md,mdc}']),
+    limits,
+  }),
+  'skills': async (cfg, limits) => scanFileDocs({
+    source: 'skills',
+    editor: 'Skills Hub',
+    kind: 'skill',
+    files: cfg.files || [],
+    roots: cfg.roots || [],
+    globs: normalizeGlobs(cfg, ['*.md', '*/SKILL.md', '**/SKILL.md']),
+    limits,
+  }),
+  'mcp': async (cfg, limits) => scanFileDocs({
+    source: 'mcp',
+    editor: 'MCP Hub',
+    kind: 'mcp-tool',
+    files: cfg.files || [],
+    roots: cfg.roots || [],
+    globs: normalizeGlobs(cfg, ['*.md', '*.json', '*.yaml', '*.yml', '**/manifest.*']),
+    limits,
+  }),
+  'skill': async (cfg, limits) => scanFileDocs({
+    source: 'skill',
+    editor: 'Skill Hub',
+    kind: 'skill',
+    files: cfg.files || [],
+    roots: cfg.roots || [],
+    globs: normalizeGlobs(cfg, ['*.md', '*/SKILL.md', '**/SKILL.md']),
+    limits,
+  }),
+  // Future: obsidian
+};
+
+function normalizeGlobs(cfg, defaults) {
+  if (Array.isArray(cfg.globs)) return cfg.globs.filter(Boolean);
+  if (cfg.glob) return [cfg.glob].flat().filter(Boolean);
+  return defaults;
+}
+
+/**
+ * Run all enabled adapters and return a flat IR list.
+ * @returns {Promise<import('./types').SkillItem[]>}
+ */
+export async function scan() {
+  const cfg = loadConfig();
+  if (!cfg) return [];
+
+  const limits = {
+    maxFiles: cfg.limits?.maxFiles ?? 5000,
+    maxFileBytes: cfg.limits?.maxFileBytes ?? 1024 * 1024,
+  };
+
+  const all = [];
+  const stats = [];
+  for (const [name, src] of Object.entries(cfg.sources || {})) {
+    if (!src?.enabled) continue;
+    const fn = ADAPTERS[name];
+    if (!fn) continue; // unknown adapter (e.g. mcp-config in P1) — skip silently
+    try {
+      const { items, stats: s } = await fn(src, limits);
+      all.push(...items);
+      stats.push(s);
+    } catch (e) {
+      stats.push({ source: name, available: false, error: e.message });
+    }
+  }
+
+  const out = dedupeSemantic(all);
+
+  if (process.env.HUHAA_DEBUG) {
+    console.error('[scan] stats:', JSON.stringify(stats, null, 2));
+  }
+  return out;
+}
+
+export async function getWatchTargets() {
+  const cfg = loadConfig();
+  const targets = new Set([configFile()]);
+  if (!cfg) return [...targets];
+
+  for (const [name, src] of Object.entries(cfg.sources || {})) {
+    if (!src?.enabled) continue;
+    for (const f of src.files || []) targets.add(expandTilde(f));
+
+    const roots = await expandRoots(src.roots || []);
+    const globs = normalizeGlobs(src, defaultWatchGlobs(name));
+    for (const root of roots) {
+      for (const g of globs) targets.add(`${root}/${g}`);
+    }
+  }
+
+  return [...targets];
+}
+
+function defaultWatchGlobs(name) {
+  switch (name) {
+    case 'hermes':
+    case 'claude-code':
+      return ['**/SKILL.md'];
+    case 'skills':
+    case 'skill':
+      return ['*.md', '*/SKILL.md', '**/SKILL.md'];
+    case 'mcp':
+      return ['*.md', '*.json', '*.yaml', '*.yml', '**/manifest.*'];
+    case 'hermes-plugin':
+      return ['**/{plugin.yaml,plugin.yml,plugin.json,manifest.json,package.json,README.md,readme.md}'];
+    case 'project-runbook':
+      return ['docs/RUNBOOK-*.md', 'AGENTS.md', 'CLAUDE.md', '.cursorrules', '.cursor/rules/**/*.{md,mdc}'];
+    case 'cursor':
+      return ['rules/**/*.{md,mdc}', '.cursorrules'];
+    case 'codex':
+      return ['AGENTS.md'];
+    default:
+      return ['**/*'];
+  }
+}
+
+function dedupeSemantic(items) {
+  // First remove exact same id defensively.
+  const byId = new Map();
+  for (const it of items) if (!byId.has(it.id)) byId.set(it.id, it);
+
+  // Then collapse repeated exports of the same semantic skill.
+  // gstack publishes the same skill into multiple hidden tool namespaces:
+  // .agents/.cursor/.factory/.gbrain/.hermes/.kiro/.opencode/.openclaw/...
+  // Those are useful as provenance, but noisy as separate entries in this hub.
+  const bySemantic = new Map();
+  for (const it of byId.values()) {
+    const key = `${it.source}:${it.kind}:${it.name}`;
+    const current = bySemantic.get(key);
+    if (!current || rankItem(it) > rankItem(current)) bySemantic.set(key, it);
+  }
+
+  return [...bySemantic.values()].sort((a, b) => {
+    const sa = `${a.source}\u0000${a.category || ''}\u0000${a.name}`;
+    const sb = `${b.source}\u0000${b.category || ''}\u0000${b.name}`;
+    return sa.localeCompare(sb);
+  });
+}
+
+function rankItem(it) {
+  const p = it.paths?.abs || '';
+  const parts = p.split('/').filter(Boolean);
+  const parent = parts.at(-2) || '';
+  const hiddenSegments = parts.filter(x => x.startsWith('.')).length;
+  let score = 0;
+  if (hiddenSegments === 0) score += 1000;
+  if (parent === it.name) score += 400;
+  if (it.description) score += 80;
+  if (it.triggers?.length) score += 60;
+  if (it.raw) score += Math.min(50, Math.floor(it.raw.length / 1000));
+  score -= hiddenSegments * 50;
+  score -= Math.min(200, p.length / 10);
+  return score;
+}
+
+function loadConfig() {
+  const f = configFile();
+  if (!fs.existsSync(f)) return null;
+  try {
+    const text = fs.readFileSync(f, 'utf8');
+    return YAML.parse(text) || {};
+  } catch (e) {
+    console.error(`[scan] sources.yaml parse error: ${e.message}`);
+    return null;
+  }
+}
+
+export { configFile };

package/service/packages/scanner/src/types.d.ts

@@ -0,0 +1,88 @@
+// Unified IR (Intermediate Representation) — every adapter normalizes its
+// source into this shape. The web UI only knows about SkillItem, never the
+// raw source layout.
+//
+// Type definitions live here for IDE / future TS migration. Runtime is plain
+// JS, no compile step needed.
+
+export interface ParamSpec {
+  name: string;
+  type?: string;
+  required?: boolean;
+  description?: string;
+  default?: unknown;
+}
+
+export interface SkillItem {
+  /** sha1(absPath) — stable across runs, doubles as web-side key */
+  id: string;
+
+  /** broad type — drives icon + filtering */
+  kind: 'skill' | 'plugin' | 'mcp' | 'runbook' | 'instruction' | 'config' | 'doc' | 'agent-rule';
+
+  /** which adapter produced this item */
+  source:
+    | 'hermes'
+    | 'claude-code'
+    | 'codex'
+    | 'cursor'
+    | 'obsidian'
+    | 'project'
+    | 'project-runbook'
+    | 'hermes-plugin'
+    | 'mcp-config';
+
+  /** machine name (frontmatter.name or file basename) */
+  name: string;
+
+  /** human-readable title — falls back to name */
+  title?: string;
+
+  /** one-line description — falls back to first paragraph */
+  description?: string;
+
+  /** category path, e.g. "devops", "mlops/inference" */
+  category?: string;
+
+  /** owning editor/tool surface, e.g. "Hermes Agent", "Claude Code", "Cursor" */
+  editor?: string;
+
+  /** brand inferred from description / path, e.g. "OpenAI", "Cloudflare" */
+  brand?: string;
+
+  /** product name extracted from skill, e.g. "new-api", "frp" */
+  product?: string;
+
+  /** trigger phrases (when_to_use / triggers / aliases) */
+  triggers?: string[];
+
+  /** MCP tool param schema — empty for skills */
+  params?: ParamSpec[];
+
+  /** free-form tags */
+  tags?: string[];
+
+  paths: {
+    /** absolute path on disk — copy-button payload */
+    abs: string;
+    /** path relative to project root if applicable */
+    rel?: string;
+    /** classification of the root the file lives under */
+    rootKind: 'home' | 'project' | 'icloud';
+  };
+
+  /** truncated markdown body for list-view preview */
+  preview: string;
+
+  /** complete file content — lazy-loaded by detail view via /api/skills/:id */
+  raw: string;
+
+  /** related links (docs URL, source repo) discovered in frontmatter */
+  links?: { label: string; url: string }[];
+
+  /** ISO timestamp — last modified time of the underlying file */
+  updatedAt: string;
+
+  /** present when frontmatter / format parse failed; UI shows red badge */
+  parseError?: string;
+}

package/service/packages/scanner/src/utils.mjs

@@ -0,0 +1,186 @@
+// Shared scanner utilities — every adapter MUST use these for consistency.
+//
+// parseFrontmatter   YAML frontmatter at top of markdown, tolerant
+// sha1Id             stable id from absolute path
+// expandRoots        glob-aware path resolution (~ / *)
+// inferBrand         keyword-match brand from text
+// inferProduct       extract product from skill name / category
+// classifyRoot       'home' | 'project' | 'icloud'
+// readFileSafe       size-guarded read
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+import YAML from 'yaml';
+import fg from 'fast-glob';
+
+// ─────────────────────────────── path / fs ────────────────────────────────
+
+export function expandTilde(p) {
+  if (!p) return p;
+  if (p === '~' || p.startsWith('~/')) {
+    return path.join(os.homedir(), p.slice(2));
+  }
+  return p;
+}
+
+/**
+ * Expand a list of roots (may contain ~ and glob *) into actual existing dirs.
+ * Non-existing paths are silently dropped — callers report "unavailable" via
+ * stat checks separately.
+ */
+export async function expandRoots(roots) {
+  if (!roots || !roots.length) return [];
+  const out = new Set();
+  for (const r of roots) {
+    const expanded = expandTilde(r);
+    if (expanded.includes('*')) {
+      const matches = await fg(expanded, { onlyDirectories: true, absolute: true });
+      matches.forEach(m => out.add(m));
+    } else {
+      const abs = path.resolve(expanded);
+      if (fs.existsSync(abs)) out.add(abs);
+    }
+  }
+  return [...out];
+}
+
+export function classifyRoot(absPath) {
+  if (absPath.includes('/Library/Mobile Documents/')) return 'icloud';
+  if (absPath.startsWith(os.homedir() + '/Project/')) return 'project';
+  if (absPath.startsWith(os.homedir())) return 'home';
+  return 'project';
+}
+
+export function readFileSafe(abs, maxBytes = 1024 * 1024) {
+  try {
+    const stat = fs.statSync(abs);
+    if (stat.size > maxBytes) {
+      return { text: '', truncated: true, size: stat.size, mtime: stat.mtime };
+    }
+    return {
+      text: fs.readFileSync(abs, 'utf8'),
+      truncated: false,
+      size: stat.size,
+      mtime: stat.mtime,
+    };
+  } catch (e) {
+    return { text: '', error: e.message, size: 0, mtime: new Date(0) };
+  }
+}
+
+// ─────────────────────────────── id / hash ────────────────────────────────
+
+export function sha1Id(absPath) {
+  return crypto.createHash('sha1').update(absPath).digest('hex').slice(0, 16);
+}
+
+// ─────────────────────────────── frontmatter ──────────────────────────────
+
+const FM_RE = /^---\s*\n([\s\S]*?)\n---\s*\n?([\s\S]*)$/;
+
+/**
+ * Parse YAML frontmatter at the very top of a markdown file.
+ * Returns { data, body, parseError? }. `data` is always an object (possibly
+ * empty). Body never has the frontmatter block.
+ */
+export function parseFrontmatter(text) {
+  if (!text || !text.startsWith('---')) {
+    return { data: {}, body: text || '' };
+  }
+  const m = text.match(FM_RE);
+  if (!m) {
+    return { data: {}, body: text };
+  }
+  const [, yamlBlock, body] = m;
+  try {
+    const data = YAML.parse(yamlBlock) || {};
+    return {
+      data: typeof data === 'object' && !Array.isArray(data) ? data : {},
+      body: body || '',
+    };
+  } catch (e) {
+    return { data: {}, body: body || text, parseError: `frontmatter: ${e.message}` };
+  }
+}
+
+// ─────────────────────────────── inference ────────────────────────────────
+
+const BRAND_KEYWORDS = [
+  // tooling
+  ['Hermes', /\bhermes\b/i],
+  ['Claude Code', /\bclaude[\s-]?code\b/i],
+  ['Anthropic', /\b(anthropic|claude)\b/i],
+  ['OpenAI', /\b(openai|chatgpt|gpt-?[0-9]|codex)\b/i],
+  ['Cursor', /\bcursor\b/i],
+  ['GitHub', /\b(github|\bgh\b|octokit)\b/i],
+  ['Cloudflare', /\b(cloudflare|cf-|workers|wrangler|r2|d1)\b/i],
+  ['Google', /\b(gemini|google\s|gws)\b/i],
+  ['Apple', /\b(apple|imessage|appleнотes|findmy|macos)\b/i],
+  ['Notion', /\bnotion\b/i],
+  ['Linear', /\blinear\b/i],
+  ['Obsidian', /\bobsidian\b/i],
+  ['Modal', /\bmodal\b/i],
+  ['HuggingFace', /\b(huggingface|hf-cli|hf\s+hub)\b/i],
+  ['Discord', /\bdiscord\b/i],
+  ['Telegram', /\btelegram\b/i],
+  ['X', /\b(twitter|x\.com|x-cli|xurl)\b/i],
+  ['Vercel', /\bvercel\b/i],
+  ['Docker', /\bdocker\b/i],
+  ['Kubernetes', /\b(kubernetes|kubectl|k8s)\b/i],
+  ['NVIDIA', /\b(cuda|nvidia)\b/i],
+  ['Suno', /\bsuno\b/i],
+  ['Spotify', /\bspotify\b/i],
+];
+
+export function inferBrand({ name, description, category, raw }) {
+  const hay = [name, description, category].filter(Boolean).join(' ');
+  const head = (raw || '').slice(0, 800);
+  const text = `${hay} ${head}`;
+  for (const [brand, re] of BRAND_KEYWORDS) {
+    if (re.test(text)) return brand;
+  }
+  return undefined;
+}
+
+/**
+ * Infer "product" — typically the skill name itself for product-shaped skills
+ * (e.g. "new-api-deployment" → "new-api"), or the bare name otherwise.
+ */
+export function inferProduct({ name, category }) {
+  if (!name) return undefined;
+  // Strip common suffixes that aren't product names
+  const cleaned = name
+    .replace(/-deployment$/, '')
+    .replace(/-workflow$/, '')
+    .replace(/-bot-notifications$/, '')
+    .replace(/-management$/, '');
+  return cleaned;
+}
+
+// ─────────────────────────────── markdown ─────────────────────────────────
+
+/**
+ * Extract a one-line description: prefer frontmatter.description, else first
+ * non-empty paragraph (no headings, no blank).
+ */
+export function deriveDescription(fmData, body) {
+  if (fmData?.description && typeof fmData.description === 'string') {
+    return fmData.description.trim();
+  }
+  for (const line of body.split(/\r?\n/)) {
+    const t = line.trim();
+    if (!t) continue;
+    if (t.startsWith('#')) continue;
+    if (t.startsWith('---')) continue;
+    return t.slice(0, 240);
+  }
+  return undefined;
+}
+
+export function makePreview(body, maxChars = 600) {
+  const trimmed = (body || '').trim();
+  if (trimmed.length <= maxChars) return trimmed;
+  return trimmed.slice(0, maxChars).trimEnd() + '…';
+}