@veyralabs/skills 0.1.0 → 0.2.0

package/skills/webcloner/scripts/download-assets.mjs

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+/**
+ * download-assets.mjs — Download all assets from site-manifest.json to public/
+ *
+ * Usage:
+ *   node scripts/download-assets.mjs docs/site-manifest.json public/
+ *
+ * Requires: sharp (optional, for WebP conversion)
+ *   npm install sharp
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { pipeline } from 'stream/promises';
+import { createWriteStream, mkdirSync } from 'fs';
+
+const [,, manifestPath = 'docs/site-manifest.json', outputDir = 'public'] = process.argv;
+
+if (!fs.existsSync(manifestPath)) {
+  console.error(`Manifest not found: ${manifestPath}`);
+  process.exit(1);
+}
+
+const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+const baseUrl = manifest.url;
+
+let sharp;
+try {
+  sharp = (await import('sharp')).default;
+} catch {
+  console.warn('sharp not installed — skipping WebP conversion (npm install sharp to enable)');
+}
+
+function sanitizePath(url, type) {
+  try {
+    const u = new URL(url);
+    const ext = path.extname(u.pathname).toLowerCase() || '.bin';
+    const name = path.basename(u.pathname, ext) || 'asset';
+    const safe = name.replace(/[^a-z0-9-_]/gi, '-').slice(0, 60);
+    return path.join(outputDir, type, `${safe}${ext}`);
+  } catch {
+    return null;
+  }
+}
+
+async function fetchAsset(url) {
+  const resolved = url.startsWith('http') ? url : new URL(url, baseUrl).href;
+  const res = await fetch(resolved, {
+    headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WebCloner/1.0)' },
+    redirect: 'follow',
+  });
+  if (!res.ok) throw new Error(`HTTP ${res.status}: ${resolved}`);
+  return { res, resolved };
+}
+
+async function downloadImage(url) {
+  const destPath = sanitizePath(url, 'images');
+  if (!destPath) return null;
+
+  const ext = path.extname(destPath).toLowerCase();
+  const isConvertible = ['.jpg', '.jpeg', '.png'].includes(ext) && sharp;
+  const finalPath = isConvertible ? destPath.replace(ext, '.webp') : destPath;
+
+  if (fs.existsSync(finalPath)) return finalPath;
+  mkdirSync(path.dirname(finalPath), { recursive: true });
+
+  const { res } = await fetchAsset(url);
+  const buffer = Buffer.from(await res.arrayBuffer());
+
+  if (isConvertible) {
+    await sharp(buffer).webp({ quality: 85 }).toFile(finalPath);
+  } else {
+    fs.writeFileSync(finalPath, buffer);
+  }
+
+  return finalPath;
+}
+
+async function downloadGeneric(url, type) {
+  const destPath = sanitizePath(url, type);
+  if (!destPath || fs.existsSync(destPath)) return destPath;
+  mkdirSync(path.dirname(destPath), { recursive: true });
+
+  const { res } = await fetchAsset(url);
+  const buffer = Buffer.from(await res.arrayBuffer());
+  fs.writeFileSync(destPath, buffer);
+  return destPath;
+}
+
+async function batch(items, fn, concurrency = 4) {
+  const results = [];
+  for (let i = 0; i < items.length; i += concurrency) {
+    const chunk = items.slice(i, i + concurrency);
+    const settled = await Promise.allSettled(chunk.map(fn));
+    for (const r of settled) {
+      if (r.status === 'fulfilled') results.push({ ok: true, path: r.value });
+      else results.push({ ok: false, error: r.reason?.message });
+    }
+  }
+  return results;
+}
+
+const assets = manifest.assets || {};
+const report = { images: [], videos: [], fonts: [], failed: [] };
+
+console.log('Downloading assets...\n');
+
+// Images
+if (assets.images?.length) {
+  console.log(`Images: ${assets.images.length}`);
+  const urls = [...new Set(assets.images.map(i => i.src).filter(Boolean))];
+  const results = await batch(urls, downloadImage);
+  results.forEach((r, i) => {
+    if (r.ok) { report.images.push(r.path); process.stdout.write('.'); }
+    else { report.failed.push({ url: urls[i], error: r.error }); process.stdout.write('x'); }
+  });
+  console.log();
+}
+
+// Videos
+if (assets.videos?.length) {
+  console.log(`\nVideos: ${assets.videos.length}`);
+  const urls = [...new Set(assets.videos.map(v => v.src).filter(Boolean))];
+  const results = await batch(urls, u => downloadGeneric(u, 'videos'), 2);
+  results.forEach((r, i) => {
+    if (r.ok) { report.videos.push(r.path); process.stdout.write('.'); }
+    else { report.failed.push({ url: urls[i], error: r.error }); process.stdout.write('x'); }
+  });
+  console.log();
+}
+
+// Self-hosted fonts
+if (assets.fonts?.length) {
+  const selfHosted = assets.fonts.filter(f => !f.includes('googleapis') && !f.includes('typekit'));
+  if (selfHosted.length) {
+    console.log(`\nFonts (self-hosted): ${selfHosted.length}`);
+    const results = await batch(selfHosted, u => downloadGeneric(u, 'fonts'));
+    results.forEach((r, i) => {
+      if (r.ok) { report.fonts.push(r.path); process.stdout.write('.'); }
+      else { report.failed.push({ url: selfHosted[i], error: r.error }); process.stdout.write('x'); }
+    });
+    console.log();
+  }
+}
+
+// Write report
+const reportPath = 'docs/assets-report.json';
+mkdirSync('docs', { recursive: true });
+fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+
+console.log(`\n--- Download Report ---`);
+console.log(`Images:  ${report.images.length}`);
+console.log(`Videos:  ${report.videos.length}`);
+console.log(`Fonts:   ${report.fonts.length}`);
+console.log(`Failed:  ${report.failed.length}`);
+if (report.failed.length) {
+  console.log('\nFailed assets:');
+  report.failed.forEach(f => console.log(`  ✗ ${f.url}\n    ${f.error}`));
+}
+console.log(`\nReport: ${reportPath}`);

package/skills/webcloner/scripts/extract.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""
+extract.py — Site manifest extractor using Scrapling.
+
+Usage:
+  python scripts/extract.py <url> [--output path/to/manifest.json] [--section ".selector"]
+
+Output: JSON manifest with DOM structure, computed CSS, assets, animations, and tech stack.
+"""
+
+import asyncio
+import json
+import re
+import sys
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+try:
+    from scrapling import AsyncPlaywrightFetcher
+except ImportError:
+    print("ERROR: scrapling not installed. Run: pip install scrapling && scrapling install")
+    sys.exit(1)
+
+
+# CSS properties to extract for each significant element
+CSS_PROPERTIES = [
+    "display", "position", "top", "right", "bottom", "left", "z-index",
+    "width", "min-width", "max-width", "height", "min-height", "max-height",
+    "margin", "margin-top", "margin-right", "margin-bottom", "margin-left",
+    "padding", "padding-top", "padding-right", "padding-bottom", "padding-left",
+    "flex-direction", "flex-wrap", "justify-content", "align-items", "align-self",
+    "flex", "flex-grow", "flex-shrink", "flex-basis", "gap", "row-gap", "column-gap",
+    "grid-template-columns", "grid-template-rows", "grid-column", "grid-row",
+    "font-family", "font-size", "font-weight", "font-style", "line-height",
+    "letter-spacing", "text-align", "text-transform", "text-decoration", "color",
+    "background-color", "background-image", "background-size", "background-position",
+    "border", "border-radius", "border-color", "border-width", "border-style",
+    "box-shadow", "opacity", "overflow", "overflow-x", "overflow-y",
+    "transform", "transition", "animation", "animation-name", "animation-duration",
+    "cursor", "pointer-events", "user-select",
+    "object-fit", "object-position", "aspect-ratio",
+]
+
+
+ANIMATION_SIGNATURES = {
+    "gsap": [
+        "window.gsap !== undefined",
+        "window.ScrollTrigger !== undefined",
+        "window.ScrollSmoother !== undefined",
+    ],
+    "framer-motion": [
+        "document.querySelector('[data-framer-appear]') !== null",
+        "document.querySelector('[data-projection-id]') !== null",
+    ],
+    "lenis": [
+        "window.lenis !== undefined",
+        "document.documentElement.classList.contains('lenis')",
+        "document.querySelector('.lenis') !== null",
+    ],
+    "aos": [
+        "document.querySelector('[data-aos]') !== null",
+    ],
+    "swiper": [
+        "window.Swiper !== undefined",
+        "document.querySelector('.swiper') !== null",
+    ],
+    "locomotive": [
+        "document.querySelector('[data-scroll-container]') !== null",
+        "window.LocomotiveScroll !== undefined",
+    ],
+}
+
+
+EXTRACTION_SCRIPT = """
+() => {
+  const CSS_PROPS = %s;
+
+  function getComputedStyleMap(el) {
+    const cs = window.getComputedStyle(el);
+    const result = {};
+    for (const prop of CSS_PROPS) {
+      const val = cs.getPropertyValue(prop);
+      if (val && val !== 'none' && val !== 'normal' && val !== 'auto' && val !== '0px' && val !== 'rgba(0, 0, 0, 0)') {
+        result[prop] = val;
+      }
+    }
+    return result;
+  }
+
+  function extractElement(el, depth) {
+    if (depth > 4) return null;
+    const rect = el.getBoundingClientRect();
+    if (rect.width === 0 && rect.height === 0) return null;
+
+    const result = {
+      tag: el.tagName.toLowerCase(),
+      id: el.id || null,
+      classes: Array.from(el.classList).slice(0, 10),
+      text: el.childNodes.length === 1 && el.childNodes[0].nodeType === 3
+        ? el.textContent.trim().slice(0, 200)
+        : null,
+      href: el.tagName === 'A' ? el.href : null,
+      src: el.tagName === 'IMG' ? el.src : null,
+      alt: el.tagName === 'IMG' ? el.alt : null,
+      styles: getComputedStyleMap(el),
+      bounds: {
+        top: Math.round(rect.top + window.scrollY),
+        left: Math.round(rect.left),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+      },
+      children: [],
+    };
+
+    if (depth < 3) {
+      for (const child of el.children) {
+        const childData = extractElement(child, depth + 1);
+        if (childData) result.children.push(childData);
+      }
+    }
+
+    return result;
+  }
+
+  // Extract sections (top-level structural blocks)
+  const sectionSelectors = [
+    'header', 'nav', 'main', 'footer', 'section', 'article',
+    '[class*="section"]', '[class*="hero"]', '[class*="banner"]',
+    '[id*="section"]', '[id*="hero"]',
+  ];
+
+  const seen = new Set();
+  const sections = [];
+
+  for (const sel of sectionSelectors) {
+    for (const el of document.querySelectorAll(sel)) {
+      if (seen.has(el)) continue;
+      seen.add(el);
+      const data = extractElement(el, 0);
+      if (data) sections.push(data);
+    }
+  }
+
+  // Color palette from all computed background-colors
+  const colors = new Set();
+  document.querySelectorAll('*').forEach(el => {
+    const cs = window.getComputedStyle(el);
+    const bg = cs.backgroundColor;
+    const color = cs.color;
+    if (bg && bg !== 'rgba(0, 0, 0, 0)') colors.add(bg);
+    if (color && color !== 'rgba(0, 0, 0, 0)') colors.add(color);
+  });
+
+  // Typography from headings and body text
+  const typography = [];
+  ['h1','h2','h3','h4','p','a','span','li'].forEach(tag => {
+    const el = document.querySelector(tag);
+    if (el) {
+      const cs = window.getComputedStyle(el);
+      typography.push({
+        tag,
+        fontFamily: cs.fontFamily,
+        fontSize: cs.fontSize,
+        fontWeight: cs.fontWeight,
+        lineHeight: cs.lineHeight,
+        letterSpacing: cs.letterSpacing,
+        color: cs.color,
+      });
+    }
+  });
+
+  // Asset inventory
+  const images = Array.from(document.querySelectorAll('img')).map(img => ({
+    src: img.src,
+    alt: img.alt,
+    width: img.naturalWidth || img.width,
+    height: img.naturalHeight || img.height,
+    lazy: img.loading === 'lazy',
+  }));
+
+  const videos = Array.from(document.querySelectorAll('video')).map(v => ({
+    src: v.src || (v.querySelector('source') ? v.querySelector('source').src : null),
+    poster: v.poster,
+    autoplay: v.autoplay,
+    loop: v.loop,
+    muted: v.muted,
+  }));
+
+  const fonts = Array.from(document.querySelectorAll('link[rel="stylesheet"]'))
+    .map(l => l.href)
+    .filter(h => h.includes('fonts.googleapis') || h.includes('typekit') || h.includes('fonts.adobe'));
+
+  const svgs = Array.from(document.querySelectorAll('svg')).map((svg, i) => ({
+    id: svg.id || `svg-${i}`,
+    viewBox: svg.getAttribute('viewBox'),
+    html: svg.outerHTML.slice(0, 500),
+  }));
+
+  return { sections, colorPalette: Array.from(colors), typography, assets: { images, videos, fonts, svgs } };
+}
+""" % json.dumps(CSS_PROPERTIES)
+
+
+async def detect_animations(page):
+    detected = []
+    for lib, checks in ANIMATION_SIGNATURES.items():
+        for check in checks:
+            try:
+                result = await page.evaluate(f"() => {{ try {{ return Boolean({check}); }} catch(e) {{ return false; }} }}")
+                if result:
+                    detected.append(lib)
+                    break
+            except Exception:
+                pass
+    return list(set(detected))
+
+
+async def detect_tech_stack(page):
+    stack = {}
+
+    checks = {
+        "react": "window.__NEXT_DATA__ !== undefined || window.React !== undefined",
+        "nextjs": "window.__NEXT_DATA__ !== undefined",
+        "vue": "window.__VUE__ !== undefined || document.querySelector('[data-v-app]') !== null",
+        "nuxt": "window.__NUXT__ !== undefined",
+        "svelte": "document.querySelector('[data-svelte]') !== null",
+        "webflow": "document.querySelector('html[data-wf-site]') !== null",
+        "framer": "document.querySelector('html[data-framer-hydrate-v2]') !== null",
+        "shopify": "window.Shopify !== undefined",
+        "wordpress": "document.querySelector('meta[name=\"generator\"][content^=\"WordPress\"]') !== null",
+        "tailwind": "Array.from(document.querySelectorAll('*')).some(el => Array.from(el.classList).some(c => /^(flex|grid|p-|m-|text-|bg-|w-|h-)/.test(c)))",
+    }
+
+    for name, check in checks.items():
+        try:
+            result = await page.evaluate(f"() => {{ try {{ return Boolean({check}); }} catch(e) {{ return false; }} }}")
+            if result:
+                stack[name] = True
+        except Exception:
+            pass
+
+    return stack
+
+
+async def extract(url, output_path, section_selector=None):
+    print(f"Extracting: {url}")
+
+    fetcher = AsyncPlaywrightFetcher(headless=True)
+
+    page_data = await fetcher.async_fetch(
+        url,
+        wait_selector="body",
+        wait_for="networkidle",
+        page_action=None,
+    )
+
+    # Use the underlying Playwright page for JS evaluation
+    async with fetcher._get_browser_context() as context:
+        page = await context.new_page()
+        await page.goto(url, wait_until="networkidle")
+        await page.wait_for_timeout(2000)  # let animations settle
+
+        # Scroll to trigger lazy loads
+        await page.evaluate("""
+            async () => {
+                const height = document.body.scrollHeight;
+                for (let y = 0; y < height; y += 200) {
+                    window.scrollTo(0, y);
+                    await new Promise(r => setTimeout(r, 50));
+                }
+                window.scrollTo(0, 0);
+                await new Promise(r => setTimeout(r, 500));
+            }
+        """)
+
+        animations = await detect_animations(page)
+        tech_stack = await detect_tech_stack(page)
+        dom_data = await page.evaluate(EXTRACTION_SCRIPT)
+
+        # Get page title and meta
+        title = await page.title()
+        meta_desc = await page.evaluate(
+            "() => document.querySelector('meta[name=\"description\"]')?.content || ''"
+        )
+
+        # Detect breakpoints from stylesheets
+        breakpoints = await page.evaluate("""
+            () => {
+                const bps = new Set();
+                for (const sheet of document.styleSheets) {
+                    try {
+                        for (const rule of sheet.cssRules) {
+                            if (rule.type === CSSRule.MEDIA_RULE) {
+                                const match = rule.conditionText.match(/\\d+/g);
+                                if (match) match.forEach(n => bps.add(parseInt(n)));
+                            }
+                        }
+                    } catch(e) {}
+                }
+                return Array.from(bps).sort((a,b) => a-b);
+            }
+        """)
+
+    manifest = {
+        "url": url,
+        "title": title,
+        "description": meta_desc,
+        "techStack": tech_stack,
+        "animations": {
+            "libraries": animations,
+        },
+        "breakpoints": breakpoints,
+        "colorPalette": dom_data["colorPalette"][:30],
+        "typography": dom_data["typography"],
+        "sections": dom_data["sections"],
+        "assets": dom_data["assets"],
+    }
+
+    output_file = Path(output_path)
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    output_file.write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
+
+    print(f"\nManifest written to: {output_path}")
+    print(f"  Sections found:   {len(manifest['sections'])}")
+    print(f"  Images found:     {len(manifest['assets']['images'])}")
+    print(f"  SVGs found:       {len(manifest['assets']['svgs'])}")
+    print(f"  Animation libs:   {', '.join(animations) or 'none detected'}")
+    print(f"  Tech stack:       {', '.join(k for k,v in tech_stack.items() if v)}")
+    print(f"  Breakpoints:      {breakpoints}")
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Extract site manifest")
+    parser.add_argument("url", help="Target URL")
+    parser.add_argument("--output", default="docs/site-manifest.json", help="Output path")
+    parser.add_argument("--section", default=None, help="CSS selector to scope extraction")
+    args = parser.parse_args()
+
+    asyncio.run(extract(args.url, args.output, args.section))
+
+
+if __name__ == "__main__":
+    main()

package/validate.js

@@ -11,13 +11,24 @@ const SKILLS_DIR = path.join(__dirname, 'skills');
 
 function findSkillFiles(dir) {
   const results = [];
-
   if (!fs.existsSync(dir)) return results;
 
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     if (!entry.isDirectory()) continue;
-    const candidate = path.join(dir, entry.name, 'SKILL.md');
-    if (fs.existsSync(candidate)) results.push(candidate);
+    const entryPath = path.join(dir, entry.name);
+    const direct = path.join(entryPath, 'SKILL.md');
+
+    if (fs.existsSync(direct)) {
+      // Standalone skill
+      results.push(direct);
+    } else {
+      // Pack folder — scan one level deeper
+      for (const sub of fs.readdirSync(entryPath, { withFileTypes: true })) {
+        if (!sub.isDirectory()) continue;
+        const candidate = path.join(entryPath, sub.name, 'SKILL.md');
+        if (fs.existsSync(candidate)) results.push(candidate);
+      }
+    }
   }
 
   return results;
@@ -45,7 +56,6 @@ function parseFrontmatter(content) {
 
     if (!key || key.includes(' ')) { i++; continue; }
 
-    // Handle block scalars (> and |)
     if (rawValue === '>' || rawValue === '|') {
       const blockLines = [];
       i++;