dembrandt 0.12.9 → 0.12.10

package/README.md

@@ -67,9 +67,9 @@ dembrandt example.com --mobile         # Use mobile viewport (390x844) for respo
 dembrandt example.com --slow           # 3x longer timeouts (24s hydration) for JavaScript-heavy sites
 dembrandt example.com --brand-guide    # Generate a brand guide PDF
 dembrandt example.com --design-md      # Generate a DESIGN.md file for AI agents
-dembrandt example.com --pages 5        # Analyze 5 pages (homepage + 4 discovered pages), merges results
+dembrandt example.com --crawl 5        # Analyze 5 pages (homepage + 4 discovered pages), merges results
 dembrandt example.com --sitemap        # Discover pages from sitemap.xml instead of DOM links
-dembrandt example.com --pages 10 --sitemap # Combine: up to 10 pages discovered via sitemap
+dembrandt example.com --crawl 10 --sitemap # Combine: up to 10 pages discovered via sitemap
 dembrandt example.com --no-sandbox     # Disable Chromium sandbox (required for Docker/CI)
 dembrandt example.com --browser=firefox # Use Firefox instead of Chromium (better for Cloudflare bypass)
 dembrandt example.com --wcag           # WCAG 2.1 contrast analysis — real DOM pairs, AA/AAA grades
@@ -83,13 +83,13 @@ Analyze multiple pages to get a more complete picture of a site's design system.
 
 ```bash
 # Analyze homepage + 4 auto-discovered pages (default: 5 total)
-dembrandt example.com --pages 5
+dembrandt example.com --crawl 5
 
 # Use sitemap.xml for page discovery instead of DOM link scraping
 dembrandt example.com --sitemap
 
 # Combine both: up to 10 pages from sitemap
-dembrandt example.com --pages 10 --sitemap
+dembrandt example.com --crawl 10 --sitemap
 ```
 
 **Page discovery** works two ways:
@@ -197,7 +197,7 @@ dembrandt braintree.com --save-output
 
 **Multi-page audit** — get a fuller picture across the whole site
 ```bash
-dembrandt stripe.com --pages 10 --sitemap --save-output
+dembrandt stripe.com --crawl 10 --sitemap --save-output
 ```
 
 **Spot-check a value** — verify a specific token fast

package/index.js

@@ -132,6 +132,7 @@ program
             screenshotPath: opts.screenshot,
             discoverLinks: isAutoCrawl ? crawlN - 1 : null,
             wcag: opts.wcag,
+            includeRawColors: opts.rawColors,
           });
 
           // Build list of additional URLs to extract

package/lib/extractors/index.js

@@ -168,7 +168,7 @@ export async function extractBranding(url, spinner, browser, options = {}) {
     spinner.stop();
     console.log(color.info("\n  Extracting design tokens...\n"));
 
-    spinner.start("Analyzing design system (16 parallel tasks)...");
+    spinner.start("Analyzing design system (17 parallel tasks)...");
     const [
       { logo, instances: logoInstances, favicons },
       colors,

package/lib/merger.js

@@ -275,6 +275,56 @@ function mergeByName(results, getter) {
   return out;
 }
 
+function mergeGradients(results) {
+  const map = new Map();
+  results.forEach(r => {
+    (r.gradients || []).forEach(g => {
+      const key = g.gradient;
+      if (!map.has(key)) {
+        map.set(key, { ...g });
+      } else {
+        map.get(key).count += (g.count || 1);
+      }
+    });
+  });
+  return [...map.values()].sort((a, b) => b.count - a.count);
+}
+
+function mergeMotion(results) {
+  const base = results[0].motion || { durations: [], easings: [], animations: [], contexts: {}, interactiveDeltas: [] };
+
+  const durationMap = new Map();
+  results.forEach(r => {
+    (r.motion?.durations || []).forEach(d => {
+      if (!durationMap.has(d.value)) durationMap.set(d.value, { ...d });
+      else durationMap.get(d.value).count += (d.count || 1);
+    });
+  });
+
+  const easingMap = new Map();
+  results.forEach(r => {
+    (r.motion?.easings || []).forEach(e => {
+      if (!easingMap.has(e.value)) easingMap.set(e.value, { ...e });
+      else easingMap.get(e.value).count += (e.count || 1);
+    });
+  });
+
+  const animMap = new Map();
+  results.forEach(r => {
+    (r.motion?.animations || []).forEach(a => {
+      if (!animMap.has(a.name || a.value)) animMap.set(a.name || a.value, { ...a });
+      else animMap.get(a.name || a.value).count += (a.count || 1);
+    });
+  });
+
+  return {
+    ...base,
+    durations: [...durationMap.values()].sort((a, b) => a.ms - b.ms),
+    easings: [...easingMap.values()].sort((a, b) => b.count - a.count).slice(0, 8),
+    animations: [...animMap.values()].sort((a, b) => b.count - a.count).slice(0, 8),
+  };
+}
+
 /**
  * Merge an array of per-page result objects into a single unified result.
  * @param {Object[]} results - Array of extractBranding() result objects
@@ -298,6 +348,8 @@ export function mergeResults(results) {
     borderRadius: mergeBorderRadius(results),
     borders: mergeBorders(results),
     shadows: mergeShadows(results),
+    gradients: mergeGradients(results),
+    motion: mergeMotion(results),
     components: mergeComponents(results),
     breakpoints: [
       ...new Map(

package/mcp-server.js

@@ -99,16 +99,119 @@ function errorResult(message) {
   return { content: [{ type: "text", text: message }], isError: true };
 }
 
+// ── Job Queue ──────────────────────────────────────────────────────────
+
+class JobQueue {
+  #jobs = new Map();
+  #queue = [];
+  #running = new Set();
+  #maxConcurrent = 2;
+
+  enqueue(url, opts, pick) {
+    const id = `job_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    this.#jobs.set(id, {
+      status: "queued",
+      url,
+      opts,
+      pick,
+      createdAt: Date.now(),
+      startedAt: undefined,
+      completedAt: undefined,
+      result: undefined,
+      error: undefined,
+    });
+    this.#queue.push(id);
+    void this.#drain();
+    return id;
+  }
+
+  get(id) {
+    return this.#jobs.get(id) ?? null;
+  }
+
+  cancel(id) {
+    const job = this.#jobs.get(id);
+    if (!job || job.status !== "queued") return false;
+    job.status = "cancelled";
+    job.completedAt = Date.now();
+    const idx = this.#queue.indexOf(id);
+    if (idx !== -1) this.#queue.splice(idx, 1);
+    return true;
+  }
+
+  async #drain() {
+    while (this.#queue.length > 0 && this.#running.size < this.#maxConcurrent) {
+      const id = this.#queue.shift();
+      const job = this.#jobs.get(id);
+      if (!job || job.status === "cancelled") continue;
+
+      job.status = "running";
+      job.startedAt = Date.now();
+      this.#running.add(id);
+
+      runExtraction(job.url, job.opts)
+        .then((result) => {
+          if (job.status === "cancelled") return;
+          if (result.ok) {
+            job.status = "completed";
+            job.result = job.pick(result.data);
+          } else {
+            job.status = "failed";
+            job.error = result.error;
+          }
+        })
+        .catch((err) => {
+          if (job.status !== "cancelled") {
+            job.status = "failed";
+            job.error = err.message || String(err);
+          }
+        })
+        .finally(() => {
+          job.completedAt = Date.now();
+          this.#running.delete(id);
+          void this.#drain();
+        });
+    }
+  }
+
+  // Remove completed/failed/cancelled jobs older than 1 hour
+  cleanup() {
+    const cutoff = Date.now() - 3_600_000;
+    for (const [id, job] of this.#jobs) {
+      if (
+        ["completed", "failed", "cancelled"].includes(job.status) &&
+        job.completedAt !== undefined &&
+        job.completedAt < cutoff
+      ) {
+        this.#jobs.delete(id);
+      }
+    }
+  }
+}
+
+const jobQueue = new JobQueue();
+setInterval(() => jobQueue.cleanup(), 600_000);
+
+// ── Helpers ────────────────────────────────────────────────────────────
+
 /**
- * Wrapper that handles extraction + error formatting for all tools.
- * `pick` receives the full result and returns the filtered subset.
+ * Wrapper for extraction tools.
+ * Async by default: enqueues and returns a job_id immediately.
+ * Pass sync: true to block and return the result directly.
  */
 function toolHandler(pick, extraOptions = {}) {
   return async (params) => {
-    const { url, slow, darkMode } = params;
-    const result = await runExtraction(url, { slow, darkMode, ...extraOptions });
-    if (!result.ok) return errorResult(result.error);
-    return jsonResult(pick(result.data));
+    const { url, slow, darkMode, sync } = params;
+    const opts = { slow, darkMode, ...extraOptions };
+
+    if (sync) {
+      const result = await runExtraction(url, opts);
+      if (!result.ok) return errorResult(result.error);
+      return jsonResult(pick(result.data));
+    }
+
+    const jobId = jobQueue.enqueue(url, opts, pick);
+    return jsonResult({ job_id: jobId, status: "queued" });
   };
 }
 
@@ -116,21 +219,22 @@ function toolHandler(pick, extraOptions = {}) {
 
 const url = z.string().describe("Website URL (e.g. example.com)");
 const slow = z.boolean().optional().default(false).describe("3x timeouts for heavy SPAs");
+const sync = z.boolean().optional().default(false).describe("Wait for result directly instead of returning a job_id (blocks 15-40s)");
 
-// ── Tools ──────────────────────────────────────────────────────────────
+// ── Extraction tools ───────────────────────────────────────────────────
 
 server.tool(
   "get_design_tokens",
-  "Extract the full design system from a live website. Launches a real browser, navigates to the site, and returns production-ready design tokens: color palette (hex, RGB, LCH, OKLCH) with semantic roles and CSS custom properties, typography scale (families, fallbacks, sizes, weights, line heights, letter spacing by context), spacing system with grid detection, border radii, border patterns, box shadows for elevation, component styles (buttons with hover/focus states, inputs, links, badges), responsive breakpoints, logo and favicons, site name, detected CSS frameworks, and icon systems. Takes 15-40 seconds depending on site complexity.",
-  { url, slow },
+  "Extract the full design system from a live website. Launches a real browser, navigates to the site, and returns production-ready design tokens: color palette (hex, RGB, LCH, OKLCH) with semantic roles and CSS custom properties, typography scale (families, fallbacks, sizes, weights, line heights, letter spacing by context), spacing system with grid detection, border radii, border patterns, box shadows for elevation, component styles (buttons with hover/focus states, inputs, links, badges), responsive breakpoints, logo and favicons, site name, detected CSS frameworks, and icon systems. Returns a job_id by default — use get_job_status to poll for the result.",
+  { url, slow, sync },
   toolHandler((d) => d),
 );
 
 server.tool(
   "get_color_palette",
-  "Extract brand colors from a live website. Returns semantic colors (primary, secondary, accent), full palette ranked by usage frequency and confidence (high/medium/low), CSS custom properties with their design-system names, and hover/focus state colors discovered by simulating real user interactions. Each color in hex, RGB, LCH, and OKLCH.",
+  "Extract brand colors from a live website. Returns semantic colors (primary, secondary, accent), full palette ranked by usage frequency and confidence (high/medium/low), CSS custom properties with their design-system names, and hover/focus state colors discovered by simulating real user interactions. Each color in hex, RGB, LCH, and OKLCH. Returns a job_id by default — use get_job_status to poll for the result.",
   {
-    url, slow,
+    url, slow, sync,
     darkMode: z.boolean().optional().default(false).describe("Also extract dark mode palette"),
   },
   toolHandler((d) => ({ url: d.url, colors: d.colors })),
@@ -138,22 +242,22 @@ server.tool(
 
 server.tool(
   "get_typography",
-  "Extract typography from a live website. Returns every font family with its fallback stack, the complete type scale grouped by context (heading, body, button, link, caption) with pixel and rem sizes, weights, line heights, letter spacing, and text transforms. Also reports font sources: Google Fonts URLs, Adobe Fonts usage, and variable font detection.",
-  { url, slow },
+  "Extract typography from a live website. Returns every font family with its fallback stack, the complete type scale grouped by context (heading, body, button, link, caption) with pixel and rem sizes, weights, line heights, letter spacing, and text transforms. Also reports font sources: Google Fonts URLs, Adobe Fonts usage, and variable font detection. Returns a job_id by default — use get_job_status to poll for the result.",
+  { url, slow, sync },
   toolHandler((d) => ({ url: d.url, typography: d.typography })),
 );
 
 server.tool(
   "get_component_styles",
-  "Extract UI component styles from a live website. Returns button variants with default, hover, active, and focus states (background, text color, padding, border radius, border, shadow, outline, opacity), input field styles (border, focus ring, padding, placeholder), link styles (color, text decoration, hover changes), and badge/tag styles.",
-  { url, slow },
+  "Extract UI component styles from a live website. Returns button variants with default, hover, active, and focus states (background, text color, padding, border radius, border, shadow, outline, opacity), input field styles (border, focus ring, padding, placeholder), link styles (color, text decoration, hover changes), and badge/tag styles. Returns a job_id by default — use get_job_status to poll for the result.",
+  { url, slow, sync },
   toolHandler((d) => ({ url: d.url, components: d.components })),
 );
 
 server.tool(
   "get_surfaces",
-  "Extract surface treatment tokens from a live website: border radii with element context (which radii are used on buttons vs cards vs inputs vs modals), border patterns (width + style + color combinations), and box shadow elevation levels.",
-  { url, slow },
+  "Extract surface treatment tokens from a live website: border radii with element context (which radii are used on buttons vs cards vs inputs vs modals), border patterns (width + style + color combinations), and box shadow elevation levels. Returns a job_id by default — use get_job_status to poll for the result.",
+  { url, slow, sync },
   toolHandler((d) => ({
     url: d.url,
     borderRadius: d.borderRadius,
@@ -164,15 +268,15 @@ server.tool(
 
 server.tool(
   "get_spacing",
-  "Extract the spacing system from a live website: common margin and padding values sorted by frequency, pixel and rem values, and grid system detection (4px, 8px, or custom scale).",
-  { url, slow },
+  "Extract the spacing system from a live website: common margin and padding values sorted by frequency, pixel and rem values, and grid system detection (4px, 8px, or custom scale). Returns a job_id by default — use get_job_status to poll for the result.",
+  { url, slow, sync },
   toolHandler((d) => ({ url: d.url, spacing: d.spacing })),
 );
 
 server.tool(
   "get_brand_identity",
-  "Extract brand identity from a live website: site name, logo (source, dimensions, safe zone), all favicon variants (icon, apple-touch-icon, og:image, twitter:image with sizes and URLs), detected CSS frameworks (Tailwind, Bootstrap, MUI, etc.), icon systems (Font Awesome, Material Icons, SVG), and responsive breakpoints.",
-  { url, slow },
+  "Extract brand identity from a live website: site name, logo (source, dimensions, safe zone), all favicon variants (icon, apple-touch-icon, og:image, twitter:image with sizes and URLs), detected CSS frameworks (Tailwind, Bootstrap, MUI, etc.), icon systems (Font Awesome, Material Icons, SVG), and responsive breakpoints. Returns a job_id by default — use get_job_status to poll for the result.",
+  { url, slow, sync },
   toolHandler((d) => ({
     url: d.url,
     siteName: d.siteName,
@@ -184,6 +288,31 @@ server.tool(
   })),
 );
 
+// ── Job management tools ───────────────────────────────────────────────
+
+server.tool(
+  "get_job_status",
+  "Poll for the result of an async extraction job. Returns status (queued/running/completed/failed/cancelled) and the full result once completed. Call this after any extraction tool that returned a job_id.",
+  { job_id: z.string().describe("The job_id returned by an extraction tool") },
+  ({ job_id }) => {
+    const job = jobQueue.get(job_id);
+    if (!job) return errorResult(`No job found with id: ${job_id}`);
+    if (job.status === "completed") return jsonResult({ job_id, status: "completed", result: job.result });
+    if (job.status === "failed") return errorResult(`Job failed: ${job.error}`);
+    return jsonResult({ job_id, status: job.status });
+  },
+);
+
+server.tool(
+  "cancel_job",
+  "Cancel a queued extraction job. Has no effect on jobs that are already running.",
+  { job_id: z.string().describe("The job_id to cancel") },
+  ({ job_id }) => {
+    const cancelled = jobQueue.cancel(job_id);
+    return jsonResult({ job_id, cancelled });
+  },
+);
+
 // ── Start ──────────────────────────────────────────────────────────────
 
 const transport = new StdioServerTransport();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dembrandt",
-  "version": "0.12.9",
+  "version": "0.12.10",
   "description": "Extract design tokens and publicly visible CSS information from any website",
   "mcpName": "io.github.dembrandt/dembrandt",
   "main": "index.js",