dembrandt 0.10.0 → 0.11.0

package/README.md

@@ -4,37 +4,25 @@
 [![npm downloads](https://img.shields.io/npm/dm/dembrandt.svg)](https://www.npmjs.com/package/dembrandt)
 [![license](https://img.shields.io/npm/l/dembrandt.svg)](https://github.com/dembrandt/dembrandt/blob/main/LICENSE)
 
-Extract any website’s design system into design tokens in a few seconds: logo, colors, typography, borders, and more. One command.
+Extract a website's design system into design tokens in a few seconds: logo, colors, typography, borders, and more. One command.
 
 ![Dembrandt — Any website to design tokens](https://raw.githubusercontent.com/dembrandt/dembrandt/main/docs/images/banner.png)
 
-**CLI output**
-
-![CLI extraction of netflix.com](https://raw.githubusercontent.com/dembrandt/dembrandt/main/docs/images/cli-output.png)
-
-**Brand Guide PDF**
-
-![Brand guide PDF extracted from any URL](https://raw.githubusercontent.com/dembrandt/dembrandt/main/docs/images/brand-guide.png)
-
-**Local UI**
-
-![Local UI showing extracted brand](https://raw.githubusercontent.com/dembrandt/dembrandt/main/docs/images/local-ui.png)
-
 ## Install
 
 Install globally: `npm install -g dembrandt`
 
 ```bash
-dembrandt bmw.de
+dembrandt example.com
 ```
 
-Or use npx without installing: `npx dembrandt bmw.de`
+Or use npx without installing: `npx dembrandt example.com`
 
 Requires Node.js 18+
 
 ## AI Agent Integration (MCP)
 
-Use Dembrandt as a tool in Claude Code, Cursor, Windsurf, or any MCP-compatible client. Ask your agent to "extract the color palette from stripe.com" and it calls Dembrandt automatically.
+Use Dembrandt as a tool in Claude Code, Cursor, Windsurf, or any MCP-compatible client. Ask your agent to "extract the color palette from example.com" and it calls Dembrandt automatically.
 
 ```bash
 claude mcp add --transport stdio dembrandt -- npx -y dembrandt-mcp
@@ -69,44 +57,44 @@ Or add to your project's `.mcp.json`:
 ## Usage
 
 ```bash
-dembrandt <url>                    # Basic extraction (terminal display only)
-dembrandt bmw.de --json-only       # Output raw JSON to terminal (no formatted display, no file save)
-dembrandt bmw.de --save-output     # Save JSON to output/bmw.de/YYYY-MM-DDTHH-MM-SS.json
-dembrandt bmw.de --dtcg            # Export in W3C Design Tokens (DTCG) format (auto-saves as .tokens.json)
-dembrandt bmw.de --dark-mode       # Extract colors from dark mode variant
-dembrandt bmw.de --mobile          # Use mobile viewport (390x844, iPhone 12/13/14/15) for responsive analysis
-dembrandt bmw.de --slow            # 3x longer timeouts (24s hydration) for JavaScript-heavy sites
-dembrandt bmw.de --brand-guide      # Generate a brand guide PDF
-dembrandt bmw.de --design-md        # Generate a DESIGN.md file for AI agents
-dembrandt bmw.de --pages 5         # Analyze 5 pages (homepage + 4 discovered pages), merges results
-dembrandt bmw.de --sitemap          # Discover pages from sitemap.xml instead of DOM links
-dembrandt bmw.de --pages 10 --sitemap # Combine: up to 10 pages discovered via sitemap
-dembrandt bmw.de --no-sandbox      # Disable Chromium sandbox (required for Docker/CI)
-dembrandt bmw.de --browser=firefox # Use Firefox instead of Chromium (better for Cloudflare bypass)
+dembrandt <url>                        # Basic extraction (terminal display only)
+dembrandt example.com --json-only      # Output raw JSON to terminal (no formatted display, no file save)
+dembrandt example.com --save-output    # Save JSON to output/example.com/YYYY-MM-DDTHH-MM-SS.json
+dembrandt example.com --dtcg           # Export in W3C Design Tokens (DTCG) format (auto-saves as .tokens.json)
+dembrandt example.com --dark-mode      # Extract colors from dark mode variant
+dembrandt example.com --mobile         # Use mobile viewport (390x844) for responsive analysis
+dembrandt example.com --slow           # 3x longer timeouts (24s hydration) for JavaScript-heavy sites
+dembrandt example.com --brand-guide    # Generate a brand guide PDF
+dembrandt example.com --design-md      # Generate a DESIGN.md file for AI agents
+dembrandt example.com --pages 5        # Analyze 5 pages (homepage + 4 discovered pages), merges results
+dembrandt example.com --sitemap        # Discover pages from sitemap.xml instead of DOM links
+dembrandt example.com --pages 10 --sitemap # Combine: up to 10 pages discovered via sitemap
+dembrandt example.com --no-sandbox     # Disable Chromium sandbox (required for Docker/CI)
+dembrandt example.com --browser=firefox # Use Firefox instead of Chromium (better for Cloudflare bypass)
 ```
 
 Default: formatted terminal display only. Use `--save-output` to persist results as JSON files. Browser automatically retries in visible mode if headless extraction fails.
 
 ### Multi-Page Extraction
 
-Analyze multiple pages to get a more complete picture of a site's design system. Results are merged into a single unified output with cross-page confidence boosting — colors appearing on multiple pages get higher confidence scores.
+Analyze multiple pages to get a more complete picture of a site's design system. Results are merged into a single unified output with cross-page confidence boosting — tokens appearing on multiple pages get higher confidence scores.
 
 ```bash
 # Analyze homepage + 4 auto-discovered pages (default: 5 total)
-dembrandt stripe.com --pages 5
+dembrandt example.com --pages 5
 
 # Use sitemap.xml for page discovery instead of DOM link scraping
-dembrandt stripe.com --sitemap
+dembrandt example.com --sitemap
 
 # Combine both: up to 10 pages from sitemap
-dembrandt stripe.com --pages 10 --sitemap
+dembrandt example.com --pages 10 --sitemap
 ```
 
 **Page discovery** works two ways:
-- **DOM links** (default): Scrapes navigation, header, and footer links from the homepage, prioritizing key pages like /pricing, /about, /features
+- **DOM links** (default): Reads navigation, header, and footer links from the homepage, prioritizing key pages like /pricing, /about, /features
 - **Sitemap** (`--sitemap`): Parses sitemap.xml (checks robots.txt first), follows sitemapindex references, and scores URLs by importance
 
-Pages are crawled sequentially with polite delays. Failed pages are skipped without aborting the run.
+Pages are fetched sequentially with polite delays. Failed pages are skipped without aborting the run.
 
 ### Browser Selection
 
@@ -114,10 +102,10 @@ By default, dembrandt uses Chromium. If you encounter bot detection or timeouts
 
 ```bash
 # Use Firefox instead of Chromium
-dembrandt bmw.de --browser=firefox
+dembrandt example.com --browser=firefox
 
 # Combine with other flags
-dembrandt bmw.de --browser=firefox --save-output --dtcg
+dembrandt example.com --browser=firefox --save-output --dtcg
 ```
 
 **When to use Firefox:**
@@ -137,24 +125,33 @@ npx playwright install firefox
 Use `--dtcg` to export in the standardized [W3C Design Tokens Community Group](https://www.designtokens.org/) format:
 
 ```bash
-dembrandt stripe.com --dtcg
-# Saves to: output/stripe.com/TIMESTAMP.tokens.json
+dembrandt example.com --dtcg
+# Saves to: output/example.com/TIMESTAMP.tokens.json
 ```
 
 The DTCG format is an industry-standard JSON schema that can be consumed by design tools and token transformation libraries like [Style Dictionary](https://styledictionary.com).
 
 ### DESIGN.md
 
-Use `--design-md` to generate a [DESIGN.md](https://stitch.withgoogle.com/docs/design-md) file — a plain-text design system document readable by AI agents like Google Stitch.
+Use `--design-md` to generate a [DESIGN.md](https://stitch.withgoogle.com/docs/design-md) file — a plain-text design system document readable by AI agents.
 
 ```bash
-dembrandt stripe.com --design-md
-# Saves to: output/stripe.com/DESIGN.md
+dembrandt example.com --design-md
+# Saves to: output/example.com/DESIGN.md
+```
+
+### Brand Guide PDF
+
+Use `--brand-guide` to generate a printable PDF summarizing the extracted design system — colors, typography, components, and logo on a single document.
+
+```bash
+dembrandt example.com --brand-guide
+# Saves to: output/example.com/TIMESTAMP.brand-guide.pdf
 ```
 
 ## Local UI
 
-Browse your extracted brands in a visual interface.
+Browse your extractions in a visual interface.
 
 ### Setup
 
@@ -173,7 +170,7 @@ Opens http://localhost:5173 with API on port 3002.
 
 ### Features
 
-- Visual grid of all extracted brands
+- Visual grid of all extractions
 - Color palettes with click-to-copy
 - Typography specimens
 - Spacing, shadows, border radius visualization
@@ -185,14 +182,14 @@ Extractions are performed via CLI (`dembrandt <url> --save-output`) and automati
 
 ## Use Cases
 
-- Brand audits & competitive analysis
 - Design system documentation
-- Reverse engineering brands
-- Multi-site brand consolidation
+- Multi-site design consolidation
+- Internal design audits on your own properties
+- Learning how design tokens map to real CSS
 
 ## How It Works
 
-Uses Playwright to render the page, extracts computed styles from the DOM, analyzes color usage and confidence, groups similar typography, detects spacing patterns, and returns actionable design tokens.
+Uses Playwright to render the page, reads computed styles from the DOM, analyzes color usage and confidence, groups similar typography, detects spacing patterns, and returns design tokens.
 
 ### Extraction Process
 
@@ -207,35 +204,33 @@ Uses Playwright to render the page, extracts computed styles from the DOM, analy
 
 ### Color Confidence
 
-- High — Logo, brand elements, primary buttons
-- Medium — Interactive elements, icons, navigation
+- High — Logo, primary interactive elements
+- Medium — Secondary interactive elements, icons, navigation
 - Low — Generic UI components (filtered from display)
 - Only shows high and medium confidence colors in terminal. Full palette in JSON.
 
 ## Limitations
 
-- Dark mode requires --dark-mode flag (not automatically detected)
+- Dark mode requires `--dark-mode` flag (not automatically detected)
 - Hover/focus states extracted from CSS (not fully interactive)
-- Canvas/WebGL-rendered sites cannot be analyzed (e.g., Tesla, Apple Vision Pro demos)
+- Canvas/WebGL-rendered sites cannot be analyzed (no DOM to read)
 - JavaScript-heavy sites require hydration time (8s initial + 4s stabilization)
 - Some dynamically-loaded content may be missed
-- Default viewport is 1920x1080 (use --mobile for 390x844 iPhone viewport)
+- Default viewport is 1920x1080 (use `--mobile` for 390x844 mobile viewport)
 
-## Ethics & Legality
+## Intended Use
 
-Dembrandt extracts publicly available design information (colors, fonts, spacing) from website DOMs for analysis purposes. This falls under fair use in most jurisdictions (USA's DMCA § 1201(f), EU Software Directive 2009/24/EC) when used for competitive analysis, documentation, or learning.
+Dembrandt reads publicly available CSS and computed styles from website DOMs for documentation, learning, and analysis of design systems you own or have permission to analyze.
 
-Legal: Analyzing public HTML/CSS is generally legal. Does not bypass protections or violate copyright. Check site ToS before mass extraction.
+Only run Dembrandt against sites whose Terms of Service permit automated access, or against your own properties. Do not use extracted material to reproduce third-party brand identities, logos, or trademarks. Respect robots.txt, rate limits, and copyright.
 
-Ethical: Use for inspiration and analysis, not direct copying. Respect servers (no mass crawling), give credit to sources, be transparent about data origin.
+Dembrandt does not host, redistribute, or claim rights to any third-party brand assets.
 
 ## Contributing
 
-Bugs you found? Weird websites that make it cry? Pull requests (even one-liners make me happy)?
-
-Spam me in [Issues](https://github.com/dembrandt/dembrandt/issues) or PRs. I reply to everything.
+Bugs, weird sites, pull requests — all welcome.
 
-Let's keep the light alive together.
+Open an [Issue](https://github.com/dembrandt/dembrandt/issues) or PR.
 
 @thevangelist
 

package/index.js

@@ -20,11 +20,12 @@ import { parseSitemap } from "./lib/discovery.js";
 import { mergeResults } from "./lib/merger.js";
 import { writeFileSync, mkdirSync } from "fs";
 import { join } from "path";
+import { checkRobotsTxt } from "./lib/robots.js";
 
 program
   .name("dembrandt")
   .description("Extract design tokens from any website")
-  .version("0.10.0")
+  .version("0.11.0")
   .argument("<url>")
   .option("--browser <type>", "Browser to use (chromium|firefox)", "chromium")
   .option("--json-only", "Output raw JSON")
@@ -57,6 +58,21 @@ program
     }
 
     const spinner = ora({ text: "Starting extraction...", stream: opts.jsonOnly ? process.stderr : process.stdout }).start();
+
+    try {
+      const robots = await checkRobotsTxt(url);
+      if (robots.status === "ok" && robots.allowed === false) {
+        spinner.warn(
+          chalk.hex("#FFB86C")(
+            `robots.txt disallows this path (rule: "${robots.rule}"). Proceeding anyway — respect the site's terms.`
+          )
+        );
+        spinner.start("Starting extraction...");
+      }
+    } catch {
+      // robots check is advisory; never block extraction
+    }
+
     let browser = null;
 
     try {
@@ -101,8 +117,7 @@ program
             let additionalUrls;
             if (opts.sitemap) {
               // Try post-redirect URL first, fall back to user-provided URL
-              // (sites like spotify.com redirect browser to open.spotify.com
-              // but sitemap lives at www.spotify.com)
+              // (some sites redirect to a subdomain while the sitemap stays on www)
               additionalUrls = await parseSitemap(result.url, maxPages);
               if (additionalUrls.length === 0 && result.url !== url) {
                 additionalUrls = await parseSitemap(url, maxPages);

package/lib/extractors.js

@@ -131,7 +131,7 @@ export async function extractBranding(
           timeouts.push('Body content rendering');
         }
 
-        // Give SPAs time to hydrate (Linear, Figma, Notion, etc.)
+        // Give SPAs time to hydrate
         spinner.start("Waiting for SPA hydration...");
         const hydrationTime = 8000 * timeoutMultiplier;
         await page.waitForTimeout(hydrationTime);
@@ -627,7 +627,7 @@ export async function extractBranding(
       frameworks,
     };
 
-    // Detect canvas-only / WebGL sites (Tesla, Apple Vision Pro, etc.)
+    // Detect canvas-only / WebGL sites
     const isCanvasOnly = await page.evaluate(() => {
       const canvases = document.querySelectorAll("canvas");
       const hasRealContent = document.body.textContent.trim().length > 200;
@@ -641,7 +641,7 @@ export async function extractBranding(
 
     if (isCanvasOnly) {
       result.note =
-        "This website uses canvas/WebGL rendering (e.g. Tesla, Apple Vision Pro). Design system cannot be extracted from DOM.";
+        "This website uses canvas/WebGL rendering. Design system cannot be extracted from DOM.";
       result.isCanvasOnly = true;
     }
 

package/lib/robots.js

@@ -0,0 +1,101 @@
+const UA = "Dembrandt";
+
+export async function checkRobotsTxt(targetUrl, { timeoutMs = 5000 } = {}) {
+  const u = new URL(targetUrl);
+  const robotsUrl = `${u.protocol}//${u.host}/robots.txt`;
+  const path = u.pathname || "/";
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  let body;
+  try {
+    const res = await fetch(robotsUrl, {
+      signal: controller.signal,
+      headers: { "User-Agent": UA },
+    });
+    if (!res.ok) return { status: "unavailable", robotsUrl };
+    body = await res.text();
+  } catch {
+    return { status: "unavailable", robotsUrl };
+  } finally {
+    clearTimeout(timer);
+  }
+
+  const groups = parseRobots(body);
+  const rules = matchGroup(groups, UA) || matchGroup(groups, "*") || [];
+  const decision = evaluate(rules, path);
+
+  return { status: "ok", robotsUrl, ...decision };
+}
+
+function parseRobots(text) {
+  const groups = [];
+  let current = null;
+  let lastWasAgent = false;
+
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, "").trim();
+    if (!line) continue;
+    const idx = line.indexOf(":");
+    if (idx === -1) continue;
+    const field = line.slice(0, idx).trim().toLowerCase();
+    const value = line.slice(idx + 1).trim();
+
+    if (field === "user-agent") {
+      if (!current || !lastWasAgent) {
+        current = { agents: [], rules: [] };
+        groups.push(current);
+      }
+      current.agents.push(value.toLowerCase());
+      lastWasAgent = true;
+    } else if (field === "allow" || field === "disallow") {
+      if (!current) {
+        current = { agents: ["*"], rules: [] };
+        groups.push(current);
+      }
+      current.rules.push({ type: field, value });
+      lastWasAgent = false;
+    }
+  }
+  return groups;
+}
+
+function matchGroup(groups, agent) {
+  const wanted = agent.toLowerCase();
+  for (const g of groups) {
+    if (g.agents.includes(wanted)) return g.rules;
+  }
+  return null;
+}
+
+function evaluate(rules, path) {
+  let best = { type: null, length: -1, value: "" };
+  for (const r of rules) {
+    if (!r.value) continue;
+    if (!pathMatches(path, r.value)) continue;
+    if (r.value.length > best.length) best = { ...r, length: r.value.length };
+  }
+  if (best.type === "disallow") return { allowed: false, rule: best.value };
+  return { allowed: true, rule: best.value || null };
+}
+
+function pathMatches(path, pattern) {
+  const anchored = pattern.endsWith("$");
+  const p = anchored ? pattern.slice(0, -1) : pattern;
+  const parts = p.split("*");
+  let i = 0;
+  for (let k = 0; k < parts.length; k++) {
+    const seg = parts[k];
+    if (k === 0) {
+      if (!path.startsWith(seg)) return false;
+      i = seg.length;
+    } else {
+      const found = path.indexOf(seg, i);
+      if (found === -1) return false;
+      i = found + seg.length;
+    }
+  }
+  if (anchored && i !== path.length) return false;
+  return true;
+}

package/mcp-server.js

@@ -114,7 +114,7 @@ function toolHandler(pick, extraOptions = {}) {
 
 // ── Shared params ──────────────────────────────────────────────────────
 
-const url = z.string().describe("Website URL (e.g. stripe.com)");
+const url = z.string().describe("Website URL (e.g. example.com)");
 const slow = z.boolean().optional().default(false).describe("3x timeouts for heavy SPAs");
 
 // ── Tools ──────────────────────────────────────────────────────────────

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dembrandt",
-  "version": "0.10.0",
-  "description": "Extract design tokens and brand assets from any website",
+  "version": "0.11.0",
+  "description": "Extract design tokens and publicly visible CSS information from any website",
   "mcpName": "io.github.dembrandt/dembrandt",
   "main": "index.js",
   "type": "module",
@@ -16,8 +16,6 @@
   ],
   "scripts": {
     "start": "node index.js",
-    "brand-challenge": "node run-no-login-challenge.mjs",
-    "brand-challenge:report": "node run-no-login-challenge.mjs || true",
     "install-browser": "npx playwright install chromium firefox || echo 'Playwright browser installation failed. You may need to install system dependencies manually.'",
     "local-ui": "cd local-ui && npm start",
     "qa:baseline": "node test/qa.mjs --baseline",
@@ -28,10 +26,9 @@
     "design-tokens",
     "design-system",
     "branding",
-    "web-scraping",
+    "css-analysis",
     "cli",
-    "playwright",
-    "extraction"
+    "playwright"
   ],
   "repository": {
     "type": "git",
@@ -56,4 +53,4 @@
   "engines": {
     "node": ">=18.0.0"
   }
-}
+}