designlang 7.0.0 → 7.2.0

package/tests/routes-reconciliation.test.js

@@ -0,0 +1,120 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { reconcileRoutes, slugForPath, formatRoutesReport } from '../src/formatters/routes-reconciliation.js';
+
+function leaf(v) { return { $value: v, $type: 'color' }; }
+
+const baseTokens = {
+  primitive: {
+    color: {
+      brand: { primary: leaf('#ff0000'), secondary: leaf('#00ff00') },
+      neutral: { 100: leaf('#ffffff') },
+    },
+  },
+  semantic: {},
+};
+
+describe('slugForPath', () => {
+  it('treats / and empty as index', () => {
+    assert.equal(slugForPath('/'), 'index');
+    assert.equal(slugForPath(''), 'index');
+    assert.equal(slugForPath(null), 'index');
+  });
+  it('slugifies standard paths', () => {
+    assert.equal(slugForPath('/pricing'), 'pricing');
+    assert.equal(slugForPath('/docs/getting-started'), 'docs-getting-started');
+  });
+});
+
+describe('reconcileRoutes', () => {
+  it('returns empty-safe output for no routes', () => {
+    const r = reconcileRoutes([]);
+    assert.equal(r.summary.routeCount, 0);
+    assert.equal(r.summary.sharedTokenCount, 0);
+    assert.deepEqual(r.perRoute, {});
+  });
+
+  it('intersects identical tokens across routes into shared', () => {
+    const routes = [
+      { url: 'https://x.com/', path: '/', tokens: baseTokens },
+      { url: 'https://x.com/about', path: '/about', tokens: baseTokens },
+    ];
+    const r = reconcileRoutes(routes);
+    assert.equal(r.summary.routeCount, 2);
+    assert.ok(r.summary.sharedTokenCount >= 3);
+    // shared tree has the primitive.color.brand.primary leaf
+    assert.equal(r.shared.primitive.color.brand.primary.$value, '#ff0000');
+    // neither route should have "added" or "changed" entries
+    assert.deepEqual(r.perRoute.index.added, {});
+    assert.deepEqual(r.perRoute.index.changed, {});
+    assert.deepEqual(r.perRoute.about.added, {});
+  });
+
+  it('emits `added` tokens for a route that has unique tokens', () => {
+    const priceTokens = {
+      primitive: {
+        color: {
+          brand: { primary: leaf('#ff0000'), secondary: leaf('#00ff00') },
+          neutral: { 100: leaf('#ffffff') },
+          accent: { gold: leaf('#ffd700') }, // unique to /pricing
+        },
+      },
+      semantic: {},
+    };
+    const routes = [
+      { url: 'https://x.com/', path: '/', tokens: baseTokens },
+      { url: 'https://x.com/pricing', path: '/pricing', tokens: priceTokens },
+    ];
+    const r = reconcileRoutes(routes);
+    assert.equal(r.perRoute.pricing.added.primitive.color.accent.gold.$value, '#ffd700');
+    // The base route has no "added" since all its tokens are shared or absent.
+    assert.deepEqual(r.perRoute.index.added, {});
+  });
+
+  it('emits `changed` when a route overrides an otherwise-shared token value', () => {
+    const darkTokens = {
+      primitive: {
+        color: {
+          brand: { primary: leaf('#aa0000'), secondary: leaf('#00ff00') }, // primary differs
+          neutral: { 100: leaf('#ffffff') },
+        },
+      },
+      semantic: {},
+    };
+    const routes = [
+      { url: 'https://x.com/', path: '/', tokens: baseTokens },
+      { url: 'https://x.com/dark', path: '/dark', tokens: darkTokens },
+    ];
+    const r = reconcileRoutes(routes);
+    // primary shouldn't be shared because values disagree
+    assert.ok(!('primary' in (r.shared.primitive?.color?.brand || {})),
+      'disagreeing primary should not be shared');
+    // both routes should have primary in their `changed` bucket
+    assert.equal(r.perRoute.index.changed.primitive.color.brand.primary.$value, '#ff0000');
+    assert.equal(r.perRoute.dark.changed.primitive.color.brand.primary.$value, '#aa0000');
+    assert.ok(r.summary.drift.length >= 2);
+  });
+
+  it('resolves slug collisions by appending a numeric suffix', () => {
+    const routes = [
+      { url: 'https://x.com/docs/a', path: '/docs-a', tokens: baseTokens },
+      { url: 'https://x.com/docs-a/', path: '/docs-a', tokens: baseTokens },
+    ];
+    const r = reconcileRoutes(routes);
+    const slugs = Object.keys(r.perRoute);
+    assert.equal(slugs.length, 2);
+    assert.ok(slugs.includes('docs-a'));
+    assert.ok(slugs.some(s => s.startsWith('docs-a-')));
+  });
+
+  it('formatRoutesReport produces readable markdown', () => {
+    const routes = [
+      { url: 'https://x.com/', path: '/', tokens: baseTokens },
+      { url: 'https://x.com/pricing', path: '/pricing', tokens: baseTokens },
+    ];
+    const md = formatRoutesReport(reconcileRoutes(routes));
+    assert.match(md, /Routes crawled.*2/);
+    assert.match(md, /Shared tokens/);
+    assert.match(md, /pricing/);
+  });
+});

package/tests/wide-gamut.test.js

@@ -0,0 +1,90 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { oklchToSrgb, oklabToSrgb, oklchLikeToHex, parseOklchOrOklab, rgbToHex } from '../src/utils/color-gamut.js';
+import { extractWideGamut } from '../src/extractors/wide-gamut.js';
+import { extractTokenSources } from '../src/extractors/token-sources.js';
+
+describe('color-gamut', () => {
+  it('converts oklch(1 0 0) to white', () => {
+    const hex = oklchLikeToHex('oklch(1 0 0)');
+    assert.equal(hex, '#ffffff');
+  });
+
+  it('converts oklch(0 0 0) to black', () => {
+    const hex = oklchLikeToHex('oklch(0 0 0)');
+    assert.equal(hex, '#000000');
+  });
+
+  it('converts a mid-chroma oklch to approximate sRGB', () => {
+    // oklch(62.8% 0.258 29.23) ≈ red (#ff0000) region
+    const hex = oklchLikeToHex('oklch(62.8% 0.258 29.23)');
+    assert.ok(/^#[0-9a-f]{6}$/.test(hex));
+    // Red channel should dominate
+    const r = parseInt(hex.slice(1, 3), 16);
+    const g = parseInt(hex.slice(3, 5), 16);
+    const b = parseInt(hex.slice(5, 7), 16);
+    assert.ok(r > g, `expected r>g, got ${hex}`);
+    assert.ok(r > b, `expected r>b, got ${hex}`);
+  });
+
+  it('parses oklab values with percentages', () => {
+    const p = parseOklchOrOklab('oklab(62.8% 0.1 -0.1)');
+    assert.equal(p.type, 'oklab');
+    assert.ok(Math.abs(p.L - 0.628) < 1e-6);
+  });
+
+  it('rgbToHex clamps and formats correctly', () => {
+    assert.equal(rgbToHex(1, 0, 0), '#ff0000');
+    assert.equal(rgbToHex(0, 1, 0), '#00ff00');
+    assert.equal(rgbToHex(1.5, -0.5, 0.5), '#ff0080');
+  });
+});
+
+describe('extractWideGamut', () => {
+  it('returns zeros for empty input', () => {
+    const r = extractWideGamut([]);
+    assert.equal(r.totalCount, 0);
+    assert.equal(r.oklch.count, 0);
+  });
+
+  it('buckets colors by type and emits hex for oklch samples', () => {
+    const r = extractWideGamut([
+      { raw: 'oklch(62.8% 0.258 29.23)', type: 'oklch', property: 'color', selector: '.btn' },
+      { raw: 'color-mix(in oklab, red, blue)', type: 'color-mix', property: 'background', selector: '.x' },
+      { raw: 'light-dark(white, black)', type: 'light-dark', property: 'color', selector: '.y' },
+      { raw: 'color(display-p3 1 0 0)', type: 'display-p3', property: 'background', selector: '.p3' },
+    ]);
+    assert.equal(r.oklch.count, 1);
+    assert.ok(r.oklch.samples[0].value?.startsWith('#'));
+    assert.equal(r.colorMix.count, 1);
+    assert.equal(r.lightDark.count, 1);
+    assert.equal(r.displayP3.count, 1);
+    assert.equal(r.totalCount, 4);
+  });
+});
+
+describe('extractTokenSources', () => {
+  it('returns an array of token→sourceUrl entries based on first matching element', () => {
+    const design = {
+      colors: { primary: { hex: '#0072f5' }, text: ['#111111'] },
+      typography: { families: [{ name: 'Geist' }] },
+      spacing: { base: 8 },
+      borders: { radii: [{ value: '8px' }] },
+    };
+    const styles = [
+      { color: 'rgb(0, 114, 245)', fontFamily: '"Geist", sans-serif', paddingTop: '8px', borderRadius: '8px', sources: [{ url: 'https://cdn.example/app.css', mediaText: '' }] },
+      { color: 'rgb(17, 17, 17)', sources: [{ url: 'https://cdn.example/text.css', mediaText: '' }] },
+    ];
+    const out = extractTokenSources(design, styles);
+    const primary = out.find(t => t.token === 'color.primary');
+    assert.equal(primary.sourceUrl, 'https://cdn.example/app.css');
+    const text = out.find(t => t.token === 'color.text');
+    assert.equal(text.sourceUrl, 'https://cdn.example/text.css');
+    const font = out.find(t => t.token === 'font.body');
+    assert.equal(font.sourceUrl, 'https://cdn.example/app.css');
+    const spacing = out.find(t => t.token === 'spacing.base');
+    assert.equal(spacing.sourceUrl, 'https://cdn.example/app.css');
+    const radius = out.find(t => t.token === 'radius.base');
+    assert.equal(radius.sourceUrl, 'https://cdn.example/app.css');
+  });
+});

package/website/app/api/extract/route.js

@@ -1,18 +1,50 @@
+// POST /api/extract
+// Streaming NDJSON: one JSON event per line.
+// Events:
+//   { type:'cache', cached:true }               — if served from Blob
+//   { type:'stage', name:'crawl'|... }           — progress markers
+//   { type:'token', category, path, value, $type } — one per semantic token
+//   { type:'summary', summary }
+//   { type:'files', files }                      — final, full file map
+//   { type:'error', error }                      — terminal failure
+
 import { extractDesignLanguage } from '../../../../src/index.js';
 import { formatMarkdown } from '../../../../src/formatters/markdown.js';
-import { formatTokens } from '../../../../src/formatters/tokens.js';
 import { formatTailwind } from '../../../../src/formatters/tailwind.js';
 import { formatCssVars } from '../../../../src/formatters/css-vars.js';
 import { formatPreview } from '../../../../src/formatters/preview.js';
 import { formatFigma } from '../../../../src/formatters/figma.js';
 import { formatReactTheme, formatShadcnTheme } from '../../../../src/formatters/theme.js';
+import { formatWordPress, formatWordPressTheme } from '../../../../src/formatters/wordpress.js';
+import { formatDtcgTokens } from '../../../../src/formatters/dtcg-tokens.js';
+import { formatIosSwiftUI } from '../../../../src/formatters/ios-swiftui.js';
+import { formatAndroidCompose } from '../../../../src/formatters/android-compose.js';
+import { formatFlutterDart } from '../../../../src/formatters/flutter-dart.js';
+import { formatAgentRules } from '../../../../src/formatters/agent-rules.js';
 import { nameFromUrl } from '../../../../src/utils.js';
 
-export const maxDuration = 60;
+import { validateTargetUrl } from '../../../../website/lib/url-safety.js';
+import { checkRate } from '../../../../website/lib/rate-limit.js';
+import { cacheKey, getCached, putCached } from '../../../../website/lib/cache.js';
+
+export const runtime = 'nodejs';
 export const dynamic = 'force-dynamic';
+export const maxDuration = 60;
+
+const STAGES = [
+  'crawl',
+  'colors',
+  'typography',
+  'spacing',
+  'shadows',
+  'borders',
+  'components',
+  'regions',
+  'a11y',
+  'score',
+];
 
 async function getBrowserOptions() {
-  // On Vercel/Lambda, use @sparticuz/chromium; locally, use playwright's bundled browser
   if (process.env.VERCEL || process.env.AWS_LAMBDA_FUNCTION_NAME) {
     const chromium = (await import('@sparticuz/chromium')).default;
     return {
@@ -23,63 +55,191 @@ async function getBrowserOptions() {
   return {};
 }
 
+function ndjson(obj) {
+  return new TextEncoder().encode(JSON.stringify(obj) + '\n');
+}
+
+// Walk a DTCG token tree and yield every leaf ({ $value, $type }).
+function* walkDtcgTokens(tree, path = []) {
+  if (!tree || typeof tree !== 'object') return;
+  if (tree.$value !== undefined && tree.$type !== undefined) {
+    yield { path: path.join('.'), value: tree.$value, $type: tree.$type };
+    return;
+  }
+  for (const key of Object.keys(tree)) {
+    if (key.startsWith('$')) continue;
+    yield* walkDtcgTokens(tree[key], [...path, key]);
+  }
+}
+
+function buildSummary(design) {
+  return {
+    url: design.meta?.url,
+    title: design.meta?.title,
+    colors: design.colors?.all?.length ?? 0,
+    colorList: (design.colors?.all || []).slice(0, 20).map((c) => c.hex),
+    fonts: design.typography?.families?.map((f) => f.name).join(', ') || 'none detected',
+    spacingCount: design.spacing?.scale?.length ?? 0,
+    spacingBase: design.spacing?.base ?? null,
+    shadowCount: design.shadows?.values?.length ?? 0,
+    radiiCount: design.borders?.radii?.length ?? 0,
+    componentCount: Object.keys(design.components || {}).length,
+    cssVarCount: Object.values(design.variables || {}).reduce((s, v) => s + Object.keys(v).length, 0),
+    a11yScore: design.accessibility?.score ?? null,
+    a11yFailCount: design.accessibility?.failCount ?? 0,
+    score: design.score,
+  };
+}
+
+function buildFiles(design, targetUrl) {
+  const prefix = nameFromUrl(targetUrl);
+  const dtcg = formatDtcgTokens(design);
+  const dtcgJson = JSON.stringify(dtcg, null, 2);
+
+  const files = {
+    [`${prefix}-design-language.md`]: formatMarkdown(design),
+    [`${prefix}-design-tokens.json`]: dtcgJson,
+    [`${prefix}-tailwind.config.js`]: formatTailwind(design),
+    [`${prefix}-variables.css`]: formatCssVars(design),
+    [`${prefix}-preview.html`]: formatPreview(design),
+    [`${prefix}-figma-variables.json`]: formatFigma(design),
+    [`${prefix}-theme.js`]: formatReactTheme(design),
+    [`${prefix}-shadcn-theme.css`]: formatShadcnTheme(design),
+    [`${prefix}-wordpress-theme.json`]: formatWordPress(design),
+  };
+
+  // MCP companion JSON — same subset the CLI writes.
+  files[`${prefix}-mcp.json`] = JSON.stringify({
+    colors: { all: design.colors?.all || [] },
+    regions: design.regions || [],
+    componentClusters: design.componentClusters || [],
+    accessibility: { remediation: design.accessibility?.remediation || [] },
+    cssHealth: design.cssHealth || null,
+  }, null, 2);
+
+  // iOS
+  files['ios/DesignTokens.swift'] = formatIosSwiftUI(dtcg);
+
+  // Android (returns { filename: content })
+  const android = formatAndroidCompose(dtcg);
+  for (const name of Object.keys(android)) {
+    files[`android/${name}`] = android[name];
+  }
+
+  // Flutter
+  files['flutter/design_tokens.dart'] = formatFlutterDart(dtcg);
+
+  // WordPress block theme (5 files)
+  const wpTheme = formatWordPressTheme(dtcg, design);
+  for (const name of Object.keys(wpTheme)) {
+    files[`wordpress-theme/${name}`] = wpTheme[name];
+  }
+
+  // Agent rules
+  const agentFiles = formatAgentRules({ design, tokens: dtcg, url: targetUrl });
+  for (const name of Object.keys(agentFiles)) {
+    files[name] = agentFiles[name];
+  }
+
+  return { files, dtcg };
+}
+
+function extractIp(request) {
+  const xff = request.headers.get('x-forwarded-for');
+  if (xff) return xff.split(',')[0].trim();
+  const real = request.headers.get('x-real-ip');
+  if (real) return real.trim();
+  return 'unknown';
+}
+
+// Emit cached payload as a simulated stream so the hero paints consistently.
+async function streamCached(controller, cached, targetUrl) {
+  controller.enqueue(ndjson({ type: 'cache', cached: true }));
+  for (const stage of STAGES) {
+    controller.enqueue(ndjson({ type: 'stage', name: stage }));
+    await new Promise((r) => setTimeout(r, 40));
+  }
+  // Re-derive DTCG tokens from the cached design so the token-by-token paint
+  // still happens on a cache hit.
+  const { files, dtcg } = buildFiles(cached.design, targetUrl);
+  for (const { path, value, $type } of walkDtcgTokens(dtcg)) {
+    const category = path.split('.')[1] || 'misc';
+    controller.enqueue(ndjson({ type: 'token', category, path, value, $type }));
+  }
+  controller.enqueue(ndjson({ type: 'summary', summary: buildSummary(cached.design) }));
+  controller.enqueue(ndjson({ type: 'files', files }));
+}
+
 export async function POST(request) {
-  try {
-    const { url } = await request.json();
-
-    if (!url) {
-      return Response.json({ error: 'URL is required' }, { status: 400 });
-    }
-
-    let targetUrl = url;
-    if (!targetUrl.startsWith('http')) targetUrl = `https://${targetUrl}`;
-
-    // Validate URL
-    try {
-      new URL(targetUrl);
-    } catch {
-      return Response.json({ error: 'Invalid URL' }, { status: 400 });
-    }
-
-    const browserOpts = await getBrowserOptions();
-    const design = await extractDesignLanguage(targetUrl, browserOpts);
-
-    const prefix = nameFromUrl(targetUrl);
-
-    const files = {
-      [`${prefix}-design-language.md`]: formatMarkdown(design),
-      [`${prefix}-design-tokens.json`]: formatTokens(design),
-      [`${prefix}-tailwind.config.js`]: formatTailwind(design),
-      [`${prefix}-variables.css`]: formatCssVars(design),
-      [`${prefix}-preview.html`]: formatPreview(design),
-      [`${prefix}-figma-variables.json`]: formatFigma(design),
-      [`${prefix}-theme.js`]: formatReactTheme(design),
-      [`${prefix}-shadcn-theme.css`]: formatShadcnTheme(design),
-    };
+  let body;
+  try { body = await request.json(); } catch {
+    return Response.json({ error: 'Invalid JSON body' }, { status: 400 });
+  }
 
-    const summary = {
-      url: design.meta.url,
-      title: design.meta.title,
-      colors: design.colors.all.length,
-      colorList: design.colors.all.slice(0, 20).map(c => c.hex),
-      fonts: design.typography.families.map(f => f.name).join(', ') || 'none detected',
-      spacingCount: design.spacing.scale.length,
-      spacingBase: design.spacing.base,
-      shadowCount: design.shadows.values.length,
-      radiiCount: design.borders.radii.length,
-      componentCount: Object.keys(design.components).length,
-      cssVarCount: Object.values(design.variables).reduce((s, v) => s + Object.keys(v).length, 0),
-      a11yScore: design.accessibility?.score ?? null,
-      a11yFailCount: design.accessibility?.failCount ?? 0,
-      score: design.score,
-    };
+  const validation = validateTargetUrl(body?.url);
+  if (!validation.ok) {
+    return Response.json({ error: validation.reason }, { status: validation.status });
+  }
+  const targetUrl = validation.url;
 
-    return Response.json({ summary, files });
-  } catch (err) {
-    console.error('Extraction failed:', err);
+  const ip = extractIp(request);
+  const rate = checkRate(`extract:${ip}`);
+  if (!rate.allowed) {
     return Response.json(
-      { error: err.message || 'Extraction failed' },
-      { status: 500 }
+      { error: 'Rate limit — 3 extractions per day. Try again later.', resetAt: rate.resetAt },
+      { status: 429 }
     );
   }
+
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        const key = cacheKey(targetUrl);
+        const cached = await getCached(key);
+        if (cached) {
+          await streamCached(controller, cached, targetUrl);
+          controller.close();
+          return;
+        }
+
+        // Pre-stage markers — best-effort progress since extraction is atomic.
+        controller.enqueue(ndjson({ type: 'stage', name: 'crawl' }));
+
+        const browserOpts = await getBrowserOptions();
+        const design = await extractDesignLanguage(targetUrl, browserOpts);
+
+        // Post-stage markers once extraction resolves.
+        for (const stage of STAGES.slice(1)) {
+          controller.enqueue(ndjson({ type: 'stage', name: stage }));
+        }
+
+        const { files, dtcg } = buildFiles(design, targetUrl);
+
+        // Token-by-token paint — every DTCG leaf becomes its own event.
+        for (const { path, value, $type } of walkDtcgTokens(dtcg)) {
+          const category = path.split('.')[1] || 'misc';
+          controller.enqueue(ndjson({ type: 'token', category, path, value, $type }));
+        }
+
+        controller.enqueue(ndjson({ type: 'summary', summary: buildSummary(design) }));
+        controller.enqueue(ndjson({ type: 'files', files }));
+
+        // Fire-and-forget cache write.
+        putCached(key, { design }).catch(() => {});
+      } catch (err) {
+        console.error('[extract] failed', { url: targetUrl, ip, message: err?.message });
+        controller.enqueue(ndjson({ type: 'error', error: err?.message || 'Extraction failed' }));
+      } finally {
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      'content-type': 'application/x-ndjson; charset=utf-8',
+      'cache-control': 'no-store',
+      'x-accel-buffering': 'no',
+    },
+  });
 }