design-learn-server 0.1.1

package/src/pipeline/index.js

@@ -0,0 +1,335 @@
+const { EventEmitter } = require('events');
+const { randomUUID } = require('crypto');
+const path = require('path');
+const { pathToFileURL } = require('url');
+const { ensurePlaywrightInstalled, loadPlaywright } = require('../playwrightSupport');
+
+const DEFAULT_TIMEOUT_MS = 30000;
+
+function createExtractionPipeline({ storage }) {
+  const emitter = new EventEmitter();
+  const jobs = new Map();
+  const queue = [];
+  let running = false;
+  let closed = false;
+
+  function toPublicJob(job) {
+    return {
+      id: job.id,
+      type: job.type,
+      status: job.status,
+      progress: job.progress,
+      message: job.message,
+      createdAt: job.createdAt,
+      updatedAt: job.updatedAt,
+      result: job.result || null,
+      error: job.error || null,
+    };
+  }
+
+  function emitProgress(job, extra = {}) {
+    job.updatedAt = new Date().toISOString();
+    emitter.emit('progress', {
+      job: toPublicJob(job),
+      event: extra.event || 'progress',
+      detail: extra.detail || null,
+    });
+  }
+
+  function enqueue(type, payload) {
+    if (closed) {
+      throw new Error('pipeline_closed');
+    }
+
+    const now = new Date().toISOString();
+    const job = {
+      id: randomUUID(),
+      type,
+      status: 'queued',
+      progress: 0,
+      message: 'queued',
+      createdAt: now,
+      updatedAt: now,
+      payload,
+      result: null,
+      error: null,
+    };
+    jobs.set(job.id, job);
+    queue.push(job.id);
+    emitProgress(job, { event: 'queued' });
+    processQueue();
+    return toPublicJob(job);
+  }
+
+  function onProgress(listener) {
+    emitter.on('progress', listener);
+    return () => emitter.off('progress', listener);
+  }
+
+  function listJobs() {
+    return Array.from(jobs.values()).map((job) => toPublicJob(job));
+  }
+
+  function getJob(jobId) {
+    const job = jobs.get(jobId);
+    return job ? toPublicJob(job) : null;
+  }
+
+  function updateJob(job, progress, message, detail) {
+    if (typeof progress === 'number') {
+      job.progress = Math.max(0, Math.min(100, progress));
+    }
+    if (message) {
+      job.message = message;
+    }
+    emitProgress(job, { event: 'progress', detail });
+  }
+
+  async function processQueue() {
+    if (running || closed) {
+      return;
+    }
+    running = true;
+    while (queue.length > 0 && !closed) {
+      const jobId = queue.shift();
+      const job = jobs.get(jobId);
+      if (!job || job.status !== 'queued') {
+        continue;
+      }
+
+      job.status = 'running';
+      job.progress = 5;
+      job.message = 'started';
+      emitProgress(job, { event: 'started' });
+
+      try {
+        const result = await runJob(job, updateJob);
+        job.status = 'completed';
+        job.progress = 100;
+        job.message = 'completed';
+        job.result = result;
+        emitProgress(job, { event: 'completed' });
+      } catch (error) {
+        job.status = 'failed';
+        job.message = 'failed';
+        job.error = { message: error.message };
+        emitProgress(job, { event: 'failed' });
+      }
+    }
+    running = false;
+  }
+
+  async function runJob(job, report) {
+    if (job.type === 'import_browser') {
+      return importFromBrowser(storage, job.payload, (progress, message, detail) =>
+        report(job, progress, message, detail)
+      );
+    }
+    if (job.type === 'import_url') {
+      return importFromUrl(storage, job.payload, (progress, message, detail) =>
+        report(job, progress, message, detail)
+      );
+    }
+    throw new Error(`unknown_job_type:${job.type}`);
+  }
+
+  function close() {
+    closed = true;
+    emitter.removeAllListeners();
+    queue.length = 0;
+  }
+
+  return {
+    enqueueImportFromBrowser: (payload) => enqueue('import_browser', payload),
+    enqueueImportFromUrl: (payload) => enqueue('import_url', payload),
+    onProgress,
+    listJobs,
+    getJob,
+    close,
+  };
+}
+
+function normalizeBrowserPayload(payload = {}) {
+  const website = payload.website || {};
+  let snapshot = payload.snapshot || null;
+
+  if (!snapshot && Array.isArray(payload.snapshots) && payload.snapshots.length > 0) {
+    snapshot = payload.snapshots[0];
+  }
+
+  if (!snapshot && (payload.html || payload.css)) {
+    snapshot = payload;
+  }
+
+  const url = website.url || payload.url || (snapshot ? snapshot.url : '') || '';
+  if (!snapshot) {
+    throw new Error('snapshot_required');
+  }
+  if (!url) {
+    throw new Error('url_required');
+  }
+
+  return {
+    website: {
+      url,
+      title: website.title || payload.title || snapshot.title || '',
+      favicon: website.favicon || payload.favicon || '',
+    },
+    snapshot,
+    analysis: payload.analysis || null,
+    source: payload.source || payload.extractedFrom || 'browser-extension',
+    version: payload.version || payload.extractorVersion || '',
+    designId: payload.designId || null,
+    mergeByDomain: payload.mergeByDomain ?? true,
+  };
+}
+
+async function importFromBrowser(storage, payload, report) {
+  report(15, 'normalizing');
+  const normalized = normalizeBrowserPayload(payload);
+  report(30, 'storing_design');
+  const result = await storeImport(storage, normalized, report);
+  report(90, 'stored');
+  return result;
+}
+
+async function importFromUrl(storage, payload = {}, report) {
+  const url = payload.url || '';
+  if (!url) {
+    throw new Error('url_required');
+  }
+
+  await ensurePlaywrightInstalled({
+    onProgress: (message) => report(8, message),
+  });
+
+  report(10, 'launching_browser');
+  const snapshot = await extractWithPlaywright(url, payload.options || {}, report);
+  report(65, 'extracted');
+
+  const normalized = {
+    website: {
+      url,
+      title: snapshot.title || '',
+      favicon: '',
+    },
+    snapshot,
+    analysis: payload.analysis || null,
+    source: 'playwright',
+    version: payload.extractorVersion || snapshot.metadata?.extractorVersion || '',
+    designId: payload.designId || null,
+    mergeByDomain: payload.mergeByDomain ?? true,
+  };
+
+  report(75, 'storing_design');
+  const result = await storeImport(storage, normalized, report);
+  report(95, 'stored');
+  return result;
+}
+
+function extractDomain(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return '';
+  }
+}
+
+async function storeImport(storage, normalized, report) {
+  const { website, snapshot, analysis, source, version, designId, mergeByDomain } = normalized;
+  const url = website.url || snapshot.url || '';
+  const title = website.title || snapshot.title || url || 'Untitled';
+  const domain = extractDomain(url);
+
+  let design = null;
+  if (designId) {
+    design = await storage.getDesign(designId);
+  }
+
+  // 先按完全相同 URL 查找
+  if (!design && url) {
+    const existing = storage.listDesigns().find((item) => item.url === url);
+    if (existing) {
+      design = existing;
+    }
+  }
+
+  // 如果启用域名合并，按域名查找现有 design
+  if (!design && mergeByDomain && domain) {
+    const existing = storage.listDesigns().find((item) => extractDomain(item.url) === domain);
+    if (existing) {
+      design = existing;
+    }
+  }
+
+  if (!design) {
+    design = await storage.createDesign({
+      name: mergeByDomain && domain ? domain : title,
+      url,
+      source: source === 'playwright' ? 'script' : 'browser',
+      metadata: {
+        extractedFrom: source,
+        extractorVersion: version || '',
+        tags: [],
+      },
+    });
+  }
+
+  report(55, 'storing_version');
+  const versionRecord = await storage.createVersion({
+    designId: design.id,
+    styleguideMarkdown: analysis?.styleguide || '',
+    rules: analysis?.rules || {},
+    snapshots: snapshot ? [snapshot] : [],
+    createdBy: 'import',
+  });
+
+  return {
+    designId: design.id,
+    versionId: versionRecord.id,
+  };
+}
+
+async function extractWithPlaywright(url, options = {}, report) {
+  const playwright = await loadPlaywright({
+    onProgress: (message) => report(8, message),
+  });
+
+  const extractorPath = path.resolve(__dirname, '../../../scripts/lib/extractor.js');
+  let extractPage;
+  try {
+    const extractorModule = await import(pathToFileURL(extractorPath).href);
+    extractPage = extractorModule.extractPage;
+  } catch (error) {
+    throw new Error('extractor_script_missing');
+  }
+
+  if (typeof extractPage !== 'function') {
+    throw new Error('extractor_script_invalid');
+  }
+
+  const { chromium } = playwright;
+  const timeoutMs = Number(options.timeoutMs || DEFAULT_TIMEOUT_MS);
+  const headless = options.headless !== false;
+
+  const browser = await chromium.launch({ headless });
+  report(20, 'browser_launched');
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  try {
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+    report(40, 'page_loaded');
+    const snapshot = await extractPage(page, options.extractOptions || {});
+    report(55, 'snapshot_ready');
+    return snapshot;
+  } finally {
+    await page.close().catch(() => undefined);
+    await context.close().catch(() => undefined);
+    await browser.close().catch(() => undefined);
+  }
+}
+
+module.exports = {
+  createExtractionPipeline,
+};

package/src/playwrightSupport.js

@@ -0,0 +1,65 @@
+const { spawn } = require('child_process');
+const path = require('path');
+
+const serverRoot = path.resolve(__dirname, '..');
+const autoInstallPlaywright = process.env.DESIGN_LEARN_AUTO_INSTALL_PLAYWRIGHT !== '0';
+let installPromise = null;
+
+function isPlaywrightInstalled() {
+  try {
+    require.resolve('playwright', { paths: [serverRoot] });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function installPlaywright(logger) {
+  const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+  return new Promise((resolve, reject) => {
+    const child = spawn(npmCmd, ['install', 'playwright'], {
+      cwd: serverRoot,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    child.stdout?.on('data', (chunk) => logger?.(chunk.toString()));
+    child.stderr?.on('data', (chunk) => logger?.(chunk.toString()));
+    child.on('error', reject);
+    child.on('exit', (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`npm_install_failed:${code ?? 'unknown'}`));
+    });
+  });
+}
+
+async function ensurePlaywrightInstalled(options = {}) {
+  if (!autoInstallPlaywright) return;
+  if (isPlaywrightInstalled()) return;
+  if (typeof options.onProgress === 'function') {
+    options.onProgress('installing_playwright');
+  }
+  if (!installPromise) {
+    const logger = options.logger || ((text) => process.stderr.write(text));
+    installPromise = installPlaywright(logger).finally(() => {
+      installPromise = null;
+    });
+  }
+  await installPromise;
+}
+
+async function loadPlaywright(options = {}) {
+  try {
+    return await import('playwright');
+  } catch {
+    await ensurePlaywrightInstalled(options);
+  }
+  try {
+    return await import('playwright');
+  } catch {
+    throw new Error('playwright_not_installed');
+  }
+}
+
+module.exports = {
+  ensurePlaywrightInstalled,
+  loadPlaywright,
+};

package/src/preview/index.js

@@ -0,0 +1,204 @@
+const { EventEmitter } = require('events');
+const { randomUUID } = require('crypto');
+
+const DEFAULT_DAILY_LIMIT = Number(process.env.PREVIEW_DAILY_LIMIT || 100);
+const DEFAULT_MAX_ATTEMPTS = Number(process.env.PREVIEW_MAX_ATTEMPTS || 3);
+const DEFAULT_RETRY_DELAY_MS = Number(process.env.PREVIEW_RETRY_DELAY_MS || 500);
+
+function getDayStamp(date = new Date()) {
+  return date.toISOString().slice(0, 10);
+}
+
+function escapeXml(input = '') {
+  return String(input)
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}
+
+function createPreviewPipeline({ storage }) {
+  const emitter = new EventEmitter();
+  const jobs = new Map();
+  const queue = [];
+  let running = false;
+  let closed = false;
+  let quota = { day: getDayStamp(), count: 0 };
+
+  function toPublicJob(job) {
+    return {
+      id: job.id,
+      type: job.type,
+      status: job.status,
+      attempts: job.attempts,
+      maxAttempts: job.maxAttempts,
+      createdAt: job.createdAt,
+      updatedAt: job.updatedAt,
+      result: job.result || null,
+      error: job.error || null,
+      payload: {
+        componentId: job.payload.componentId,
+      },
+    };
+  }
+
+  function emitProgress(job, event, detail) {
+    job.updatedAt = new Date().toISOString();
+    emitter.emit('progress', {
+      job: toPublicJob(job),
+      event,
+      detail: detail || null,
+    });
+    if (event === 'completed' || event === 'failed' || event === 'retrying') {
+      console.log(`[preview] ${job.id} ${event}`);
+    }
+  }
+
+  function enqueuePreview(payload) {
+    if (closed) {
+      throw new Error('preview_pipeline_closed');
+    }
+    if (!payload || !payload.componentId) {
+      throw new Error('component_id_required');
+    }
+
+    const now = new Date().toISOString();
+    const job = {
+      id: randomUUID(),
+      type: 'preview_generate',
+      status: 'queued',
+      attempts: 0,
+      maxAttempts: DEFAULT_MAX_ATTEMPTS,
+      createdAt: now,
+      updatedAt: now,
+      payload,
+      result: null,
+      error: null,
+    };
+    jobs.set(job.id, job);
+    queue.push(job.id);
+    emitProgress(job, 'queued');
+    processQueue();
+    return toPublicJob(job);
+  }
+
+  function onProgress(listener) {
+    emitter.on('progress', listener);
+    return () => emitter.off('progress', listener);
+  }
+
+  function listJobs() {
+    return Array.from(jobs.values()).map((job) => toPublicJob(job));
+  }
+
+  function getJob(jobId) {
+    const job = jobs.get(jobId);
+    return job ? toPublicJob(job) : null;
+  }
+
+  function updateQuota() {
+    const today = getDayStamp();
+    if (quota.day !== today) {
+      quota = { day: today, count: 0 };
+    }
+    return quota;
+  }
+
+  async function processQueue() {
+    if (running || closed) {
+      return;
+    }
+    running = true;
+    while (queue.length > 0 && !closed) {
+      const jobId = queue.shift();
+      const job = jobs.get(jobId);
+      if (!job || job.status !== 'queued') {
+        continue;
+      }
+
+      const currentQuota = updateQuota();
+      if (currentQuota.count >= DEFAULT_DAILY_LIMIT) {
+        job.status = 'failed';
+        job.error = { message: 'quota_exceeded' };
+        emitProgress(job, 'failed', { reason: 'quota_exceeded' });
+        continue;
+      }
+
+      job.status = 'running';
+      job.attempts += 1;
+      currentQuota.count += 1;
+      emitProgress(job, 'started');
+
+      try {
+        const result = await runJob(job);
+        job.status = 'completed';
+        job.result = result;
+        emitProgress(job, 'completed');
+      } catch (error) {
+        if (job.attempts < job.maxAttempts) {
+          job.status = 'queued';
+          emitProgress(job, 'retrying', { attempt: job.attempts });
+          setTimeout(() => {
+            if (!closed) {
+              queue.push(job.id);
+              processQueue();
+            }
+          }, DEFAULT_RETRY_DELAY_MS);
+        } else {
+          job.status = 'failed';
+          job.error = { message: error.message };
+          emitProgress(job, 'failed');
+        }
+      }
+    }
+    running = false;
+  }
+
+  async function runJob(job) {
+    if (job.type !== 'preview_generate') {
+      throw new Error(`unknown_job_type:${job.type}`);
+    }
+    const component = await storage.getComponent(job.payload.componentId);
+    if (!component) {
+      throw new Error('component_not_found');
+    }
+    const preview = buildPreview(component);
+    await storage.updateComponentPreview(component.id, preview);
+    return { componentId: component.id, preview };
+  }
+
+  function buildPreview(component) {
+    const label = escapeXml(component.name || component.id || 'component');
+    const svg = [
+      '<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"320\" height=\"180\">',
+      '<rect width=\"100%\" height=\"100%\" fill=\"#111827\"/>',
+      `<text x=\"50%\" y=\"50%\" dominant-baseline=\"middle\" text-anchor=\"middle\" fill=\"#F9FAFB\" font-size=\"14\">${label}</text>`,
+      '</svg>',
+    ].join('');
+    const imageUrl = `data:image/svg+xml;base64,${Buffer.from(svg).toString('base64')}`;
+    return {
+      provider: 'nanobanana_stub',
+      imageUrl,
+      generatedAt: new Date().toISOString(),
+    };
+  }
+
+  function close() {
+    closed = true;
+    emitter.removeAllListeners();
+    queue.length = 0;
+  }
+
+  return {
+    enqueuePreview,
+    listJobs,
+    getJob,
+    onProgress,
+    close,
+  };
+}
+
+module.exports = {
+  createPreviewPipeline,
+};