@opendatalabs/darshana 1.0.0 → 1.2.0

package/README.md

@@ -1,26 +1,62 @@
 # darshana
 
-Crawl any web app and generate a labeled PDF, HTML viewer, or image set for AI-assisted design review.
+Crawl any web app and generate a labeled PDF, HTML viewer, or image set for design review.
 
 *Darśana* — Sanskrit for "the act of seeing clearly."
 
+## Try it now
+
+```bash
+npx @opendatalabs/darshana --url https://vana.org --public
+```
+
+Output lands in `./darshana-output/` — a PDF and a self-contained HTML viewer with sidebar nav, filters, and keyboard navigation.
+
+For a private app, darshana opens a browser so you can log in, then saves the session:
+
+```bash
+npx @opendatalabs/darshana --url https://app.vana.org
+# A browser opens → log in → press Enter → capture begins
+```
+
 ## Install
 
 ```bash
 npm install -g @opendatalabs/darshana
-# or use directly:
-npx @opendatalabs/darshana --config review.config.json
 ```
 
-After installing, set up Playwright's browser:
+Chromium is installed automatically. Or skip the install entirely and use `npx @opendatalabs/darshana`.
 
-```bash
-npx playwright install chromium
+## CLI reference
+
+```
+darshana --url <url> [options]         # zero-config
+darshana --config <path> [options]     # file-based (CLI args override config)
+
+--url <url>              Base URL to crawl
+--config <path>          Path to a JSON config file
+--title <string>         Review title (default: hostname)
+--start <path>           Starting path (default: /)
+--public                 Skip auth — use for public sites
+--auth-storage <path>    Where to save/load the session (default: ./darshana-output/auth.json)
+--auth-script <path>     Headless login script (see Auth below)
+--themes <list>          Comma-separated: system,dark,light (default: system)
+--viewports <list>       Comma-separated: desktop,mobile (default: desktop)
+--max-depth <n>          BFS depth limit (default: 5)
+--max-pages <n>          Page cap (default: 100)
+--delay <ms>             Wait after page load before capture (default: 400)
+--outputs <list>         Comma-separated: pdf,html,images (default: pdf,html)
+--output-dir <path>      Output directory (default: ./darshana-output)
+--include <regex>        Crawl only paths matching this pattern (repeatable)
+--exclude <regex>        Skip paths matching this pattern (repeatable)
+--dry-run                Discover URLs without capturing
+--route <path>           Capture a single route only
+--auth-only              Save auth session and exit
 ```
 
-## Quick start
+## Config file
 
-**1. Create a config file** (`review.config.json`):
+For complex projects, a JSON config gives you per-route sampling rules and capture overrides. CLI args always override config file values.
 
 ```json
 {
@@ -29,103 +65,92 @@ npx playwright install chromium
   "start": "/dashboard",
   "public": false,
   "authStorage": "./auth.json",
+  "authScript": "./auth.mjs",
   "crawl": {
     "include": ["^/dashboard"],
     "exclude": ["logout", "delete"],
     "maxDepth": 3,
     "maxPages": 50,
-    "extraRoutes": []
+    "routes": [
+      { "pattern": "/dashboard/records/:id", "sample": 1, "follow": false },
+      { "pattern": "/dashboard/runs/:id",    "sample": 2, "follow": false },
+      { "pattern": "/dashboard/**",          "follow": true }
+    ]
   },
   "capture": {
-    "themes": ["dark"],
+    "themes": ["dark", "light"],
     "viewports": ["desktop", "mobile"],
-    "delay": 400
+    "delay": 400,
+    "overrides": [
+      { "route": "/dashboard/records/", "delay": 1000 }
+    ]
   },
   "outputs": ["pdf", "html"],
   "outputDir": "./output"
 }
 ```
 
-**2. Authenticate** (opens a browser — log in, press Enter):
+### Routes DSL
 
-```bash
-npx darshana --config review.config.json --auth-only
-```
+Without routes, darshana visits every discovered URL. For apps with millions of records or runs, use routes to sample:
 
-Or provide an `authScript` for headless login (see [examples/auth-example.mjs](examples/auth-example.mjs)).
+| Field | Type | Default | Description |
+|---|---|---|---|
+| `pattern` | string | required | Express-style path using `:param` and `/**` |
+| `sample` | number | unlimited | Max pages to capture matching this pattern |
+| `follow` | boolean | `true` | Whether to BFS-follow links on matching pages |
 
-**3. Generate the review**:
+First match wins.
 
-```bash
-npx darshana --config review.config.json
-```
+### Config reference
 
-## Config reference
+**Top-level**
 
 | Field | Type | Default | Description |
 |---|---|---|---|
-| `title` | string | `"Design Review"` | Title shown on cover page and HTML header |
-| `url` | string | required | Base URL of the app |
-| `start` | string | required | Path to start crawling from |
-| `public` | boolean | `false` | Skip auth entirely for public sites |
-| `authStorage` | string | `"./auth.json"` | Path to saved Playwright storageState |
-| `authScript` | string | — | Path to a JS file that handles login programmatically |
-| `outputs` | string[] | `["pdf"]` | Any of `"pdf"`, `"html"`, `"images"` |
-| `outputDir` | string | same dir as config | Directory for generated output files |
+| `title` | string | hostname | Cover page and HTML header title |
+| `url` | string | required | Base URL |
+| `start` | string | `/` | Path to start crawling from |
+| `public` | boolean | `false` | Skip auth |
+| `authStorage` | string | `./auth.json` | Saved session path |
+| `authScript` | string | — | Headless login script |
+| `outputs` | string[] | `["pdf","html"]` | Any of `"pdf"`, `"html"`, `"images"` |
+| `outputDir` | string | `./darshana-output` | Output directory |
 
-### `crawl`
+**`crawl`**
 
 | Field | Type | Default | Description |
 |---|---|---|---|
-| `include` | string[] | `[]` | Regex patterns — URL pathname must match all |
-| `exclude` | string[] | `[]` | Regex patterns — URL pathname must not match any |
-| `maxDepth` | number | `5` | Max BFS depth from start URL |
-| `maxPages` | number | `100` | Hard cap on total pages crawled |
-| `extraRoutes` | string[] | `[]` | Additional paths to capture (not crawled for links) |
-| `routes` | Route[] | `[]` | Per-pattern sampling rules (see Routes DSL) |
+| `include` | string[] | `[]` | Regex patterns — pathname must match all |
+| `exclude` | string[] | `[]` | Regex patterns — pathname must not match any |
+| `maxDepth` | number | `5` | Max BFS depth |
+| `maxPages` | number | `100` | Hard page cap |
+| `extraRoutes` | string[] | `[]` | Extra paths to capture (not crawled for links) |
+| `routes` | Route[] | `[]` | Per-pattern sampling rules |
 
-### `capture`
+**`capture`**
 
 | Field | Type | Default | Description |
 |---|---|---|---|
-| `themes` | string[] | `["dark"]` | Theme names to capture — injected as `data-theme` + CSS class |
+| `themes` | string[] | `["system"]` | `"system"` (no injection), `"dark"`, `"light"` |
 | `viewports` | string[] | `["desktop"]` | `"desktop"` (1440×900) or `"mobile"` (390×844) |
-| `fullPage` | boolean | `true` | Capture full scrollable page height |
-| `delay` | number | `400` | ms to wait after page load before capture |
-| `waitFor` | string | — | CSS selector (prefix `$`) or JS expression to wait for |
-| `overrides` | Override[] | `[]` | Per-route capture overrides |
-| `contextOptions` | object | `{}` | Passed directly to `browser.newContext()` |
-| `launchOptions` | object | `{}` | Passed directly to `chromium.launch()` |
-| `playwrightOptions` | object | `{}` | Passed directly to `page.screenshot()` |
-| `routeOptions` | object | — | `{ blockPatterns: string[] }` — abort matching network requests |
+| `fullPage` | boolean | `true` | Capture full scrollable height |
+| `delay` | number | `400` | ms to wait before capture |
+| `waitFor` | string | — | CSS selector (prefix `$`) or JS expression to await |
+| `overrides` | Override[] | `[]` | Per-route overrides for any capture field |
+| `contextOptions` | object | `{}` | Passed to `browser.newContext()` |
+| `launchOptions` | object | `{}` | Passed to `chromium.launch()` |
+| `playwrightOptions` | object | `{}` | Passed to `page.screenshot()` |
+| `routeOptions` | object | — | `{ blockPatterns: string[] }` — abort matching requests |
 
-### Routes DSL
+## Auth
 
-Limit how many pages of each "shape" are captured. Uses Express-style `:param` notation.
+**Headed handover** (default): darshana opens a Chromium window, you log in, press Enter. The session is saved to `authStorage` and reused for 12 hours.
 
-```json
-"routes": [
-  { "pattern": "/dashboard/records/:id", "sample": 1, "follow": false },
-  { "pattern": "/dashboard/runs/:id",    "sample": 2, "follow": false },
-  { "pattern": "/dashboard/**",          "follow": true }
-]
-```
-
-| Field | Type | Default | Description |
-|---|---|---|---|
-| `pattern` | string | required | Path pattern using `:param` and `/**` |
-| `sample` | number | unlimited | Max pages to visit matching this pattern |
-| `follow` | boolean | `true` | Whether to BFS-follow links on matching pages |
-
-Patterns are matched in order — first match wins.
-
-## Auth options
-
-**Headed handover** (default when no `authScript`): darshana opens a browser, you log in manually, press Enter — session is saved to `authStorage`. Sessions are reused for 12 hours.
-
-**Headless auth script**: Create a JS file that exports a default function:
+**Headless auth script**: export a default function that receives a `Browser` and returns the path to a saved `storageState`:
 
 ```javascript
+// auth.mjs
 export default async function login(browser) {
   const context = await browser.newContext();
   const page = await context.newPage();
@@ -133,29 +158,19 @@ export default async function login(browser) {
   await page.fill('#password', process.env.APP_PASSWORD);
   await page.click('button[type="submit"]');
   await page.waitForURL(/\/dashboard/);
-  const storagePath = './auth.json';
-  await context.storageState({ path: storagePath });
+  await context.storageState({ path: './auth.json' });
   await context.close();
-  return storagePath;
+  return './auth.json';
 }
 ```
 
-Set `"authScript": "./my-auth.mjs"` in config.
-
-## CLI
-
-```bash
-darshana --config <path>          # run full pipeline
-darshana --config <path> --dry-run   # discover URLs without capturing
-darshana --config <path> --route /dashboard  # capture one route only
-darshana --config <path> --auth-only  # save auth session and exit
-```
+See [examples/auth-example.mjs](examples/auth-example.mjs) for a full example.
 
 ## Outputs
 
-- **`pdf`** — `<outputDir>/console-review.pdf` — labeled pages, cover page, one page per capture
-- **`html`** — `<outputDir>/console-review.html` — self-contained HTML with sidebar nav, filters, keyboard navigation, viewport-correct sizing
-- **`images`** — `<outputDir>/images/<viewport>/NNN-slug-theme.png` — individual screenshots grouped by viewport
+- **`pdf`** — one page per capture, labeled header, cover page
+- **`html`** — self-contained file with sidebar nav, theme/viewport filters, keyboard navigation (↑↓), viewport-correct image sizing
+- **`images`** — `<outputDir>/images/<viewport>/NNN-slug-theme.png`
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opendatalabs/darshana",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "Crawl any web app and generate a labeled PDF, HTML viewer, or image set for design review.",
   "type": "module",
   "bin": {
@@ -16,6 +16,9 @@
     "pdf-lib": "^1.17.1",
     "path-to-regexp": "^8.0.0"
   },
+  "scripts": {
+    "postinstall": "playwright install chromium --with-deps 2>/dev/null || playwright install chromium"
+  },
   "devDependencies": {
     "semantic-release": "^25.0.0",
     "@semantic-release/commit-analyzer": "^13.0.0",

package/src/capture.mjs

@@ -63,17 +63,20 @@ export async function captureAll(browser, config, urls) {
           await page.goto(url, { waitUntil: 'networkidle', timeout: 60000 });
           await page.addStyleTag({ content: NEXTJS_HIDE_STYLE }).catch(() => {});
 
-          await page.evaluate((t) => {
-            const html = document.documentElement;
-            html.setAttribute('data-theme', t);
-            if (t === 'dark') {
-              html.classList.add('dark');
-              html.classList.remove('light');
-            } else {
-              html.classList.add('light');
-              html.classList.remove('dark');
-            }
-          }, theme);
+          // 'system' = no injection; let the page's own prefers-color-scheme take effect
+          if (theme !== 'system') {
+            await page.evaluate((t) => {
+              const html = document.documentElement;
+              html.setAttribute('data-theme', t);
+              if (t === 'dark') {
+                html.classList.add('dark');
+                html.classList.remove('light');
+              } else {
+                html.classList.add('light');
+                html.classList.remove('dark');
+              }
+            }, theme);
+          }
 
           if (waitFor) {
             if (waitFor.startsWith('$')) {

package/src/config.mjs

@@ -19,10 +19,16 @@ export function loadConfig(configPath) {
   if (!raw.url) throw new Error('Config missing required field: url');
   if (!raw.start) throw new Error('Config missing required field: start');
 
+  return buildConfig(raw, configDir);
+}
+
+// Build a config object from a plain object (used by both loadConfig and CLI --url mode).
+// configDir is used to resolve relative paths; defaults to cwd when not loading from a file.
+export function buildConfig(raw, configDir = process.cwd()) {
   const config = {
     title: raw.title ?? 'Design Review',
     url: raw.url.replace(/\/$/, ''),
-    start: raw.start,
+    start: raw.start ?? '/',
     public: raw.public ?? false,
     authStorage: raw.authStorage ?? './auth.json',
     authScript: raw.authScript ?? null,
@@ -37,7 +43,7 @@ export function loadConfig(configPath) {
     },
 
     capture: {
-      themes: raw.capture?.themes ?? ['dark'],
+      themes: raw.capture?.themes ?? ['system'],
       viewports: raw.capture?.viewports ?? ['desktop'],
       fullPage: raw.capture?.fullPage ?? true,
       delay: raw.capture?.delay ?? 400,
@@ -50,11 +56,11 @@ export function loadConfig(configPath) {
     },
 
     pdf: {
-      output: raw.pdf?.output ?? './console-review.pdf',
+      output: raw.pdf?.output ?? './darshana-output/review.pdf',
       pageSize: raw.pdf?.pageSize ?? 'A4',
     },
 
-    outputs: raw.outputs ?? ['pdf'],
+    outputs: raw.outputs ?? ['pdf', 'html'],
     outputDir: raw.outputDir ? path.resolve(configDir, raw.outputDir) : null,
   };
 

package/src/index.mjs

@@ -2,33 +2,156 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { chromium } from 'playwright';
-import { loadConfig } from './config.mjs';
+import { loadConfig, buildConfig } from './config.mjs';
 import { ensureAuth } from './auth.mjs';
 import { crawl } from './crawl.mjs';
 import { captureAll } from './capture.mjs';
 import { assemblePdf } from './pdf.mjs';
 import { assembleHtml } from './html.mjs';
 
+const USAGE = `
+Usage:
+  darshana --url <url> [options]          # zero-config mode
+  darshana --config <path> [options]      # file-based config
+
+Options:
+  --url <url>              Base URL to crawl (required if no --config)
+  --config <path>          Path to a JSON config file
+  --title <string>         Review title (default: hostname)
+  --start <path>           Starting path (default: /)
+  --public                 Skip auth entirely
+  --auth-storage <path>    Path to saved Playwright storageState (default: ./auth.json)
+  --auth-script <path>     Path to a headless auth script
+  --themes <list>          Comma-separated: dark,light,system (default: system)
+  --viewports <list>       Comma-separated: desktop,mobile (default: desktop)
+  --max-depth <n>          BFS depth limit (default: 5)
+  --max-pages <n>          Page cap (default: 100)
+  --delay <ms>             Wait after page load before capture (default: 400)
+  --outputs <list>         Comma-separated: pdf,html,images (default: pdf,html)
+  --output-dir <path>      Directory for output files (default: ./darshana-output)
+  --include <regex>        Crawl only paths matching this pattern (repeatable)
+  --exclude <regex>        Skip paths matching this pattern (repeatable)
+  --dry-run                Discover URLs without capturing
+  --route <path>           Capture a single route only
+  --auth-only              Save auth session and exit
+`.trim();
+
 function parseArgs(argv) {
-  const args = { config: null, dryRun: false, route: null, authOnly: false };
+  const args = {
+    config: null,
+    url: null,
+    title: null,
+    start: null,
+    public: false,
+    authStorage: null,
+    authScript: null,
+    themes: null,
+    viewports: null,
+    maxDepth: null,
+    maxPages: null,
+    delay: null,
+    outputs: null,
+    outputDir: null,
+    include: [],
+    exclude: [],
+    dryRun: false,
+    route: null,
+    authOnly: false,
+  };
+
   for (let i = 0; i < argv.length; i++) {
-    if (argv[i] === '--config' && argv[i + 1]) { args.config = argv[++i]; continue; }
-    if (argv[i] === '--dry-run') { args.dryRun = true; continue; }
-    if (argv[i] === '--route' && argv[i + 1]) { args.route = argv[++i]; continue; }
-    if (argv[i] === '--auth-only') { args.authOnly = true; continue; }
+    const a = argv[i];
+    const next = () => { if (!argv[i + 1]) { console.error(`Missing value for ${a}`); process.exit(1); } return argv[++i]; };
+    if (a === '--config')       { args.config = next(); continue; }
+    if (a === '--url')          { args.url = next(); continue; }
+    if (a === '--title')        { args.title = next(); continue; }
+    if (a === '--start')        { args.start = next(); continue; }
+    if (a === '--public')       { args.public = true; continue; }
+    if (a === '--auth-storage') { args.authStorage = next(); continue; }
+    if (a === '--auth-script')  { args.authScript = next(); continue; }
+    if (a === '--themes')       { args.themes = next().split(',').map(s => s.trim()); continue; }
+    if (a === '--viewports')    { args.viewports = next().split(',').map(s => s.trim()); continue; }
+    if (a === '--max-depth')    { args.maxDepth = parseInt(next(), 10); continue; }
+    if (a === '--max-pages')    { args.maxPages = parseInt(next(), 10); continue; }
+    if (a === '--delay')        { args.delay = parseInt(next(), 10); continue; }
+    if (a === '--outputs')      { args.outputs = next().split(',').map(s => s.trim()); continue; }
+    if (a === '--output-dir')   { args.outputDir = next(); continue; }
+    if (a === '--include')      { args.include.push(next()); continue; }
+    if (a === '--exclude')      { args.exclude.push(next()); continue; }
+    if (a === '--dry-run')      { args.dryRun = true; continue; }
+    if (a === '--route')        { args.route = next(); continue; }
+    if (a === '--auth-only')    { args.authOnly = true; continue; }
+    if (a === '--help' || a === '-h') { console.log(USAGE); process.exit(0); }
+    console.error(`Unknown argument: ${a}\n\n${USAGE}`);
+    process.exit(1);
   }
   return args;
 }
 
+function configFromArgs(args) {
+  const url = args.url.replace(/\/$/, '');
+  const hostname = new URL(url).hostname;
+  const outputDir = args.outputDir ?? './darshana-output';
+
+  const raw = {
+    title: args.title ?? hostname,
+    url,
+    start: args.start ?? '/',
+    public: args.public,
+    authStorage: args.authStorage ?? path.join(outputDir, 'auth.json'),
+    authScript: args.authScript ?? null,
+    crawl: {
+      include: args.include,
+      exclude: args.exclude,
+      maxDepth: args.maxDepth ?? 5,
+      maxPages: args.maxPages ?? 100,
+    },
+    capture: {
+      themes: args.themes ?? ['system'],
+      viewports: args.viewports ?? ['desktop'],
+      delay: args.delay ?? 400,
+    },
+    outputs: args.outputs ?? ['pdf', 'html'],
+    outputDir,
+  };
+
+  return buildConfig(raw, process.cwd());
+}
+
+function applyCliOverrides(config, args) {
+  if (args.title)     config.title = args.title;
+  if (args.start)     config.start = args.start;
+  if (args.public)    config.public = true;
+  if (args.authStorage) config.authStorage = path.resolve(args.authStorage);
+  if (args.authScript)  config.authScript = path.resolve(args.authScript);
+  if (args.themes)    config.capture.themes = args.themes;
+  if (args.viewports) config.capture.viewports = args.viewports;
+  if (args.maxDepth)  config.crawl.maxDepth = args.maxDepth;
+  if (args.maxPages)  config.crawl.maxPages = args.maxPages;
+  if (args.delay)     config.capture.delay = args.delay;
+  if (args.outputs)   config.outputs = args.outputs;
+  if (args.outputDir) config.outputDir = path.resolve(args.outputDir);
+  if (args.include.length) config.crawl.include = [...config.crawl.include, ...args.include];
+  if (args.exclude.length) config.crawl.exclude = [...config.crawl.exclude, ...args.exclude];
+  return config;
+}
+
 const args = parseArgs(process.argv.slice(2));
 
-if (!args.config) {
-  console.error('Usage: darshana --config <path> [--dry-run] [--route <path>] [--auth-only]');
+if (!args.config && !args.url) {
+  console.error('Error: --url or --config is required\n\n' + USAGE);
   process.exit(1);
 }
 
 async function main() {
-  const config = loadConfig(args.config);
+  let config;
+  if (args.config) {
+    config = loadConfig(args.config);
+    config = applyCliOverrides(config, args);
+  } else {
+    config = configFromArgs(args);
+  }
+
   console.log(`[darshana] ${config.title} — ${config.url}`);
 
   const storageStatePath = await ensureAuth(config);
@@ -87,7 +210,7 @@ async function main() {
   const outputDir = config.outputDir;
   fs.mkdirSync(outputDir, { recursive: true });
 
-  const outputs = config.outputs ?? ['pdf'];
+  const outputs = config.outputs ?? ['pdf', 'html'];
 
   if (outputs.includes('pdf')) {
     await assemblePdf(captures, config, outputDir);