@dench.com/cli 2.7.8 → 2.7.9-staging.86

package/dench.ts

@@ -517,6 +517,15 @@ Live web search (Exa via the Dench Gateway, auths with the active \`dench signin
   dench search answer "<query>" [--json]
   Help: dench search help
 
+Dench web (page read, brand, sitemap, crawl, screenshot, extract — prefer these over writing your own scraper):
+  dench web scrape <url> [--json]
+  dench web brand <domain|email> [--json]
+  dench web sitemap <domain> [--limit N] [--json]
+  dench web crawl <url> [--max-pages N] [--json]
+  dench web screenshot <url> [--full-page] [--json]
+  dench web extract <url> --schema '<json>' [--json]
+  Help: dench web help
+
 Cloud browser (runs entirely in Dench Cloud):
   dench browser open [url] [--json]
   dench browser inspect [--json]
@@ -4240,6 +4249,26 @@ async function main() {
     return;
   }
 
+  if (command === "web") {
+    const subArgs = args.slice(args.indexOf("web") + 1);
+    const filteredSubArgs = subArgs.filter((a) => a !== "--json");
+    const { runWebCommand, webHelp } = await import("./web");
+    const sub = filteredSubArgs[0];
+    if (!sub || sub === "help" || sub === "--help" || sub === "-h") {
+      webHelp();
+      return;
+    }
+    const runtime = await getRuntime();
+    const gatewayAuth = await resolveGatewayCommandAuth(runtime, "dench web");
+    await runWebCommand({
+      args: filteredSubArgs,
+      jsonOutput: json,
+      bearerToken: gatewayAuth.bearerToken,
+      gatewayBaseUrl: gatewayAuth.gatewayBaseUrl,
+    });
+    return;
+  }
+
   if (command === "search") {
     const subArgs = args.slice(args.indexOf("search") + 1);
     const filteredSubArgs = subArgs.filter((a) => a !== "--json");

package/lib/api-schemas.ts

@@ -2963,6 +2963,29 @@ export const requestExamples: Record<string, unknown> = {
     schedule: { kind: "every", everyMs: 86_400_000 },
   },
   "search.web": { query: "latest AI CRM workflows", numResults: 5 },
+  "web.scrape": {
+    input: { url: "https://example.com", useMainContentOnly: true },
+  },
+  "web.brand": { input: { domain: "stripe.com" } },
+  "web.sitemap": { input: { domain: "example.com", maxLinks: 1000 } },
+  "web.crawl": {
+    input: { url: "https://example.com", maxPages: 15, useMainContentOnly: true },
+  },
+  "web.screenshot": {
+    input: { directUrl: "https://example.com", fullScreenshot: false },
+  },
+  "web.extract": {
+    input: {
+      url: "https://example.com",
+      schema: { type: "object", properties: { title: { type: "string" } } },
+    },
+  },
+  "web.product": { input: { url: "https://example.com/product" } },
+  "web.competitors": { input: { domain: "stripe.com", numCompetitors: 5 } },
+  "web.styleguide": { input: { domain: "stripe.com" } },
+  "web.fonts": { input: { domain: "stripe.com" } },
+  "web.classify": { input: { input: "stripe.com" } },
+  "web.brandSearch": { input: { query: "stripe" } },
   "image.generate": {
     prompt: "A minimal flat-vector app icon",
     size: "1024x1024",

package/lib/command-registry.ts

@@ -2220,6 +2220,138 @@ const rawApiOperations = [
     responseSchema: anyResponse,
     backend: gateway("POST", "/search/answer"),
   }),
+  op({
+    id: "web.scrape",
+    group: "gateway",
+    summary: "Read a page as clean Markdown through Dench web.",
+    cli: "dench web scrape",
+    method: "POST",
+    path: "/context/web-scrape-markdown",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-scrape-markdown"),
+  }),
+  op({
+    id: "web.brand",
+    group: "gateway",
+    summary: "Look up a brand kit from a domain or email.",
+    cli: "dench web brand",
+    method: "POST",
+    path: "/context/brand-retrieve",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/brand-retrieve"),
+  }),
+  op({
+    id: "web.sitemap",
+    group: "gateway",
+    summary: "List sitemap URLs for a domain.",
+    cli: "dench web sitemap",
+    method: "POST",
+    path: "/context/web-sitemap",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-sitemap"),
+  }),
+  op({
+    id: "web.crawl",
+    group: "gateway",
+    summary: "Crawl a site to Markdown (capped).",
+    cli: "dench web crawl",
+    method: "POST",
+    path: "/context/web-crawl",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-crawl"),
+  }),
+  op({
+    id: "web.screenshot",
+    group: "gateway",
+    summary: "Capture a rendered page screenshot.",
+    cli: "dench web screenshot",
+    method: "POST",
+    path: "/context/web-screenshot",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-screenshot"),
+  }),
+  op({
+    id: "web.extract",
+    group: "gateway",
+    summary: "Extract typed fields from a page using a JSON Schema.",
+    cli: "dench web extract",
+    method: "POST",
+    path: "/context/web-extract",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-extract"),
+  }),
+  op({
+    id: "web.product",
+    group: "gateway",
+    summary: "Extract a product listing from a URL.",
+    cli: "dench web product",
+    method: "POST",
+    path: "/context/web-product",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-product"),
+  }),
+  op({
+    id: "web.competitors",
+    group: "gateway",
+    summary: "Find competitors of a domain.",
+    cli: "dench web competitors",
+    method: "POST",
+    path: "/context/web-competitors",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/web-competitors"),
+  }),
+  op({
+    id: "web.styleguide",
+    group: "gateway",
+    summary: "Read a brand styleguide from a domain.",
+    cli: "dench web styleguide",
+    method: "POST",
+    path: "/context/brand-styleguide",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/brand-styleguide"),
+  }),
+  op({
+    id: "web.fonts",
+    group: "gateway",
+    summary: "Read site fonts from a domain.",
+    cli: "dench web fonts",
+    method: "POST",
+    path: "/context/brand-fonts",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/brand-fonts"),
+  }),
+  op({
+    id: "web.classify",
+    group: "gateway",
+    summary: "Classify a domain with NAICS or SIC.",
+    cli: "dench web classify",
+    method: "POST",
+    path: "/context/classify-naics",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/classify-naics"),
+  }),
+  op({
+    id: "web.brandSearch",
+    group: "gateway",
+    summary: "Autocomplete brand names.",
+    cli: "dench web brand-search",
+    method: "POST",
+    path: "/context/brand-search",
+    requestSchema: anyObject,
+    responseSchema: anyResponse,
+    backend: gateway("POST", "/context/brand-search"),
+  }),
   op({
     id: "image.generate",
     group: "gateway",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dench.com/cli",
-  "version": "2.7.8",
+  "version": "2.7.9-staging.86",
   "description": "Dench agent workspace CLI.",
   "type": "module",
   "bin": {
@@ -30,6 +30,7 @@
     "browser.ts",
     "ads.ts",
     "search.ts",
+    "web.ts",
     "image.ts",
     "tools.ts",
     "apps.ts",

package/web.ts

@@ -0,0 +1,356 @@
+/**
+ * `dench web` — brand, scrape, sitemap, crawl, screenshot, and extract
+ * through the Dench Cloud Gateway (`POST /v1/context/{op}`).
+ *
+ * Agents and humans use this instead of writing their own scrapers.
+ * Auth matches `dench search`: org DENCH_API_KEY / paid-workspace gateway key.
+ */
+
+import { assertNoUnknownFlags, getFlag, hasFlag } from "./lib/cli-args";
+
+type WebCliContext = {
+  args: string[];
+  jsonOutput: boolean;
+  bearerToken?: string;
+  gatewayBaseUrl?: string;
+};
+
+class WebCliError extends Error {}
+
+const CRAWL_DEFAULT_PAGES = 15;
+const CRAWL_MAX_PAGES = 50;
+
+function parsePositiveInt(name: string, raw: string | undefined): number | undefined {
+  if (raw === undefined) return undefined;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    throw new WebCliError(`Invalid ${name} value: ${raw}`);
+  }
+  return Math.floor(parsed);
+}
+
+function normalizeGatewayBaseUrl(value: string): string {
+  return value.replace(/\/+$/, "").replace(/\/v1$/, "");
+}
+
+function resolveGatewayBaseUrl(
+  ctx?: Pick<WebCliContext, "gatewayBaseUrl">,
+): string {
+  if (ctx?.gatewayBaseUrl?.trim()) {
+    return normalizeGatewayBaseUrl(ctx.gatewayBaseUrl.trim());
+  }
+  const explicit =
+    process.env.DENCH_GATEWAY_URL?.trim() || process.env.GATEWAY_URL?.trim();
+  if (explicit) return normalizeGatewayBaseUrl(explicit);
+  const host = process.env.DENCH_HOST?.trim();
+  if (host && /localhost|127\.0\.0\.1/.test(host)) {
+    return "http://localhost:8787";
+  }
+  return "https://gateway.merseoriginals.com";
+}
+
+function requireBearerToken(ctx?: Pick<WebCliContext, "bearerToken">): string {
+  const bearerToken =
+    ctx?.bearerToken?.trim() || process.env.DENCH_API_KEY?.trim();
+  if (!bearerToken) {
+    throw new WebCliError(
+      "No Dench gateway credential found. Set DENCH_API_KEY or run `dench signin` for a paid workspace.",
+    );
+  }
+  return bearerToken;
+}
+
+async function callWebOp(
+  ctx: WebCliContext,
+  op: string,
+  input: Record<string, unknown>,
+): Promise<unknown> {
+  const bearerToken = requireBearerToken(ctx);
+  const baseUrl = resolveGatewayBaseUrl(ctx);
+  const response = await fetch(`${baseUrl}/v1/context/${op}`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      Authorization: `Bearer ${bearerToken}`,
+    },
+    body: JSON.stringify({ input }),
+  });
+  if (!response.ok) {
+    let detail = "";
+    try {
+      detail = await response.text();
+    } catch {
+      // ignore
+    }
+    const truncated = detail.length > 500 ? `${detail.slice(0, 500)}…` : detail;
+    throw new WebCliError(
+      `Gateway /v1/context/${op} returned ${response.status}: ${truncated || response.statusText}`,
+    );
+  }
+  return response.json();
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function out(ctx: WebCliContext, payload: unknown, formatted: string): void {
+  if (ctx.jsonOutput) {
+    console.log(JSON.stringify(payload, null, 2));
+    return;
+  }
+  console.log(formatted);
+}
+
+function formatBrand(data: unknown): string {
+  const record = asRecord(data);
+  if (!record) return "(empty brand)";
+  const lines = [
+    asString(record.title) ?? asString(record.name) ?? "(untitled brand)",
+    asString(record.domain) ? `  ${asString(record.domain)}` : "",
+    asString(record.description) ? `  ${asString(record.description)}` : "",
+  ].filter((line) => line.length > 0);
+  return lines.join("\n");
+}
+
+function formatMarkdown(data: unknown): string {
+  const record = asRecord(data);
+  return asString(record?.markdown) ?? JSON.stringify(data, null, 2);
+}
+
+export function webHelp(): void {
+  console.log(`Dench web
+
+Read pages, brand kits, sitemaps, screenshots, and structured extracts
+through the Dench gateway. Prefer these over writing your own scraper.
+
+  dench web scrape <url> [--json]
+  dench web brand <domain|email> [--json]
+  dench web sitemap <domain> [--limit N] [--json]
+  dench web crawl <url> [--max-pages N] [--json]
+  dench web screenshot <url> [--full-page] [--json]
+  dench web extract <url> --schema '<json>' [--json]
+  dench web product <url> [--json]
+  dench web competitors <domain> [--json]
+  dench web styleguide <domain> [--json]
+  dench web fonts <domain> [--json]
+  dench web classify <domain> [--scheme naics|sic] [--json]
+  dench web brand-search <query> [--json]
+
+Crawl defaults to 15 pages and is hard-capped at 50. Search stays on
+\`dench search\` (Exa). \`dench context\` is workspace orientation, not this.
+`);
+}
+
+async function runWebSubcommand(ctx: WebCliContext): Promise<void> {
+  const sub = ctx.args[0];
+  const rest = ctx.args.slice(1);
+
+  if (!sub || sub === "help" || sub === "--help" || sub === "-h") {
+    webHelp();
+    return;
+  }
+
+  if (sub === "scrape") {
+    assertNoUnknownFlags(rest, "web scrape");
+    const url = rest[0];
+    if (!url) throw new WebCliError("Usage: dench web scrape <url> [--json]");
+    const data = await callWebOp(ctx, "web-scrape-markdown", {
+      url,
+      useMainContentOnly: true,
+    });
+    out(ctx, data, formatMarkdown(data));
+    return;
+  }
+
+  if (sub === "brand") {
+    assertNoUnknownFlags(rest, "web brand");
+    const identifier = rest[0];
+    if (!identifier) {
+      throw new WebCliError("Usage: dench web brand <domain|email> [--json]");
+    }
+    const input = identifier.includes("@")
+      ? { email: identifier }
+      : { domain: identifier };
+    const data = await callWebOp(ctx, "brand-retrieve", input);
+    out(ctx, data, formatBrand(data));
+    return;
+  }
+
+  if (sub === "brand-search") {
+    assertNoUnknownFlags(rest, "web brand-search");
+    const query = rest.join(" ").trim();
+    if (!query) {
+      throw new WebCliError("Usage: dench web brand-search <query> [--json]");
+    }
+    const data = await callWebOp(ctx, "brand-search", { query });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "sitemap") {
+    const limit = parsePositiveInt("--limit", getFlag(rest, "--limit"));
+    assertNoUnknownFlags(rest, "web sitemap");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError("Usage: dench web sitemap <domain> [--limit N] [--json]");
+    }
+    const data = await callWebOp(ctx, "web-sitemap", {
+      domain,
+      ...(limit != null ? { maxLinks: limit } : {}),
+    });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "crawl") {
+    const maxPages =
+      parsePositiveInt("--max-pages", getFlag(rest, "--max-pages")) ??
+      CRAWL_DEFAULT_PAGES;
+    if (maxPages > CRAWL_MAX_PAGES) {
+      throw new WebCliError(`--max-pages cannot exceed ${CRAWL_MAX_PAGES}.`);
+    }
+    assertNoUnknownFlags(rest, "web crawl");
+    const url = rest[0];
+    if (!url) {
+      throw new WebCliError(
+        "Usage: dench web crawl <url> [--max-pages N] [--json]",
+      );
+    }
+    const data = await callWebOp(ctx, "web-crawl", {
+      url,
+      maxPages,
+      useMainContentOnly: true,
+    });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "screenshot") {
+    const fullPage = hasFlag(rest, "--full-page");
+    assertNoUnknownFlags(rest, "web screenshot");
+    const url = rest[0];
+    if (!url) {
+      throw new WebCliError(
+        "Usage: dench web screenshot <url> [--full-page] [--json]",
+      );
+    }
+    const data = await callWebOp(ctx, "web-screenshot", {
+      directUrl: url,
+      fullScreenshot: fullPage,
+    });
+    const record = asRecord(data);
+    out(ctx, data, asString(record?.screenshot) ?? JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "extract") {
+    const schemaRaw = getFlag(rest, "--schema");
+    assertNoUnknownFlags(rest, "web extract");
+    const url = rest[0];
+    if (!url || !schemaRaw) {
+      throw new WebCliError(
+        "Usage: dench web extract <url> --schema '<json>' [--json]",
+      );
+    }
+    let schema: unknown;
+    try {
+      schema = JSON.parse(schemaRaw);
+    } catch {
+      throw new WebCliError("--schema must be valid JSON.");
+    }
+    if (!schema || typeof schema !== "object" || Array.isArray(schema)) {
+      throw new WebCliError("--schema must be a JSON object.");
+    }
+    const data = await callWebOp(ctx, "web-extract", { url, schema });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "product") {
+    assertNoUnknownFlags(rest, "web product");
+    const url = rest[0];
+    if (!url) throw new WebCliError("Usage: dench web product <url> [--json]");
+    const data = await callWebOp(ctx, "web-product", { url });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "competitors") {
+    assertNoUnknownFlags(rest, "web competitors");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError("Usage: dench web competitors <domain> [--json]");
+    }
+    const data = await callWebOp(ctx, "web-competitors", {
+      domain,
+      numCompetitors: 5,
+    });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "styleguide") {
+    assertNoUnknownFlags(rest, "web styleguide");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError("Usage: dench web styleguide <domain> [--json]");
+    }
+    const data = await callWebOp(ctx, "brand-styleguide", { domain });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "fonts") {
+    assertNoUnknownFlags(rest, "web fonts");
+    const domain = rest[0];
+    if (!domain) throw new WebCliError("Usage: dench web fonts <domain> [--json]");
+    const data = await callWebOp(ctx, "brand-fonts", { domain });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "classify") {
+    const scheme = getFlag(rest, "--scheme") ?? "naics";
+    if (scheme !== "naics" && scheme !== "sic") {
+      throw new WebCliError("Invalid --scheme. Allowed: naics, sic");
+    }
+    assertNoUnknownFlags(rest, "web classify");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError(
+        "Usage: dench web classify <domain> [--scheme naics|sic] [--json]",
+      );
+    }
+    const data = await callWebOp(
+      ctx,
+      scheme === "sic" ? "classify-sic" : "classify-naics",
+      { input: domain },
+    );
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  throw new WebCliError(
+    `Unknown dench web command "${sub}". Run \`dench web --help\`.`,
+  );
+}
+
+export async function runWebCommand(ctx: WebCliContext): Promise<void> {
+  try {
+    await runWebSubcommand(ctx);
+  } catch (error) {
+    if (error instanceof WebCliError) {
+      console.error(error.message);
+      process.exitCode = 1;
+      return;
+    }
+    throw error;
+  }
+}