@dench.com/cli 2.7.8 → 2.7.10-staging.100

package/web.ts

@@ -0,0 +1,356 @@
+/**
+ * `dench web` — brand, scrape, sitemap, crawl, screenshot, and extract
+ * through the Dench Cloud Gateway (`POST /v1/context/{op}`).
+ *
+ * Agents and humans use this instead of writing their own scrapers.
+ * Auth matches `dench search`: org DENCH_API_KEY / paid-workspace gateway key.
+ */
+
+import { assertNoUnknownFlags, getFlag, hasFlag } from "./lib/cli-args";
+
+type WebCliContext = {
+  args: string[];
+  jsonOutput: boolean;
+  bearerToken?: string;
+  gatewayBaseUrl?: string;
+};
+
+class WebCliError extends Error {}
+
+const CRAWL_DEFAULT_PAGES = 15;
+const CRAWL_MAX_PAGES = 50;
+
+function parsePositiveInt(name: string, raw: string | undefined): number | undefined {
+  if (raw === undefined) return undefined;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    throw new WebCliError(`Invalid ${name} value: ${raw}`);
+  }
+  return Math.floor(parsed);
+}
+
+function normalizeGatewayBaseUrl(value: string): string {
+  return value.replace(/\/+$/, "").replace(/\/v1$/, "");
+}
+
+function resolveGatewayBaseUrl(
+  ctx?: Pick<WebCliContext, "gatewayBaseUrl">,
+): string {
+  if (ctx?.gatewayBaseUrl?.trim()) {
+    return normalizeGatewayBaseUrl(ctx.gatewayBaseUrl.trim());
+  }
+  const explicit =
+    process.env.DENCH_GATEWAY_URL?.trim() || process.env.GATEWAY_URL?.trim();
+  if (explicit) return normalizeGatewayBaseUrl(explicit);
+  const host = process.env.DENCH_HOST?.trim();
+  if (host && /localhost|127\.0\.0\.1/.test(host)) {
+    return "http://localhost:8787";
+  }
+  return "https://gateway.merseoriginals.com";
+}
+
+function requireBearerToken(ctx?: Pick<WebCliContext, "bearerToken">): string {
+  const bearerToken =
+    ctx?.bearerToken?.trim() || process.env.DENCH_API_KEY?.trim();
+  if (!bearerToken) {
+    throw new WebCliError(
+      "No Dench gateway credential found. Set DENCH_API_KEY or run `dench signin` for a paid workspace.",
+    );
+  }
+  return bearerToken;
+}
+
+async function callWebOp(
+  ctx: WebCliContext,
+  op: string,
+  input: Record<string, unknown>,
+): Promise<unknown> {
+  const bearerToken = requireBearerToken(ctx);
+  const baseUrl = resolveGatewayBaseUrl(ctx);
+  const response = await fetch(`${baseUrl}/v1/context/${op}`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      Authorization: `Bearer ${bearerToken}`,
+    },
+    body: JSON.stringify({ input }),
+  });
+  if (!response.ok) {
+    let detail = "";
+    try {
+      detail = await response.text();
+    } catch {
+      // ignore
+    }
+    const truncated = detail.length > 500 ? `${detail.slice(0, 500)}…` : detail;
+    throw new WebCliError(
+      `Gateway /v1/context/${op} returned ${response.status}: ${truncated || response.statusText}`,
+    );
+  }
+  return response.json();
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function out(ctx: WebCliContext, payload: unknown, formatted: string): void {
+  if (ctx.jsonOutput) {
+    console.log(JSON.stringify(payload, null, 2));
+    return;
+  }
+  console.log(formatted);
+}
+
+function formatBrand(data: unknown): string {
+  const record = asRecord(data);
+  if (!record) return "(empty brand)";
+  const lines = [
+    asString(record.title) ?? asString(record.name) ?? "(untitled brand)",
+    asString(record.domain) ? `  ${asString(record.domain)}` : "",
+    asString(record.description) ? `  ${asString(record.description)}` : "",
+  ].filter((line) => line.length > 0);
+  return lines.join("\n");
+}
+
+function formatMarkdown(data: unknown): string {
+  const record = asRecord(data);
+  return asString(record?.markdown) ?? JSON.stringify(data, null, 2);
+}
+
+export function webHelp(): void {
+  console.log(`Dench web
+
+Read pages, brand kits, sitemaps, screenshots, and structured extracts
+through the Dench gateway. Prefer these over writing your own scraper.
+
+  dench web scrape <url> [--json]
+  dench web brand <domain|email> [--json]
+  dench web sitemap <domain> [--limit N] [--json]
+  dench web crawl <url> [--max-pages N] [--json]
+  dench web screenshot <url> [--full-page] [--json]
+  dench web extract <url> --schema '<json>' [--json]
+  dench web product <url> [--json]
+  dench web competitors <domain> [--json]
+  dench web styleguide <domain> [--json]
+  dench web fonts <domain> [--json]
+  dench web classify <domain> [--scheme naics|sic] [--json]
+  dench web brand-search <query> [--json]
+
+Crawl defaults to 15 pages and is hard-capped at 50. Search stays on
+\`dench search\` (Exa). \`dench context\` is workspace orientation, not this.
+`);
+}
+
+async function runWebSubcommand(ctx: WebCliContext): Promise<void> {
+  const sub = ctx.args[0];
+  const rest = ctx.args.slice(1);
+
+  if (!sub || sub === "help" || sub === "--help" || sub === "-h") {
+    webHelp();
+    return;
+  }
+
+  if (sub === "scrape") {
+    assertNoUnknownFlags(rest, "web scrape");
+    const url = rest[0];
+    if (!url) throw new WebCliError("Usage: dench web scrape <url> [--json]");
+    const data = await callWebOp(ctx, "web-scrape-markdown", {
+      url,
+      useMainContentOnly: true,
+    });
+    out(ctx, data, formatMarkdown(data));
+    return;
+  }
+
+  if (sub === "brand") {
+    assertNoUnknownFlags(rest, "web brand");
+    const identifier = rest[0];
+    if (!identifier) {
+      throw new WebCliError("Usage: dench web brand <domain|email> [--json]");
+    }
+    const input = identifier.includes("@")
+      ? { email: identifier }
+      : { domain: identifier };
+    const data = await callWebOp(ctx, "brand-retrieve", input);
+    out(ctx, data, formatBrand(data));
+    return;
+  }
+
+  if (sub === "brand-search") {
+    assertNoUnknownFlags(rest, "web brand-search");
+    const query = rest.join(" ").trim();
+    if (!query) {
+      throw new WebCliError("Usage: dench web brand-search <query> [--json]");
+    }
+    const data = await callWebOp(ctx, "brand-search", { query });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "sitemap") {
+    const limit = parsePositiveInt("--limit", getFlag(rest, "--limit"));
+    assertNoUnknownFlags(rest, "web sitemap");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError("Usage: dench web sitemap <domain> [--limit N] [--json]");
+    }
+    const data = await callWebOp(ctx, "web-sitemap", {
+      domain,
+      ...(limit != null ? { maxLinks: limit } : {}),
+    });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "crawl") {
+    const maxPages =
+      parsePositiveInt("--max-pages", getFlag(rest, "--max-pages")) ??
+      CRAWL_DEFAULT_PAGES;
+    if (maxPages > CRAWL_MAX_PAGES) {
+      throw new WebCliError(`--max-pages cannot exceed ${CRAWL_MAX_PAGES}.`);
+    }
+    assertNoUnknownFlags(rest, "web crawl");
+    const url = rest[0];
+    if (!url) {
+      throw new WebCliError(
+        "Usage: dench web crawl <url> [--max-pages N] [--json]",
+      );
+    }
+    const data = await callWebOp(ctx, "web-crawl", {
+      url,
+      maxPages,
+      useMainContentOnly: true,
+    });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "screenshot") {
+    const fullPage = hasFlag(rest, "--full-page");
+    assertNoUnknownFlags(rest, "web screenshot");
+    const url = rest[0];
+    if (!url) {
+      throw new WebCliError(
+        "Usage: dench web screenshot <url> [--full-page] [--json]",
+      );
+    }
+    const data = await callWebOp(ctx, "web-screenshot", {
+      directUrl: url,
+      fullScreenshot: fullPage,
+    });
+    const record = asRecord(data);
+    out(ctx, data, asString(record?.screenshot) ?? JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "extract") {
+    const schemaRaw = getFlag(rest, "--schema");
+    assertNoUnknownFlags(rest, "web extract");
+    const url = rest[0];
+    if (!url || !schemaRaw) {
+      throw new WebCliError(
+        "Usage: dench web extract <url> --schema '<json>' [--json]",
+      );
+    }
+    let schema: unknown;
+    try {
+      schema = JSON.parse(schemaRaw);
+    } catch {
+      throw new WebCliError("--schema must be valid JSON.");
+    }
+    if (!schema || typeof schema !== "object" || Array.isArray(schema)) {
+      throw new WebCliError("--schema must be a JSON object.");
+    }
+    const data = await callWebOp(ctx, "web-extract", { url, schema });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "product") {
+    assertNoUnknownFlags(rest, "web product");
+    const url = rest[0];
+    if (!url) throw new WebCliError("Usage: dench web product <url> [--json]");
+    const data = await callWebOp(ctx, "web-product", { url });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "competitors") {
+    assertNoUnknownFlags(rest, "web competitors");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError("Usage: dench web competitors <domain> [--json]");
+    }
+    const data = await callWebOp(ctx, "web-competitors", {
+      domain,
+      numCompetitors: 5,
+    });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "styleguide") {
+    assertNoUnknownFlags(rest, "web styleguide");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError("Usage: dench web styleguide <domain> [--json]");
+    }
+    const data = await callWebOp(ctx, "brand-styleguide", { domain });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "fonts") {
+    assertNoUnknownFlags(rest, "web fonts");
+    const domain = rest[0];
+    if (!domain) throw new WebCliError("Usage: dench web fonts <domain> [--json]");
+    const data = await callWebOp(ctx, "brand-fonts", { domain });
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  if (sub === "classify") {
+    const scheme = getFlag(rest, "--scheme") ?? "naics";
+    if (scheme !== "naics" && scheme !== "sic") {
+      throw new WebCliError("Invalid --scheme. Allowed: naics, sic");
+    }
+    assertNoUnknownFlags(rest, "web classify");
+    const domain = rest[0];
+    if (!domain) {
+      throw new WebCliError(
+        "Usage: dench web classify <domain> [--scheme naics|sic] [--json]",
+      );
+    }
+    const data = await callWebOp(
+      ctx,
+      scheme === "sic" ? "classify-sic" : "classify-naics",
+      { input: domain },
+    );
+    out(ctx, data, JSON.stringify(data, null, 2));
+    return;
+  }
+
+  throw new WebCliError(
+    `Unknown dench web command "${sub}". Run \`dench web --help\`.`,
+  );
+}
+
+export async function runWebCommand(ctx: WebCliContext): Promise<void> {
+  try {
+    await runWebSubcommand(ctx);
+  } catch (error) {
+    if (error instanceof WebCliError) {
+      console.error(error.message);
+      process.exitCode = 1;
+      return;
+    }
+    throw error;
+  }
+}