crawlmark-mcp 0.1.0 → 0.1.2

package/dist/tools/reports.js

@@ -261,17 +261,17 @@ export function registerReportTools(server) {
             "The same problem found by two sources is one item, listing both sources with their own severity; `affected` is the largest page count any one source reports. " +
             "`changes` compares with the newest snapshot older than 24 hours: new, persistent, resolved (only when the source that reported it ran again) and unverified (not re-checked, never counted as fixed); null on the first run. " +
             "Each call saves a snapshot (local). Never starts a crawl or audit; stale sources (older than maxAgeDays) are shown with their date. " +
-            "Free by default. The ahrefs_site_audit and semrush_site_audit sources read the client's existing Site Audit project (clients.yaml ahrefsSiteAuditProject / semrushProject, else matched by domain) and spend units " +
-            "(Ahrefs: 50 for the issues, plus 50 per issueUrls call; Semrush: about 100 per call, up to 400 with the project list and titles): they need allowPaid: true, read only (never start a crawl or create a project). " +
+            "Free by default. The ahrefs_site_audit source reads the client's existing Ahrefs Site Audit project (clients.yaml ahrefsSiteAuditProject, else matched by domain) and spends units " +
+            "(50 for the issues, plus 50 per issueUrls call): it needs allowPaid: true and is read only (never starts a crawl or creates a project). Semrush's API has no Site Audit. " +
             "Call with estimateOnly: true first, state the units and ask before passing allowPaid.",
         inputSchema: {
             client: clientArg,
             sources: z.array(z.enum(ISSUE_SOURCES)).min(1).max(ISSUE_SOURCES.length).optional().describe("Default: every free source (audit, local_crawl, site_hygiene, dataforseo_crawl, gsc_inspection)."),
             minSeverity: z.enum(["critical", "high", "medium", "low"]).optional().describe("Leave out issues below this severity. Default low (everything)."),
             limit: z.number().int().min(1).max(200).optional().describe("Issues listed (counts cover all). Default 50."),
-            issueUrls: z.number().int().min(0).max(5).optional().describe("Top issues to fetch up to 3 example URLs for from a provider Site Audit source (paid: one call each). Default 0."),
+            issueUrls: z.number().int().min(0).max(5).optional().describe("Top issues to fetch up to 3 example URLs for from the Ahrefs Site Audit source (paid: one call each). Default 0."),
             maxAgeDays: z.number().int().min(7).max(120).optional().describe("Sources older than this are flagged stale. Default 45."),
-            allowPaid: z.boolean().optional().describe("Needed for the Ahrefs / Semrush Site Audit sources (they spend provider units); set only after the user agreed to the estimate."),
+            allowPaid: z.boolean().optional().describe("Needed for the Ahrefs Site Audit source (it spends Ahrefs units); set only after the user agreed to the estimate."),
             estimateOnly: z.boolean().optional().describe("Return the provider units this would spend, read nothing and save nothing."),
         },
         annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true },
@@ -279,11 +279,11 @@ export function registerReportTools(server) {
         const c = await resolveClient(client);
         const sources = o.sources ?? [...FREE_ISSUE_SOURCES];
         if ((o.issueUrls ?? 0) > 0 && !sources.some((x) => PAID_ISSUE_SOURCES.has(x)))
-            throw new Error("issueUrls needs an ahrefs_site_audit or semrush_site_audit source.");
+            throw new Error("issueUrls needs the ahrefs_site_audit source.");
         const ti = await import("../reports/technical-issues.js");
         if (estimateOnly) {
             const paid = await ti.planPaidIssueSources(c, sources, o.issueUrls ?? 0);
-            const units = { ahrefs: paid.filter((p) => p.provider === "ahrefs").reduce((a, p) => a + p.units, 0), semrush: paid.filter((p) => p.provider === "semrush").reduce((a, p) => a + p.units, 0) };
+            const units = { ahrefs: paid.reduce((a, p) => a + p.units, 0), semrush: 0 };
             return {
                 estimateOnly: true,
                 sources,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlmark-mcp",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "MCP server for automated SEO audits and reports: Google Search Console, GA4, Tag Manager, DataForSEO, Lighthouse and axe-core",
   "keywords": [
     "mcp",

package/dist/providers/semrush-site-audit.js

@@ -1,175 +0,0 @@
-import { recordUnits } from "../lib/budget.js";
-import { TTL } from "../lib/cache.js";
-import { cleanText, cleanUrl } from "../lib/inspections.js";
-import { ProviderUnavailableError } from "../lib/provider-ids.js";
-import { isTimeoutOrAbort, unitsCall, withUnitsReservation } from "./units-billing.js";
-const ID = "\\d{1,20}";
-const SNAP = "[A-Za-z0-9_-]{1,64}";
-/** The only Semrush calls this module can make (D12). Paths are matched whole; no query string may name a snapshot. */
-export const ALLOWED = [
-    { method: "GET", pattern: /^\/management\/v1\/projects$/, units: 100, label: "projects list" },
-    { method: "GET", pattern: new RegExp(`^/reports/v1/projects/${ID}/siteaudit/info$`), units: 100, label: "siteaudit/info" },
-    { method: "GET", pattern: new RegExp(`^/reports/v1/projects/${ID}/siteaudit/meta/issues$`), units: 100, label: "siteaudit/meta/issues" },
-    { method: "GET", pattern: new RegExp(`^/reports/v1/projects/${ID}/siteaudit/snapshot/${SNAP}/issue/${ID}$`), units: 100, label: "siteaudit/issue" },
-];
-export const CALL_UNITS = 100;
-export class SemrushAuditRefused extends Error {
-    name = "SemrushAuditRefused";
-}
-/** Throws unless (method, path) is allowed and no parameter names a snapshot (the 10,000-unit snapshot report). */
-export function assertSemrushAllowed(method, path, params = {}) {
-    const hit = ALLOWED.find((a) => a.method === method && a.pattern.test(path));
-    if (!hit || "snapshot_id" in params || /[?#]/.test(path))
-        throw new SemrushAuditRefused(`Refused: ${method} ${path.slice(0, 120)} isn't an allowed read-only Semrush Site Audit call.`);
-    return { label: hit.label, units: hit.units };
-}
-/** The real transport: allowlist, the shared Semrush limiter and key handling, units recorded on success. */
-export function semrushAuditRequest(method, path, params = {}) {
-    const { label, units } = assertSemrushAllowed(method, path, params);
-    return async (client) => {
-        const { semrushProjectsGet, semrushRedact } = await import("./semrush.js");
-        let res;
-        try {
-            res = await semrushProjectsGet(path, label, params);
-        }
-        catch (err) {
-            if (!isTimeoutOrAbort(err))
-                throw err;
-            // Semrush may still have billed it: the estimate is recorded against the cap (as v3Report does).
-            recordUnits(client.slug, "semrush", `${label} (timeout, estimated)`, units);
-            throw new Error(`Semrush ${label} timed out; Semrush may still have billed it. The estimated ${units} units were recorded against the cap.`);
-        }
-        let json;
-        try {
-            json = JSON.parse(res.text);
-        }
-        catch {
-            json = undefined;
-        }
-        if (res.status >= 400 || json === undefined || /^\s*ERROR \d+/.test(res.text)) {
-            throw new Error(`Semrush ${label} failed (HTTP ${res.status}): ${semrushRedact(res.text).slice(0, 200)}`);
-        }
-        recordUnits(client.slug, "semrush", label, units);
-        return json;
-    };
-}
-const realTransport = { get: (client, path, params) => semrushAuditRequest("GET", path, params)(client) };
-/** True when a project's host is the client's domain or one of its subdomains (SEC-1). */
-export const onClientDomain = (host, domain) => {
-    const own = bareHost(domain);
-    return Boolean(host) && (host === own || host.endsWith(`.${own}`));
-};
-const bareHost = (u) => {
-    try {
-        return new URL(/^https?:\/\//i.test(u) ? u : `https://${u}`).hostname.toLowerCase().replace(/^www\./, "");
-    }
-    catch {
-        return "";
-    }
-};
-/** D9: Semrush errors (IDs 1-99) -> high, warnings (101-199) -> medium, notices (201+) -> low. */
-export function semrushSeverity(id) {
-    return id < 100 ? "high" : id < 200 ? "medium" : "low";
-}
-/** Not confirmed (live check, research Not confirmed 4): the tool name Semrush uses for Site Audit in the project list. */
-export const SITE_AUDIT_TOOL = "siteaudit";
-/** Pure apart from the cache peek: list (unless a project is saved or the list is cached) + info + meta (unless cached) + issue URLs. */
-export function planSemrushSiteAudit(o) {
-    const n = Math.max(0, Math.min(5, o.issueUrls));
-    const list = o.savedProject || o.listCached ? 0 : 1;
-    const meta = o.metaCached ? 0 : 1;
-    const calls = list + 1 + meta + n;
-    return { units: calls * CALL_UNITS, requests: calls, breakdown: `Semrush Site Audit: ${list ? "project list (100) + " : ""}info (100)${meta ? " + issue titles (100)" : ""}${n ? ` + ${n} issue URL call(s) (100 each)` : ""}` };
-}
-export async function matchSemrushProject(client, t = realTransport) {
-    if (client.semrushProject)
-        return client.semrushProject;
-    const r = await unitsCall({
-        client,
-        provider: "semrush",
-        capability: "siteAuditProjects",
-        keyParts: { list: "all" },
-        ttl: TTL.week,
-        estimate: CALL_UNITS,
-        run: async () => {
-            const json = await t.get(client, "/management/v1/projects", {});
-            const list = Array.isArray(json) ? json : (json.projects ?? []);
-            // Kept: project IDs, hosts and tool names only.
-            return { value: list.map((p) => ({ id: p.project_id === null || p.project_id === undefined ? "" : String(p.project_id), host: bareHost(p.url ?? ""), tools: (p.tools ?? []).map((x) => String(x.tool ?? "")) })), units: CALL_UNITS };
-        },
-    });
-    const own = bareHost(client.domain);
-    const matches = r.value.filter((p) => onClientDomain(p.host, client.domain) && p.tools.includes(SITE_AUDIT_TOOL) && /^\d+$/.test(p.id));
-    if (matches.length !== 1) {
-        throw new ProviderUnavailableError("siteAudit", "semrush", matches.length
-            ? `${matches.length} Semrush projects with Site Audit match ${own}; set semrushProject (crawlmark-cli clients set ${client.domain} --semrush-project <id>)`
-            : `no Semrush project with Site Audit for ${own}; set one up in Semrush, or set semrushProject`);
-    }
-    return matches[0].id;
-}
-const auditTime = (v) => {
-    if (typeof v === "number" && Number.isFinite(v))
-        return new Date(v > 1e12 ? v : v * 1000).toISOString();
-    if (typeof v === "string" && !Number.isNaN(Date.parse(v)))
-        return new Date(v).toISOString();
-    return null;
-};
-/**
- * Read the latest audit: info (counts per issue ID), titles from meta/issues (cached 30 days) and, with issueUrls,
- * example URLs for the top issues. The whole read's units are checked and reserved first.
- */
-export async function readSemrushSiteAudit(client, opts, t = realTransport) {
-    const project = await matchSemrushProject(client, t);
-    const n = Math.max(0, Math.min(5, opts.issueUrls ?? 0));
-    const plan = planSemrushSiteAudit({ savedProject: true, listCached: true, metaCached: opts.metaCached === true, issueUrls: n });
-    return withUnitsReservation(client, "semrush", plan.units, async () => {
-        const base = `/reports/v1/projects/${project}/siteaudit`;
-        const info = await t.get(client, `${base}/info`, {});
-        // SEC-1: a project (pinned or matched) whose site isn't this client's never reaches its report.
-        const infoHost = bareHost(typeof info.url === "string" ? info.url : "");
-        if (!onClientDomain(infoHost, client.domain)) {
-            throw new ProviderUnavailableError("siteAudit", "semrush", `Semrush project ${project} is for ${infoHost || "an unknown site"}, not ${bareHost(client.domain)}; fix semrushProject (crawlmark-cli clients set ${client.domain} --semrush-project <id>)`);
-        }
-        const meta = await unitsCall({
-            client,
-            provider: "semrush",
-            capability: "siteAuditMeta",
-            keyParts: { project },
-            ttl: TTL.month,
-            estimate: CALL_UNITS,
-            run: async () => {
-                const json = await t.get(client, `${base}/meta/issues`, {});
-                return { value: (json.issues ?? []).flatMap((i) => (typeof i.id === "number" ? [{ id: i.id, title: cleanText(i.title, 200) }] : [])), units: CALL_UNITS };
-            },
-        });
-        const titles = new Map(meta.value.map((m) => [m.id, m.title]));
-        const issues = [];
-        for (const [k, v] of Object.entries(info.defects ?? {})) {
-            const id = Number(k);
-            if (!Number.isInteger(id) || id <= 0 || typeof v !== "number" || v <= 0)
-                continue;
-            const severity = semrushSeverity(id);
-            const title = titles.get(id) ?? `Semrush Site Audit issue ${id}`;
-            issues.push({ id, source: "semrush_site_audit", rule: String(id), url: null, severity, nativeSeverity: severity === "high" ? "error" : severity === "medium" ? "warning" : "notice", title, count: Math.round(v) });
-        }
-        const snapshotId = typeof info.current_snapshot?.snapshot_id === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(info.current_snapshot.snapshot_id) ? info.current_snapshot.snapshot_id : null;
-        if (n > 0 && snapshotId) {
-            const order = ["critical", "high", "medium", "low"];
-            for (const i of [...issues].sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity) || (b.count ?? 0) - (a.count ?? 0)).slice(0, n)) {
-                const json = await t.get(client, `${base}/snapshot/${snapshotId}/issue/${i.id}`, { limit: "3" });
-                // Not confirmed (live check): the URL field's name in this response.
-                i.examples = (json.data ?? []).map((row) => [row.source_url, row.url, row.page_url].find((x) => typeof x === "string")).map((u) => (u ? cleanUrl(u) : null)).filter((u) => Boolean(u)).slice(0, 3);
-            }
-        }
-        const ranAt = auditTime(info.last_audit) ?? new Date().toISOString();
-        return { run: { source: "semrush_site_audit", runId: `semrush:${project}@${ranAt}`, ranAt, stale: false }, issues: issues.map(({ id: _id, ...r }) => r) };
-    });
-}
-/** The pre-flight for one read, honouring the cached project list and issue titles (no request). */
-export async function estimateSemrushSiteAudit(client, issueUrls) {
-    const { cacheKey, peekCache } = await import("../lib/cache.js");
-    const listCached = peekCache(cacheKey("seo", "semrush", "siteAuditProjects", { list: "all" })) !== undefined;
-    const metaCached = client.semrushProject ? peekCache(cacheKey("seo", "semrush", "siteAuditMeta", { project: client.semrushProject })) !== undefined : false;
-    return planSemrushSiteAudit({ savedProject: Boolean(client.semrushProject), listCached, metaCached, issueUrls });
-}
-//# sourceMappingURL=semrush-site-audit.js.map