@nurkamol/seo-audit 1.31.0

package/src/robots.mjs

@@ -0,0 +1,92 @@
+// robots.txt, read the way Google reads it.
+//
+// The subtlety that decides whether this check is useful or noise: Allow and
+// Disallow are not first-match-wins. The most specific rule wins — the longest
+// path pattern — and a tie goes to Allow. Implementing only Disallow would
+// report every site that carves an exception out of a broad block, which is
+// most sites that have ever written:
+//
+//   Disallow: /wp-admin/
+//   Allow: /wp-admin/admin-ajax.php
+//
+// Patterns support `*` for any run of characters and a trailing `$` to anchor
+// the end. Everything else is a literal prefix match.
+
+/** Group records from a robots.txt body: [{ agents, rules }]. */
+export function parseRobots(body) {
+  const groups = [];
+  let current = null;
+  // Consecutive User-agent lines share one group of rules; a User-agent line
+  // *after* a rule starts a new group.
+  let namingAgents = false;
+
+  for (const raw of (body ?? '').split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, '').trim();
+    if (!line) continue;
+    const match = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
+    if (!match) continue;
+
+    const field = match[1].toLowerCase();
+    const value = match[2].trim();
+
+    if (field === 'user-agent') {
+      if (!namingAgents) {
+        current = { agents: [], rules: [] };
+        groups.push(current);
+        namingAgents = true;
+      }
+      current.agents.push(value.toLowerCase());
+      continue;
+    }
+    if (field === 'allow' || field === 'disallow') {
+      if (!current) continue; // A rule before any User-agent line belongs to nobody.
+      namingAgents = false;
+      current.rules.push({ allow: field === 'allow', path: value });
+    }
+  }
+  return groups;
+}
+
+/** Does a rule pattern cover this path? */
+function covers(pattern, path) {
+  // `Disallow:` with no value is the documented way to say "nothing", and must
+  // not be read as "everything".
+  if (pattern === '') return false;
+  const anchored = pattern.endsWith('$');
+  const body = anchored ? pattern.slice(0, -1) : pattern;
+  const rx = body.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
+  return new RegExp(`^${rx}${anchored ? '$' : ''}`).test(path);
+}
+
+/** The group that applies to an agent: its own, or the `*` fallback. */
+function groupFor(groups, agent) {
+  return (
+    groups.find((g) => g.agents.includes(agent)) ??
+    groups.find((g) => g.agents.includes('*')) ??
+    null
+  );
+}
+
+/**
+ * Whether `path` may be crawled, and which rule decided it.
+ *
+ * Googlebot is the default agent because it is the one whose opinion shows up
+ * in search results, which is the only reason this tool asks.
+ */
+export function robotsVerdict(groups, path, agent = 'googlebot') {
+  const group = groupFor(groups, agent);
+  if (!group) return { allowed: true, rule: null };
+
+  let best = null;
+  for (const rule of group.rules) {
+    if (!covers(rule.path, path)) continue;
+    if (!best) {
+      best = rule;
+      continue;
+    }
+    // Longest pattern wins; on a tie, Allow does.
+    if (rule.path.length > best.path.length) best = rule;
+    else if (rule.path.length === best.path.length && rule.allow) best = rule;
+  }
+  return { allowed: best ? best.allow : true, rule: best };
+}

package/src/serve.mjs

@@ -0,0 +1,81 @@
+// The hosted front end, running locally.
+//
+// Not a second implementation of it: `worker/index.mjs` is written against
+// `Request` and `Response`, which Node has, so the same file answers both. What
+// is here is thirty lines of adapter between `node:http` and the fetch API, and
+// the reason the Worker was held to web standards in the first place.
+//
+// What this has that the Worker cannot: no CPU ceiling, no subrequest limit, no
+// bill. A five-thousand-page site with `maxImageChecks` past a thousand fits
+// here and nowhere else.
+import { createServer } from 'node:http';
+import { Readable } from 'node:stream';
+import { randomUUID } from 'node:crypto';
+
+import { handle } from '../worker/index.mjs';
+
+/** Start the local UI. Returns `{ url, close }`.
+ *
+ *  Bound to the loopback address, which is the whole of its security model: a
+ *  server only this machine can reach is as private as the terminal that
+ *  started it.
+ *
+ *  The Worker's password gate is left exactly as it is rather than given a
+ *  local exemption — a bypass inside the deployed code is a bypass that can
+ *  reach production one refactor later. Instead a random token is minted here
+ *  and the adapter presents it on every request, so the gate is satisfied
+ *  rather than skipped. */
+export async function serve({ port = 4321, host = '127.0.0.1', maxPages, allowedHosts, userAgent } = {}) {
+  const token = randomUUID();
+  const env = {
+    AUDIT_TOKEN: token,
+    ...(maxPages ? { MAX_PAGES: String(maxPages) } : {}),
+    ...(allowedHosts ? { ALLOWED_HOSTS: allowedHosts } : {}),
+    ...(userAgent ? { USER_AGENT: userAgent } : {}),
+    // PageSpeed is allowed here and nowhere else by default: --serve is bound to
+    // the loopback address and the person running it is the person it serves, so
+    // spending their own PSI quota is their decision. A deployed Worker leaves
+    // this unset, where a stranger passing ?psi= would be spending somebody
+    // else's.
+    ALLOW_PSI: '1',
+  };
+
+  const server = createServer(async (incoming, outgoing) => {
+    try {
+      const url = `http://${incoming.headers.host ?? `${host}:${port}`}${incoming.url}`;
+      const body =
+        incoming.method === 'GET' || incoming.method === 'HEAD'
+          ? undefined
+          : await new Promise((resolve) => {
+              const chunks = [];
+              incoming.on('data', (chunk) => chunks.push(chunk));
+              incoming.on('end', () => resolve(Buffer.concat(chunks)));
+            });
+
+      const request = new Request(url, {
+        method: incoming.method,
+        headers: { ...incoming.headers, authorization: `Bearer ${token}` },
+        body,
+      });
+
+      const response = await handle(request, env, null);
+      outgoing.writeHead(response.status, Object.fromEntries(response.headers));
+      if (response.body) Readable.fromWeb(response.body).pipe(outgoing);
+      else outgoing.end();
+    } catch (err) {
+      outgoing.writeHead(500, { 'content-type': 'text/plain' });
+      outgoing.end(`The local server failed: ${err.message}\n`);
+    }
+  });
+
+  await new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(port, host, resolve);
+  });
+
+  const { port: bound } = server.address();
+  return {
+    url: `http://${host}:${bound}/`,
+    close: () => new Promise((resolve) => server.close(resolve)),
+  };
+}