@nurkamol/seo-audit 1.32.0 → 1.33.0

package/README.md

@@ -253,7 +253,34 @@ nobody has been shown.
 Opt-in, and the only thing in this tool that needs an account. It reads
 `GSC_CLIENT_ID`, `GSC_CLIENT_SECRET` and `GSC_REFRESH_TOKEN` from the
 environment or from `~/.config/seo-audit/.env`, deliberately outside any
-repository. A domain property is named `sc-domain:example.com` rather than by
+repository.
+
+Getting the third one used to be left as an exercise, which is why this had
+never run against the live API. Now:
+
+```bash
+# once, in console.cloud.google.com:
+#   enable the Search Console API, then create an OAuth client of type
+#   "Desktop app", and put its two values in ~/.config/seo-audit/.env
+#     GSC_CLIENT_ID=…apps.googleusercontent.com
+#     GSC_CLIENT_SECRET=…
+
+npx @nurkamol/seo-audit --search-console-login
+```
+
+That opens a browser, you sign in, and the refresh token is written to the same
+file at mode `600`. It is never printed — a token echoed to a terminal is a
+token in a scrollback buffer and probably in a shell history file. The scope is
+read-only. Afterwards it lists the properties the account can actually read,
+because a token that can read nothing looks exactly like one that works, right
+up until an audit reports the property was not found.
+
+Interactive by nature, so it is deliberately **not** a GitHub Action input: a
+flag CI can accept and never satisfy is worse than no flag. In CI, set the
+three variables as secrets.
+
+The whole setup, the property-naming trap and what each failure note means:
+[docs/search-console.md](docs/search-console.md). A domain property is named `sc-domain:example.com` rather than by
 its URL. Missing credentials, or a property the account cannot read, are a note
 and the rest of the audit is unaffected.
 

package/action.yml

@@ -32,6 +32,15 @@ inputs:
       under it, sampled. Set the PSI_API_KEY env var from a secret to lift the
       quota.
     required: false
+  search-console:
+    description: >
+      Order findings by what the pages actually do in Google. The property
+      exactly as Search Console names it — a domain property is
+      "sc-domain:example.com", not a URL. Needs GSC_CLIENT_ID,
+      GSC_CLIENT_SECRET and GSC_REFRESH_TOKEN in the environment, from secrets;
+      get them once with --search-console-login on a machine with a browser.
+      See docs/search-console.md.
+    required: false
   check-external:
     description: >
       Also check links pointing off the site. Off by default because other
@@ -143,6 +152,7 @@ runs:
         [ -n "$INPUT_LIMIT" ]    && args+=(--limit "$INPUT_LIMIT")
         [ -n "$INPUT_PSI" ]      && args+=(--psi "$INPUT_PSI")
         [ -n "$INPUT_PSI_SAMPLE" ] && args+=(--psi-sample "$INPUT_PSI_SAMPLE")
+        [ -n "$INPUT_SEARCH_CONSOLE" ] && args+=(--search-console "$INPUT_SEARCH_CONSOLE")
         [ -n "$INPUT_VERBOSE" ]  && args+=(--verbose)
         [ -n "$INPUT_CHECK_EXTERNAL" ] && args+=(--check-external)
         [ -n "$INPUT_SETTLE" ]   && args+=(--settle "$INPUT_SETTLE")

package/bin/seo-audit.mjs

@@ -82,6 +82,11 @@ const HELP = `
                        GSC_REFRESH_TOKEN in the environment or in
                        ~/.config/seo-audit/.env. A domain property is named
                        "sc-domain:example.com" rather than by its URL
+    --search-console-login
+                       sign in to Google once and write GSC_REFRESH_TOKEN to
+                       ~/.config/seo-audit/.env. Needs GSC_CLIENT_ID and
+                       GSC_CLIENT_SECRET there first, from an OAuth client of
+                       type "Desktop app". Opens a browser; not for CI
     --compare-as <name> fetch a sample of pages a second time as this browser
                        or crawler and report what changed. A page that differs
                        with the reader is cloaking, or bot protection misfiring
@@ -152,6 +157,7 @@ function parseArgs(argv) {
       const next = argv[i + 1];
       opts.serve = next && /^\d+$/.test(next) ? Number(argv[++i]) : true;
     }
+    else if (arg === '--search-console-login') opts.searchConsoleLogin = true;
     else if (arg === '--search-console') {
       // Optionally the property name, since a domain property is not a URL.
       const next = argv[i + 1];
@@ -316,6 +322,37 @@ if (opts.serve !== undefined) {
   }
 } else {
 
+// --- sign in, and stop ----------------------------------------------------
+// Before anything that needs a URL: this takes none. It is the one thing here
+// that is interactive by nature, and the reason `--search-console` had never
+// run against the live API — the three variables were documented and there was
+// no way to obtain the third.
+if (opts.searchConsoleLogin) {
+  const { login } = await import('../src/console.mjs');
+  try {
+    const { dotfile, properties } = await login({
+      onNote: (note) => process.stderr.write(`  ${note}\n`),
+    });
+    process.stdout.write(`\n  Refresh token written to ${dotfile}\n`);
+    if (properties.length) {
+      process.stdout.write('\n  Properties this account can read:\n');
+      for (const p of properties) process.stdout.write(`    ${p.url}  (${p.permission})\n`);
+      process.stdout.write(`\n  Try: seo-audit https://example.com --search-console ${properties[0].url}\n\n`);
+    } else {
+      // A token that can read nothing looks exactly like one that works, right
+      // up until an audit reports the property was not found.
+      process.stdout.write(
+        '\n  This account can read no properties. Verify the site in Search Console first,\n' +
+          '  or sign in as the account that already has it.\n\n',
+      );
+    }
+    process.exit(properties.length ? 0 : 1);
+  } catch (err) {
+    process.stderr.write(`\n  ${err.message}\n\n`);
+    process.exit(1);
+  }
+}
+
 // Nothing to audit. If a person is there to ask, ask; otherwise this is a
 // script or a CI runner and the help text is the right answer.
 if (!sites.length) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nurkamol/seo-audit",
-  "version": "1.32.0",
+  "version": "1.33.0",
   "description": "Crawl a site's sitemap and check every page for SEO, metadata and structured-data problems that single-page graders miss. Zero dependencies.",
   "type": "module",
   "bin": {

package/src/config.mjs

@@ -22,6 +22,34 @@
 //     ]
 //   }
 import { readFileSync, existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+/**
+ * One secret, environment first, then `~/.config/seo-audit/.env`.
+ *
+ * Deliberately outside this repository, which is public. There were two copies
+ * of this — one in `psi.mjs`, one in `console.mjs` — and the second built its
+ * pattern with `new RegExp` and a template literal, where `\\\\s` survives as an
+ * escaped backslash rather than as whitespace. It compiled, it never threw, and
+ * it could not match a single line of a real `.env`. Search Console's dotfile
+ * fallback had therefore never worked, and nothing said so because the only
+ * tests it had used a fake API and injected credentials.
+ */
+export function readSecret(name, env = process.env, read = readFileSync) {
+  if (env[name]) return env[name];
+  let text;
+  try {
+    // A missing file and an unreadable one are the same answer, and catching
+    // beats an `existsSync` guard: it is one syscall rather than two, it closes
+    // the gap between the two calls, and it lets a test hand in its own reader
+    // without the real filesystem deciding whether the test runs.
+    text = read(join(homedir(), '.config', 'seo-audit', '.env'), 'utf8');
+  } catch {
+    return null;
+  }
+  return text.match(new RegExp(`^\\s*${name}\\s*=\\s*(\\S+)`, 'm'))?.[1] ?? null;
+}
 
 const FILENAMES = ['seo-audit.config.json', '.seo-audit.json'];
 

package/src/console.mjs

@@ -9,23 +9,17 @@
 // Opt-in, and the only thing in this tool that needs an account. Credentials
 // are read the way the PageSpeed key is — the environment first, then
 // ~/.config/seo-audit/.env — and never from the repository.
-import { readFileSync, existsSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { join } from 'node:path';
+import { readSecret } from './config.mjs';
 
 const TOKEN_URL = 'https://oauth2.googleapis.com/token';
 const API = 'https://searchconsole.googleapis.com/webmasters/v3/sites';
 
 const f = (level, id, title, detail, url) => ({ level, id, title, detail, url });
 
-/** One credential, environment first. */
-export function findCredential(name, env = process.env, read = readFileSync) {
-  if (env[name]) return env[name];
-  const dotfile = join(homedir(), '.config', 'seo-audit', '.env');
-  if (!existsSync(dotfile)) return null;
-  const match = read(dotfile, 'utf8').match(new RegExp(`^\\\\s*${name}\\\\s*=\\\\s*(\\\\S+)`, 'm'));
-  return match?.[1] ?? null;
-}
+/** One credential, environment first. The loader is shared with the PageSpeed
+ *  key: this file had its own copy, and its copy could not read the dotfile at
+ *  all. */
+export const findCredential = readSecret;
 
 /** All three, or a sentence saying which is missing. */
 export function credentials(env = process.env) {
@@ -136,11 +130,212 @@ export async function searchConsole(origin, findings, opts = {}) {
     matched++;
   }
 
-  const shown = [...traffic.values()].reduce((n, t) => n + t.impressions, 0);
+  // Impressions on the pages this crawl actually reached, counted once per
+  // page. The first live run summed every row in the property instead — the
+  // report said one finding had been "shown 98 times between them" when its
+  // page had 13, because the other 85 were on pages the crawl never touched.
+  // Both numbers are worth having; they are two different sentences.
+  const counted = new Set();
+  let shown = 0;
+  for (const finding of findings) {
+    if (!finding.traffic || counted.has(finding.url)) continue;
+    counted.add(finding.url);
+    shown += finding.traffic.impressions;
+  }
+  const everywhere = [...traffic.values()].reduce((n, t) => n + t.impressions, 0);
+
   return [
     f('info', 'search-console', `Search Console has ${traffic.size.toLocaleString()} pages for this site`,
-      `${matched.toLocaleString()} of this crawl's findings are on pages Google has shown, ` +
-        `${shown.toLocaleString()} times between them over 28 days. Findings are ordered by that where it ` +
+      `${matched.toLocaleString()} of this crawl's findings ${matched === 1 ? 'is' : 'are'} on ` +
+        `${counted.size.toLocaleString()} page${counted.size === 1 ? '' : 's'} Google has shown, ` +
+        `${shown.toLocaleString()} time${shown === 1 ? '' : 's'} over 28 days — out of ` +
+        `${everywhere.toLocaleString()} across the whole property. Findings are ordered by that where it ` +
         'is known, and by how much of the site links to a page where it is not.', origin),
   ];
 }
+
+// --- Getting a refresh token ------------------------------------------------
+//
+// The three variables above were documented for a year and there was never a
+// way to obtain the third one. That is why `--search-console` has never run
+// against the live API: not the code, the paperwork in front of it.
+//
+// Loopback OAuth, which is what Google's own docs call the installed-app flow.
+// A desktop client may redirect to any port on 127.0.0.1 without registering
+// it, so this listens on an ephemeral one, and the browser does the signing in.
+// The token is written to the same file the key lives in and is never printed:
+// a refresh token in a terminal is a refresh token in a scrollback buffer.
+
+import { createServer } from 'node:http';
+import { spawn } from 'node:child_process';
+import { mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { randomBytes } from 'node:crypto';
+
+const AUTH_URL = 'https://accounts.google.com/o/oauth2/v2/auth';
+const SCOPE = 'https://www.googleapis.com/auth/webmasters.readonly';
+
+export const DOTFILE = join(homedir(), '.config', 'seo-audit', '.env');
+
+/** Where the browser is sent. `prompt=consent` is not politeness: without it
+ *  Google returns no refresh token at all on a second authorisation, which is
+ *  the confusing half of this flow. Read-only scope, because this only ever
+ *  reads. */
+export function authUrl({ clientId, redirectUri, state }) {
+  const params = new URLSearchParams({
+    client_id: clientId,
+    redirect_uri: redirectUri,
+    response_type: 'code',
+    scope: SCOPE,
+    access_type: 'offline',
+    prompt: 'consent',
+    state,
+  });
+  return `${AUTH_URL}?${params}`;
+}
+
+/** The code the browser came back with, traded for the long-lived half. */
+export async function exchangeCode({ clientId, clientSecret, code, redirectUri }, fetcher = fetch) {
+  const res = await fetcher(TOKEN_URL, {
+    method: 'POST',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      client_id: clientId,
+      client_secret: clientSecret,
+      code,
+      redirect_uri: redirectUri,
+      grant_type: 'authorization_code',
+    }),
+  });
+  const data = await res.json();
+  if (!res.ok || !data.refresh_token) {
+    throw new Error(
+      data.error_description ??
+        data.error ??
+        'Google returned no refresh token. This happens when the account has authorised this ' +
+          'client before — revoke it at myaccount.google.com/permissions and try again.',
+    );
+  }
+  return data;
+}
+
+/** What this account can actually read. Printed after a login because a token
+ *  that works for nothing looks exactly like a token that works. */
+export async function listProperties(token, fetcher = fetch) {
+  const res = await fetcher(API, { headers: { authorization: `Bearer ${token}` } });
+  const data = await res.json();
+  if (!res.ok) throw new Error(data.error?.message ?? `HTTP ${res.status} listing properties`);
+  return (data.siteEntry ?? []).map((s) => ({ url: s.siteUrl, permission: s.permissionLevel }));
+}
+
+/** One line rewritten, the rest of the file untouched.
+ *
+ *  Kept pure and exported so it can be tested: this writes to the file holding
+ *  somebody's PageSpeed key, and clobbering that to save a Search Console token
+ *  would be a poor trade. */
+export function upsertSecret(text, name, value) {
+  const line = `${name}=${value}`;
+  const pattern = new RegExp(`^\\s*${name}\\s*=.*$`, 'm');
+  if (pattern.test(text)) return text.replace(pattern, line);
+  return text.length && !text.endsWith('\n') ? `${text}\n${line}\n` : `${text}${line}\n`;
+}
+
+function openBrowser(url) {
+  const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open';
+  try {
+    spawn(cmd, [url], { stdio: 'ignore', detached: true }).unref();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const donePage = (heading, body) =>
+  `<!doctype html><meta charset="utf-8"><title>${heading}</title>` +
+  '<style>body{font:16px/1.6 -apple-system,system-ui,sans-serif;margin:20vh auto;max-width:32rem;' +
+  'padding:0 1.5rem;color:#111}h1{font-size:1.25rem;margin:0 0 .5rem}p{color:#555;margin:0}' +
+  '@media(prefers-color-scheme:dark){body{background:#111;color:#eee}p{color:#aaa}}</style>' +
+  `<h1>${heading}</h1><p>${body}</p>`;
+
+/**
+ * Sign in once, and write the refresh token where the audit will look for it.
+ *
+ * Interactive by nature, which is why it is not an Action input: a flag CI can
+ * accept and never satisfy is worse than no flag. The pieces that can be wrong
+ * quietly — the authorisation URL, the token exchange, rewriting a file that
+ * already holds somebody's PageSpeed key — are separate exported functions with
+ * tests. What is left here is a socket and a browser.
+ */
+export async function login({
+  fetcher = fetch,
+  openUrl = openBrowser,
+  onNote = () => {},
+  dotfile = DOTFILE,
+  timeout = 300_000,
+  // Injectable for the same reason the certificate reader is: otherwise the
+  // only way to exercise this is to have real credentials on the machine, and
+  // a test that needs those is a test nobody runs.
+  client,
+} = {}) {
+  const clientId = client?.clientId ?? readSecret('GSC_CLIENT_ID');
+  const clientSecret = client?.clientSecret ?? readSecret('GSC_CLIENT_SECRET');
+  if (!clientId || !clientSecret) {
+    throw new Error(
+      'No OAuth client yet. In console.cloud.google.com: enable the Search Console API, then ' +
+        'create an OAuth client of type "Desktop app". Put its two values in ' +
+        `${dotfile} as GSC_CLIENT_ID and GSC_CLIENT_SECRET, and run this again.`,
+    );
+  }
+
+  const state = randomBytes(16).toString('hex');
+  const server = createServer();
+  // Port 0: a desktop client may redirect to any port on the loopback address
+  // without registering it, so nothing here has to be configured or be free.
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  const redirectUri = `http://127.0.0.1:${server.address().port}/callback`;
+
+  const code = await new Promise((resolve, reject) => {
+    const timer = setTimeout(
+      () => reject(new Error(`Nothing came back within ${Math.round(timeout / 1000)}s. Nothing was written.`)),
+      timeout,
+    );
+    server.on('request', (req, res) => {
+      const asked = new URL(req.url, redirectUri);
+      if (asked.pathname !== '/callback') {
+        res.writeHead(404).end();
+        return;
+      }
+      const returned = asked.searchParams.get('code');
+      const failed = asked.searchParams.get('error');
+      // The state is the only thing standing between this and a page on the
+      // internet quietly posting a code to a port on the machine.
+      const mismatched = asked.searchParams.get('state') !== state;
+      res.writeHead(failed || !returned || mismatched ? 400 : 200, { 'content-type': 'text/html; charset=utf-8' });
+      if (failed || !returned || mismatched) {
+        res.end(donePage('That did not work', 'Nothing was written. The terminal has the detail.'));
+        clearTimeout(timer);
+        reject(new Error(mismatched ? 'The reply did not match the request this started.' : (failed ?? 'No code came back.')));
+        return;
+      }
+      res.end(donePage('Signed in', 'You can close this tab and go back to the terminal.'));
+      clearTimeout(timer);
+      resolve(returned);
+    });
+
+    const url = authUrl({ clientId, redirectUri, state });
+    onNote(openUrl(url) ? `waiting on ${redirectUri}` : `open this and sign in:\n\n  ${url}\n`);
+  }).finally(() => server.close());
+
+  const granted = await exchangeCode({ clientId, clientSecret, code, redirectUri }, fetcher);
+
+  mkdirSync(dirname(dotfile), { recursive: true });
+  const existing = existsSync(dotfile) ? readFileSync(dotfile, 'utf8') : '';
+  // 0600, and never printed: a refresh token echoed to a terminal is a refresh
+  // token in a scrollback buffer and probably in a shell history file.
+  writeFileSync(dotfile, upsertSecret(existing, 'GSC_REFRESH_TOKEN', granted.refresh_token), { mode: 0o600 });
+
+  // A token that can read nothing looks exactly like a token that works, until
+  // an audit says the property was not found.
+  return { dotfile, properties: await listProperties(granted.access_token, fetcher) };
+}

package/src/options.mjs

@@ -30,6 +30,7 @@ export const OPTIONS = [
   { flag: '--browser', query: 'browser', app: true },
   { flag: '--os', query: 'os', app: true },
   { flag: '--user-agent', query: 'userAgent', app: true },
+  { flag: '--search-console', query: 'search-console', app: true },
   { flag: '--write-sitemap', query: 'sitemap-out', app: true, via: 'the Export menu' },
   { flag: '--ignore', query: 'ignore', app: true, via: 'right-clicking a finding, and the Settings list' },
   { flag: '--psi', query: 'psi', app: true, via: 'Settings → Performance' },
@@ -59,7 +60,7 @@ export const OPTIONS = [
 
 
   // --- not yet, and that is a decision rather than an oversight ------------
-  { flag: '--search-console', query: null, app: 'not yet — needs an OAuth client, and has never run against the live API' },
+  { flag: '--search-console-login', query: null, app: 'no — it opens a browser and writes a credential to disk, which is a terminal errand, not a control in a window' },
   { flag: '--against', query: null, app: 'not yet — the window compares two kept runs instead of two live deployments' },
   { flag: '--compare-as', query: null, app: 'not yet — it fetches a sample of pages a second time, and the window has no control for spending that' },
   { flag: '--compare-sample', query: null, app: 'not yet — see --compare-as' },

package/src/psi.mjs

@@ -11,10 +11,7 @@
 //
 // A key is optional but raises the quota well above the anonymous limit. Set
 // PSI_API_KEY, or put it in ~/.config/seo-audit/.env — never in the repo.
-import { readFileSync, existsSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { join } from 'node:path';
-import { matchGlob } from './config.mjs';
+import { matchGlob, readSecret } from './config.mjs';
 
 const ENDPOINT = 'https://www.googleapis.com/pagespeedonline/v5/runPagespeed';
 
@@ -34,11 +31,9 @@ const CWV = {
 };
 
 export function findKey() {
-  if (process.env.PSI_API_KEY) return process.env.PSI_API_KEY;
-  const dotfile = join(homedir(), '.config', 'seo-audit', '.env');
-  if (!existsSync(dotfile)) return null;
-  const match = readFileSync(dotfile, 'utf8').match(/^\s*PSI_API_KEY\s*=\s*(\S+)/m);
-  return match?.[1] ?? null;
+  // Shared with the Search Console credentials rather than copied. The copy
+  // this file did not have was broken for months without anything noticing.
+  return readSecret('PSI_API_KEY');
 }
 
 async function run(url, strategy, key) {

package/src/serve.mjs

@@ -38,6 +38,11 @@ export async function serve({ port = 4321, host = '127.0.0.1', maxPages, allowed
     // this unset, where a stranger passing ?psi= would be spending somebody
     // else's.
     ALLOW_PSI: '1',
+    // Same reasoning, sharper stakes: these credentials read somebody's Search
+    // Console. On the loopback address the person running the server is the
+    // person whose account it is; a deployed Worker leaves this unset, where
+    // `?search-console=` would hand a stranger somebody else's traffic data.
+    ALLOW_SEARCH_CONSOLE: '1',
   };
 
   const server = createServer(async (incoming, outgoing) => {