api-tracer-kit 1.0.0

package/cli/bin/api-tracer.mjs

@@ -0,0 +1,266 @@
+#!/usr/bin/env node
+/**
+ * api-tracer — map, exercise and watch the API a project calls.
+ *
+ *   api-tracer scan     read the source and write the endpoint catalog
+ *   api-tracer serve    the console, at http://localhost:4400
+ *   api-tracer start    scan, then serve
+ *   api-tracer report   print or export the insight report
+ *   api-tracer init     write a config file, only if the guesses need help
+ *
+ * Everything works with no config file. `init` exists for the cases guessing
+ * cannot cover, and it writes what it guessed so there is something to edit
+ * rather than a blank page.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { loadConfig, guessConfig, defaults, CONFIG_NAMES } from '../config.mjs';
+import { scanToFile, scanProject } from '../scan.mjs';
+import { buildReport, toMarkdown } from '../report.mjs';
+
+const [, , command = 'help', ...rest] = process.argv;
+const flag = (name, fallback) => {
+  const at = rest.indexOf(`--${name}`);
+  return at === -1 ? fallback : (rest[at + 1] ?? true);
+};
+
+const ROOT = resolve(String(flag('root', process.cwd())));
+const DATA = resolve(String(flag('data', process.env.API_TRACER_DATA ?? join(ROOT, '.api-tracer'))));
+const CATALOG = join(DATA, 'endpoints.json');
+
+const say = (...a) => console.log(...a);
+
+async function config() {
+  const loaded = await loadConfig(ROOT);
+  if (flag('preset')) loaded.preset = String(flag('preset'));
+  return loaded;
+}
+
+/* ------------------------------------------------------------------- scan */
+
+async function scan() {
+  const cfg = await config();
+  mkdirSync(DATA, { recursive: true });
+  const catalog = scanToFile(ROOT, cfg, CATALOG);
+
+  if (catalog.kept) {
+    say(`no source found under ${ROOT}; keeping the catalog already in ${CATALOG}`);
+    return catalog;
+  }
+  if (!catalog.endpoints.length) {
+    say(`no endpoints found in ${ROOT}.`);
+    say('The scan tries every preset and keeps whichever reads your code. If none did:');
+    say('  api-tracer init         write a config with what it guessed, then edit it');
+    say(`  --preset <name>         force one of: service-object, axios-direct, fetch-direct`);
+    say('Live recording still works without a catalog: every call becomes an endpoint.');
+    return catalog;
+  }
+
+  const modules = new Set(catalog.endpoints.map((e) => e.module));
+  const byMethod = {};
+  for (const e of catalog.endpoints) byMethod[e.method] = (byMethod[e.method] ?? 0) + 1;
+
+  say(`${catalog.endpoints.length} endpoints across ${modules.size} modules -> ${CATALOG}`);
+  say(`preset "${catalog.preset}"   ${Object.entries(byMethod).map(([m, n]) => `${m} ${n}`).join('  ')}`);
+  if (Object.keys(catalog.baseUrls).length) {
+    say(`base URLs: ${Object.entries(catalog.baseUrls).map(([k, v]) => `${k}=${v}`).join('  ')}`);
+  } else {
+    say('no base URL found; add baseUrls to a config file, or the console cannot send anything');
+  }
+  say(`auth header: ${catalog.auth.header}`);
+
+  const dead = catalog.endpoints.filter((e) => e.usedIn === 0);
+  if (dead.length) say(`${dead.length} endpoint(s) referenced nowhere outside their own file`);
+  return catalog;
+}
+
+/* ------------------------------------------------------------------- init */
+
+/** what a config file looks like, filled in with what the scan worked out */
+function configFile(guess, found) {
+  const urls = Object.entries(guess.baseUrls);
+  return `/**
+ * api-tracer config. Every field is optional — this file was generated from
+ * what the scan found, so it is a starting point, not a requirement.
+ */
+export default {
+  name: ${JSON.stringify(guess.name ?? 'API')},
+
+  // the preset that read this codebase best (${found} endpoints)
+  preset: ${JSON.stringify(guess.preset)},
+${guess.preset === 'service-object' ? `  // the object key holding the path\n  pathKey: ${JSON.stringify(defaults.pathKey)},\n` : ''}
+  // where to look for API calls
+  sources: ${JSON.stringify(defaults.sources)},
+
+  baseUrls: {
+${urls.map(([k, v]) => `    ${k}: ${JSON.stringify(v)},`).join('\n') || '    // dev: "https://api.example.com",'}
+  },
+
+  auth: { header: ${JSON.stringify(guess.auth.header)} },
+${guess.auth.alternatives?.length ? `  // other header names seen in the source: ${guess.auth.alternatives.join(', ')}\n` : ''}
+  /*
+   * Some APIs answer HTTP 200 and put the real verdict in the body. Set this to
+   * false if yours uses the status line honestly.
+   */
+  envelope: { codeFields: ['status', 'code'], okField: 'success', failFrom: 400 },
+
+  /*
+   * An optional sign-in chain, so nobody has to paste a token. Each step
+   * declares the fields to ask for, the call to make, and what to do with the
+   * answer. Delete this if pasting a token is enough.
+   *
+   * login: {
+   *   steps: [
+   *     {
+   *       name: 'credentials',
+   *       fields: [{ name: 'email', type: 'email' }, { name: 'password', type: 'password' }],
+   *       path: '/users/login.json',
+   *       body: (input) => ({ user: input }),
+   *       then: (data) => ({ token: data.token, owner: { email: data.email } }),
+   *     },
+   *   ],
+   * },
+   */
+};
+`;
+}
+
+async function init() {
+  const existing = CONFIG_NAMES.map((n) => join(ROOT, n)).find((f) => existsSync(f));
+  if (existing && !flag('force')) {
+    say(`${existing} already exists; pass --force to overwrite it`);
+    return;
+  }
+
+  const guess = guessConfig(ROOT, defaults);
+  const found = guess.scores.find((s) => s.preset === guess.preset)?.found ?? 0;
+  say(`${guess.files.length} source files scanned`);
+  for (const s of guess.scores) say(`  ${s.preset.padEnd(16)} ${s.found}`);
+
+  if (!guess.preset) {
+    say('\nNo preset read this codebase. A config file has been written anyway —');
+    say('set `preset` or supply your own `parse(src, config)` in it.');
+  }
+
+  const out = join(ROOT, CONFIG_NAMES[0]);
+  writeFileSync(out, configFile(guess, found));
+  say(`\nwrote ${out}`);
+  say('Nothing else is needed: `api-tracer start` works from here.');
+}
+
+/* ----------------------------------------------------------------- report */
+
+async function report() {
+  if (!existsSync(CATALOG)) {
+    say(`no catalog at ${CATALOG}; run \`api-tracer scan\` first`);
+    process.exit(1);
+  }
+  const catalog = JSON.parse(readFileSync(CATALOG, 'utf8'));
+  const read = (name, fallback) => {
+    try {
+      return JSON.parse(readFileSync(join(DATA, name), 'utf8'));
+    } catch {
+      return fallback;
+    }
+  };
+  const runs = (() => {
+    try {
+      return readFileSync(join(DATA, 'runs.jsonl'), 'utf8')
+        .split('\n')
+        .filter(Boolean)
+        .flatMap((l) => {
+          try {
+            return [JSON.parse(l)];
+          } catch {
+            return [];
+          }
+        });
+    } catch {
+      return [];
+    }
+  })();
+
+  const extras = read('uncatalogued.json', {});
+  const model = buildReport({
+    catalog,
+    endpoints: [...catalog.endpoints, ...Object.values(extras)],
+    samples: read('samples.json', {}),
+    results: read('results.json', {}),
+    contracts: read('contracts.json', {}),
+    runs,
+    env: String(flag('env', 'dev')),
+  });
+
+  const format = String(flag('format', 'md'));
+  const body = format === 'json' ? `${JSON.stringify(model, null, 2)}\n` : toMarkdown(model);
+  const out = flag('out');
+  if (out === undefined || out === true) process.stdout.write(body);
+  else {
+    writeFileSync(resolve(String(out)), body);
+    say(`wrote ${resolve(String(out))}`);
+  }
+}
+
+/* ------------------------------------------------------------------ serve */
+
+async function serve() {
+  process.env.API_TRACER_DATA = DATA;
+  const server = await import('../server.mjs');
+  const cfg = await config();
+  server.useConfig(cfg);
+  const port = Number(flag('port', process.env.PORT ?? 4400));
+  const host = String(flag('host', process.env.HOST ?? '127.0.0.1'));
+  server.startServer({ port, host });
+  say(`api-tracer console  ->  http://${host}:${port}${process.env.BASE_PATH ?? ''}/`);
+  say(`data: ${DATA}`);
+}
+
+/* -------------------------------------------------------------------- run */
+
+const HELP = `api-tracer
+
+  api-tracer scan            read the source, write the endpoint catalog
+  api-tracer serve           run the console (default http://127.0.0.1:4400)
+  api-tracer start           scan, then serve
+  api-tracer report          build the insight report
+  api-tracer init            write a config file from what the scan guessed
+
+Options
+  --root <dir>       the project to read           (default: cwd)
+  --data <dir>       where captures are kept       (default: <root>/.api-tracer)
+  --port <n>         console port                  (default: 4400)
+  --host <addr>      console host                  (default: 127.0.0.1)
+  --preset <name>    force a scanner preset        (service-object|axios-direct|fetch-direct)
+  --format md|json   report format                 (default: md)
+  --out <file>       write the report to a file    (default: stdout)
+  --env <name>       which base URL the report describes
+
+No configuration file is required.
+`;
+
+const commands = {
+  scan,
+  init,
+  report,
+  serve,
+  start: async () => {
+    await scan();
+    await serve();
+  },
+  help: async () => say(HELP),
+  '--help': async () => say(HELP),
+  '-h': async () => say(HELP),
+  '--version': async () =>
+    say(JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')).version),
+};
+
+const run = commands[command];
+if (!run) {
+  say(`unknown command "${command}"\n`);
+  say(HELP);
+  process.exit(1);
+}
+run().catch((e) => {
+  console.error(e.message);
+  process.exit(1);
+});

package/cli/config.mjs

@@ -0,0 +1,224 @@
+/**
+ * Configuration, and doing without it.
+ *
+ * The goal is that `api-tracer scan` works in a project it has never seen. So
+ * every setting has a default, and the ones that cannot have a useful default —
+ * which preset reads this codebase, where its API lives — are guessed by trying
+ * each option and keeping whichever actually finds endpoints.
+ *
+ * A config file is optional. It exists for the cases guessing cannot cover: a
+ * bespoke service shape, a base URL that is not written down in the source, an
+ * auth header the scanner cannot see.
+ */
+import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs';
+import { join, resolve, relative, basename, extname } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { PRESETS, PRESET_NAMES, stripComments } from './presets.mjs';
+
+export const CONFIG_NAMES = [
+  'api-tracer.config.mjs',
+  'api-tracer.config.js',
+  'api-tracer.config.json',
+];
+
+export const defaults = {
+  /** where to look for API calls, relative to the project root */
+  sources: ['src', 'app', 'lib', 'services', 'api'],
+  /** never walked */
+  ignore: ['node_modules', 'dist', 'build', 'coverage', '.git', '.next', 'out', 'vendor', '__snapshots__'],
+  extensions: ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs'],
+  /** which preset reads this codebase; null means guess */
+  preset: null,
+  /** the object key holding the path, for the service-object preset */
+  pathKey: 'subUrl',
+  /** environment -> base URL. Guessed from the source when absent. */
+  baseUrls: {},
+  /** the header the API authenticates with */
+  auth: { header: 'Authorization' },
+  /** how to read pass/fail out of a 2xx body; false turns it off */
+  envelope: { codeFields: ['status', 'code'], okField: 'success', failFrom: 400 },
+  /** the module a file's endpoints belong to; the filename by default */
+  moduleOf: null,
+};
+
+/** every source file under the configured roots */
+export function collectFiles(root, config = defaults) {
+  const out = [];
+  const ignore = new Set(config.ignore ?? defaults.ignore);
+  const extensions = new Set(config.extensions ?? defaults.extensions);
+
+  const walk = (dir) => {
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return; // unreadable directory is not a reason to fail the scan
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith('.') && entry.name !== '.') continue;
+      if (ignore.has(entry.name)) continue;
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) walk(full);
+      else if (extensions.has(extname(entry.name))) out.push(full);
+    }
+  };
+
+  const roots = (config.sources ?? defaults.sources)
+    .map((s) => resolve(root, s))
+    .filter((p) => existsSync(p) && statSync(p).isDirectory());
+  // no recognisable source directory: fall back to the project root itself
+  for (const dir of roots.length ? roots : [root]) walk(dir);
+  return out;
+}
+
+/**
+ * Files most likely to hold the API layer, so a guess is made on the code that
+ * matters rather than on the first forty components in alphabetical order.
+ */
+function pickSample(files, limit = 120) {
+  const weight = (f) => {
+    const p = f.toLowerCase();
+    if (/(services?|api|http|client|endpoints?|requests?|queries)[/\\]/.test(p)) return 0;
+    if (/(services?|api|http|client)\./.test(basename(p))) return 1;
+    return 2;
+  };
+  return [...files].sort((a, b) => weight(a) - weight(b)).slice(0, limit);
+}
+
+const read = (file) => {
+  try {
+    return stripComments(readFileSync(file, 'utf8'));
+  } catch {
+    return '';
+  }
+};
+
+/**
+ * Runs every preset over a sample of the codebase and keeps whichever finds the
+ * most. Guessing by looking for marker strings was tried first and was wrong
+ * often enough to matter — a project can import axios and still not use it for
+ * its API layer. Running the presets asks the only question that counts: which
+ * one actually reads this code?
+ */
+export function guessPreset(files, config = defaults) {
+  const sample = pickSample(files);
+  const scores = [];
+
+  for (const name of PRESET_NAMES) {
+    const parse = PRESETS[name];
+    let found = 0;
+    for (const file of sample) {
+      try {
+        found += parse(read(file), config).length;
+      } catch {
+        /* a preset that throws on odd syntax simply scores nothing there */
+      }
+    }
+    scores.push({ preset: name, found });
+  }
+
+  scores.sort((a, b) => b.found - a.found);
+  return { preset: scores[0].found ? scores[0].preset : null, scores };
+}
+
+/**
+ * Base URLs, read out of the app rather than copied into a config, so dev and
+ * stage stay in sync with what the app actually calls. Looks for an object
+ * mapping environment names to URLs, then for any bare API-looking URL.
+ */
+export function guessBaseUrls(files) {
+  const ENV = /^(dev|develop|development|stage|staging|uat|qa|test|prod|production|local)$/i;
+  const urls = {};
+
+  for (const file of pickSample(files, 200)) {
+    const src = read(file);
+    for (const block of src.matchAll(/\{([^{}]*https?:\/\/[^{}]*)\}/g)) {
+      for (const [, env, url] of block[1].matchAll(/["']?(\w+)["']?\s*:\s*["'`](https?:\/\/[^"'`]+)["'`]/g)) {
+        if (ENV.test(env)) urls[env.toLowerCase()] = url;
+      }
+    }
+    if (Object.keys(urls).length) break;
+  }
+
+  if (!Object.keys(urls).length) {
+    // one URL and no environment map is still better than none
+    for (const file of pickSample(files, 200)) {
+      const hit = read(file).match(/["'`](https?:\/\/[^"'`\s]*\/api[^"'`\s]*)["'`]/);
+      if (hit) {
+        urls.dev = hit[1];
+        break;
+      }
+    }
+  }
+  return urls;
+}
+
+/** the auth header name, and where a human can read its value */
+export function guessAuth(files) {
+  const CANDIDATE = /^(authorization|auth[-_]?token|access[-_]?token|x-auth-token|x-api-key|api[-_]?key|token)$/i;
+  const counts = new Map();
+
+  for (const file of pickSample(files, 200)) {
+    const src = read(file);
+    // used as a key: `{ AUTH_TOKEN: token }` or `headers["AUTH_TOKEN"]`
+    for (const m of src.matchAll(/["'`]([A-Za-z][\w-]{2,30})["'`]\s*\]?\s*[:=]/g)) {
+      if (CANDIDATE.test(m[1])) counts.set(m[1], (counts.get(m[1]) ?? 0) + 1);
+    }
+    for (const m of src.matchAll(/headers?\s*(?:\.|\[["'`])\s*([A-Za-z][\w-]{2,30})/g)) {
+      if (CANDIDATE.test(m[1])) counts.set(m[1], (counts.get(m[1]) ?? 0) + 2);
+    }
+    /*
+     * Held in a named constant -- `const authTokenKey = "AUTH_TOKEN"` -- which
+     * is how a codebase that uses the header in more than one place tends to
+     * write it, so it is the strongest signal of the three.
+     */
+    for (const m of src.matchAll(
+      /\b(?:const|let|var)\s+\w*(?:auth|token|header|api[-_]?key)\w*\s*=\s*["'`]([A-Za-z][\w-]{2,30})["'`]/gi,
+    )) {
+      /*
+       * A constant named for auth is trusted whatever it holds. Header names
+       * are project inventions -- X_AUTH, Sesh, Client-Token -- so a fixed list
+       * of known names would miss most of them, and the variable name has
+       * already said what this value is for.
+       */
+      counts.set(m[1], (counts.get(m[1]) ?? 0) + (CANDIDATE.test(m[1]) ? 6 : 5));
+    }
+  }
+
+  const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
+  return {
+    header: ranked[0]?.[0] ?? defaults.auth.header,
+    alternatives: ranked.slice(1, 4).map(([name]) => name),
+  };
+}
+
+/** everything the scanner needs, guessed from the codebase */
+export function guessConfig(root, config = defaults) {
+  const files = collectFiles(root, config);
+  const { preset, scores } = guessPreset(files, config);
+  return {
+    files,
+    preset,
+    scores,
+    baseUrls: guessBaseUrls(files),
+    auth: guessAuth(files),
+  };
+}
+
+/** loads a config file if there is one; every field stays optional */
+export async function loadConfig(root) {
+  for (const name of CONFIG_NAMES) {
+    const file = join(root, name);
+    if (!existsSync(file)) continue;
+    try {
+      if (name.endsWith('.json')) {
+        return { ...defaults, ...JSON.parse(readFileSync(file, 'utf8')), configFile: file };
+      }
+      const mod = await import(pathToFileURL(file).href);
+      return { ...defaults, ...(mod.default ?? mod), configFile: file };
+    } catch (e) {
+      throw new Error(`could not read ${relative(root, file)}: ${e.message}`);
+    }
+  }
+  return { ...defaults, configFile: null };
+}