api-tracer-kit 1.0.0

package/cli/import.mjs

@@ -0,0 +1,231 @@
+/**
+ * Turns real traffic into samples for the scanned endpoints.
+ *
+ * Input is either a HAR file (DevTools -> Network -> Save all as HAR with content)
+ * or a single copied cURL command. Either way the job is the same: take a concrete
+ * request like
+ *
+ *   POST https://api.example.com/appointments/4821/accept
+ *
+ * and match it back to the template the scanner found in the source
+ *
+ *   /appointments/${data.id}/accept
+ *
+ * so the captured query params, body, and path values land on the right endpoint.
+ */
+
+/** anything that smells like a credential never reaches disk */
+const SECRET_HEADER = /^(auth_token|authorization|access_token|secret_token|cookie|x-api-key|api[-_]?key|cs_token)$/i;
+const SECRET_FIELD = /(token|password|secret|api[-_]?key|authorization)/i;
+
+/** `/appointments/${data.id}.json` -> /^\/appointments\/([^/?]+)\.json$/ */
+function templateToRegex(subUrl) {
+  const escaped = subUrl
+    .split(/\$\{[^}]+\}/)
+    .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+    .join('([^/?]+)');
+  return new RegExp(`^${escaped}$`);
+}
+
+/**
+ * Strips the host and any known API base so what is left lines up with a subUrl.
+ * Returns null when the request is not ours at all.
+ */
+export function toComparablePath(rawUrl, baseUrls) {
+  let url;
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    return null;
+  }
+  const full = `${url.origin}${url.pathname}`;
+  for (const base of Object.values(baseUrls)) {
+    if (full.startsWith(base)) return full.slice(base.length) || '/';
+  }
+  return null;
+}
+
+/**
+ * Best endpoint for a concrete request. Exact templates win over ones with holes,
+ * and among those, fewer holes wins -- so a literal path is never swallowed by a
+ * greedier pattern.
+ */
+export function matchEndpoint(method, path, endpoints) {
+  const candidates = endpoints
+    .filter((ep) => ep.method === method && !ep.absolute && !ep.dynamicUrl)
+    .map((ep) => {
+      const m = templateToRegex(ep.subUrl.split('?')[0]).exec(path);
+      return m ? { ep, values: m.slice(1) } : null;
+    })
+    .filter(Boolean);
+
+  if (!candidates.length) return null;
+  candidates.sort((a, b) => a.ep.holes.length - b.ep.holes.length);
+  const best = candidates[0];
+  return {
+    endpoint: best.ep,
+    // hole expression -> the real value seen in traffic
+    pathValues: Object.fromEntries(best.ep.holes.map((h, i) => [h.expr, best.values[i]])),
+  };
+}
+
+/**
+ * `a=1&b[]=2&b[]=3&f[x]=9` -> { a: '1', b: ['2','3'], f: { x: '9' } }
+ *
+ * ponytail: covers the three shapes axios emits. Deeper nesting like `a[b][c]`
+ * is kept as a flat key rather than guessed at.
+ */
+export function parseQuery(search) {
+  const out = {};
+  for (const [rawKey, value] of new URLSearchParams(search)) {
+    const asArray = /^(.+)\[\]$/.exec(rawKey);
+    const asNested = /^([^[]+)\[([^\]]+)\]$/.exec(rawKey);
+    if (asArray) (out[asArray[1]] ??= []).push(value);
+    else if (asNested) (out[asNested[1]] ??= {})[asNested[2]] = value;
+    else out[rawKey] = value;
+  }
+  return out;
+}
+
+/** replaces secret-looking values, keeping the shape so the sample stays useful */
+export function redact(value) {
+  if (Array.isArray(value)) return value.map(redact);
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([k, v]) => [k, SECRET_FIELD.test(k) ? '<redacted>' : redact(v)]),
+    );
+  }
+  return value;
+}
+
+function parseBody(text, mimeType = '') {
+  if (!text) return undefined;
+  if (/json/i.test(mimeType) || /^\s*[[{]/.test(text)) {
+    try {
+      return redact(JSON.parse(text));
+    } catch {
+      return undefined;
+    }
+  }
+  if (/x-www-form-urlencoded/i.test(mimeType)) return redact(parseQuery(text));
+  return undefined; // multipart and friends are not worth guessing at
+}
+
+/** one sample per request, ready to merge into the catalog */
+function toSample({ method, url, queryString, bodyText, mimeType }, ctx) {
+  const path = toComparablePath(url, ctx.baseUrls);
+  if (path === null) return { skipped: 'not an API host' };
+
+  const hit = matchEndpoint(method, path, ctx.endpoints);
+
+  return {
+    id: hit?.endpoint.id ?? null,
+    method,
+    path,
+    sample: {
+      pathValues: hit?.pathValues ?? {},
+      params: redact(parseQuery(queryString ?? new URL(url).search)),
+      data: parseBody(bodyText, mimeType),
+      seenAt: new Date().toISOString(),
+      from: `${method} ${path}`,
+    },
+  };
+}
+
+/* --------------------------------------------------------------------- HAR */
+
+export function importHar(harText, ctx) {
+  let har;
+  try {
+    har = JSON.parse(harText);
+  } catch (e) {
+    throw new Error(`not a valid HAR file: ${e.message}`);
+  }
+  const entries = har?.log?.entries;
+  if (!Array.isArray(entries)) throw new Error('no log.entries in that HAR');
+
+  const requests = entries
+    .filter((e) => e?.request?.url)
+    .map((e) => ({
+      method: (e.request.method ?? 'GET').toUpperCase(),
+      url: e.request.url,
+      bodyText: e.request.postData?.text,
+      mimeType: e.request.postData?.mimeType,
+    }));
+
+  return applyAll(requests, ctx);
+}
+
+/* -------------------------------------------------------------------- cURL */
+
+/** splits a shell-ish command, respecting single and double quotes */
+function tokenize(command) {
+  const out = [];
+  const re = /'([^']*)'|"((?:[^"\\]|\\.)*)"|(\S+)/g;
+  let m;
+  while ((m = re.exec(command))) {
+    out.push(m[1] ?? m[2]?.replace(/\\(.)/g, '$1') ?? m[3]);
+  }
+  return out;
+}
+
+/** parses what Chrome's "Copy as cURL" produces */
+export function parseCurl(command) {
+  const tokens = tokenize(command.replace(/\\\r?\n/g, ' ')).filter((t) => t !== 'curl');
+  const req = { method: null, url: null, headers: {}, bodyText: undefined, mimeType: '' };
+
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i];
+    if (t === '-X' || t === '--request') req.method = tokens[++i]?.toUpperCase();
+    else if (t === '-H' || t === '--header') {
+      const [, k, v] = /^([^:]+):\s*(.*)$/.exec(tokens[++i] ?? '') ?? [];
+      if (k) req.headers[k.trim()] = v;
+    } else if (['-d', '--data', '--data-raw', '--data-binary', '--data-ascii'].includes(t)) {
+      req.bodyText = tokens[++i];
+    } else if (t === '-b' || t === '--cookie') i++; // dropped on purpose
+    else if (!t.startsWith('-') && !req.url) req.url = t;
+  }
+
+  if (!req.url) throw new Error('no URL found in that cURL command');
+  req.mimeType = req.headers['content-type'] ?? req.headers['Content-Type'] ?? '';
+  req.method ??= req.bodyText ? 'POST' : 'GET';
+  return req;
+}
+
+export function importCurl(command, ctx) {
+  return applyAll([parseCurl(command)], ctx);
+}
+
+/* ------------------------------------------------------------------ shared */
+
+function applyAll(requests, ctx) {
+  const samples = {};
+  const unmatched = [];
+  let matched = 0;
+
+  const unmatchedCalls = [];
+  for (const req of requests) {
+    const out = toSample(req, ctx);
+    if (out.skipped) continue; // not one of our hosts at all
+
+    if (!out.id) {
+      // recorded anyway, as an endpoint no service file explains
+      unmatched.push(`no endpoint matches ${out.method} ${out.path}`);
+      unmatchedCalls.push(out);
+      continue;
+    }
+    matched++;
+    // later requests win: the most recent capture is usually the most relevant
+    samples[out.id] = out.sample;
+  }
+
+  return {
+    matched,
+    endpoints: Object.keys(samples).length,
+    unmatched: [...new Set(unmatched)],
+    unmatchedCalls,
+    samples,
+  };
+}
+
+export const _internal = { templateToRegex, SECRET_HEADER };

package/cli/index.mjs

@@ -0,0 +1,10 @@
+/**
+ * `api-tracer-kit/cli` — the console's pieces, for anyone driving it from code
+ * rather than from the command line.
+ */
+export { scanProject, scanToFile, writeCatalog } from './scan.mjs';
+export { loadConfig, guessConfig, guessPreset, guessBaseUrls, guessAuth, collectFiles, defaults } from './config.mjs';
+export { PRESETS, PRESET_NAMES, holesIn, stripComments } from './presets.mjs';
+export { buildReport, toMarkdown } from './report.mjs';
+export { shapeOf, flatten, diffShapes, summarizeDrift, shapeOfBody } from './shape.mjs';
+export { importHar, importCurl, matchEndpoint, toComparablePath, parseQuery } from './import.mjs';

package/cli/presets.mjs

@@ -0,0 +1,212 @@
+/**
+ * How to find endpoints in a codebase.
+ *
+ * Each preset takes one file's source and returns the endpoints in it. They are
+ * regex-based on purpose: a parser would need a dependency per language and per
+ * flavour of syntax, and would still not understand a template literal built
+ * from three variables. A preset that reads 90% of a codebase and says so is
+ * more useful than a parser that refuses to start.
+ *
+ * A project whose services look like none of these can supply its own `parse`
+ * in the config file rather than fork the tool.
+ */
+
+/** `${data.id}` -> { expr: 'data.id', label: 'id' } */
+export function holesIn(subUrl) {
+  return [...subUrl.matchAll(/\$\{([^}]+)\}|:([A-Za-z_][\w]*)|\{([A-Za-z_][\w]*)\}/g)].map((m) => {
+    const expr = (m[1] ?? m[2] ?? m[3]).trim();
+    return { expr, label: expr.replace(/^(data|params|request|body|payload)\??\.?\./, '') };
+  });
+}
+
+export const lineOf = (src, index) => src.slice(0, index).split('\n').length;
+
+/**
+ * Blanks out commented-out lines, keeping line numbers intact.
+ *
+ * Services routinely keep the previous URL commented directly above the live
+ * one, and reading the first match in the block would silently take the dead
+ * path. Only a line-leading `//` is stripped, so an `https://` inside a string
+ * survives.
+ */
+export function stripComments(src) {
+  return src
+    .split('\n')
+    .map((line) => (/^\s*\/\//.test(line) ? '' : line))
+    .join('\n');
+}
+
+/** reads the text between quotes, tracking `${...}` nesting inside a template */
+function readQuoted(rest, keyPattern) {
+  const quote = rest.match(keyPattern);
+  if (!quote) return null;
+  const q = quote[quote.length - 1];
+  let i = quote.index + quote[0].length;
+  let out = '';
+  let depth = 0;
+  while (i < rest.length) {
+    const c = rest[i];
+    if (c === '\\') {
+      out += rest[i] + rest[i + 1];
+      i += 2;
+      continue;
+    }
+    if (q === '`' && c === '$' && rest[i + 1] === '{') depth++;
+    if (depth > 0 && c === '}') depth--;
+    if (c === q && depth === 0) return out;
+    out += c;
+    i++;
+  }
+  return null;
+}
+
+/* ------------------------------------------------- preset: service-object */
+
+const SERVICE_VERBS = {
+  get: 'GET', get2: 'GET', getRequest: 'GET',
+  post: 'POST', postRequest: 'POST',
+  put: 'PUT', patch: 'PATCH',
+  delete: 'DELETE', deletee: 'DELETE', del: 'DELETE', destroy: 'DELETE',
+};
+
+/**
+ * One exported function per endpoint, building a request object:
+ *
+ *   export const getPatients = (params) => {
+ *     const request = { subUrl: `/v1/doctor/patients.json`, params };
+ *     return get(request);
+ *   };
+ */
+function serviceObject(src, { pathKey = 'subUrl' } = {}) {
+  const found = [];
+  const verbs = Object.keys(SERVICE_VERBS).join('|');
+  const callRe = new RegExp(`\\b(${verbs})\\s*\\(\\s*(request|config|options|req)\\b`);
+  const keyRe = new RegExp(`\\b${pathKey}\\s*:\\s*(["'\`])`);
+  const identRe = new RegExp(`\\b${pathKey}\\s*:\\s*([A-Za-z_$][\\w$]*)`);
+
+  // leading whitespace is allowed: an export inside a block is still an export
+  const starts = [...src.matchAll(/^[ \t]*export\s+(?:const|let|async\s+function|function)\s+(\w+)/gm)];
+  for (let i = 0; i < starts.length; i++) {
+    const m = starts[i];
+    const chunk = src.slice(m.index, starts[i + 1]?.index ?? src.length);
+
+    const call = chunk.match(callRe);
+    if (!call) continue;
+
+    const at = chunk.search(new RegExp(`\\b${pathKey}\\s*:`));
+    if (at === -1) continue;
+    const rest = chunk.slice(at);
+
+    let subUrl = readQuoted(rest, keyRe);
+    let dynamicUrl;
+    if (subUrl === null) {
+      // `subUrl: APIURL` — the caller supplies the whole URL, nothing to read
+      const ident = rest.match(identRe);
+      if (!ident) continue;
+      subUrl = `\${${ident[1]}}`;
+      dynamicUrl = true;
+    }
+
+    const requestObj = rest;
+    found.push({
+      name: m[1],
+      method: SERVICE_VERBS[call[1]],
+      subUrl,
+      dynamicUrl,
+      usesParams: /(^|[\s,{])params\s*[,:}]/.test(requestObj),
+      usesData: /(^|[\s,{])data\s*[,:}]/.test(requestObj),
+      absolute: /\bisOnlyURL\s*:\s*true\b/.test(chunk) || /^https?:\/\//.test(subUrl),
+      chunk,
+      index: m.index,
+    });
+  }
+  return found;
+}
+
+/* -------------------------------------------------- preset: axios-direct */
+
+/** `axios.get('/users')`, `api.post(\`/users/${id}\`, body)`, `http.delete(url)` */
+function axiosDirect(src) {
+  const found = [];
+  const re = /\b([A-Za-z_$][\w$]*)\.(get|post|put|patch|delete|head|options|request)\s*\(\s*(["'`])/g;
+  const hits = [...src.matchAll(re)];
+
+  for (let i = 0; i < hits.length; i++) {
+    const m = hits[i];
+    const rest = src.slice(m.index);
+    const subUrl = readQuoted(rest, /\.\w+\s*\(\s*(["'`])/);
+    if (subUrl === null || !subUrl.trim()) continue;
+    // a bare word or a full sentence is not a path
+    if (!/^(https?:\/\/|\/|\$\{)/.test(subUrl)) continue;
+
+    const method = m[2].toUpperCase();
+    const chunk = chunkFrom(src, m.index, hits[i + 1]?.index);
+    found.push({
+      name: nameNear(src, m.index) ?? `${method.toLowerCase()}${subUrl.replace(/[^\w]+/g, '_')}`,
+      method: method === 'REQUEST' ? 'GET' : method,
+      subUrl,
+      usesParams: /\bparams\s*:/.test(chunk),
+      usesData: ['POST', 'PUT', 'PATCH'].includes(method),
+      absolute: /^https?:\/\//.test(subUrl),
+      chunk,
+      index: m.index,
+    });
+  }
+  return found;
+}
+
+/* -------------------------------------------------- preset: fetch-direct */
+
+/** `fetch('/api/users', { method: 'POST' })` */
+function fetchDirect(src) {
+  const found = [];
+  const re = /\bfetch\s*\(\s*(["'`])/g;
+  const hits = [...src.matchAll(re)];
+
+  for (let i = 0; i < hits.length; i++) {
+    const m = hits[i];
+    const rest = src.slice(m.index);
+    const subUrl = readQuoted(rest, /\bfetch\s*\(\s*(["'`])/);
+    if (subUrl === null || !/^(https?:\/\/|\/|\$\{)/.test(subUrl)) continue;
+    const chunk = chunkFrom(src, m.index, hits[i + 1]?.index);
+    const method = (chunk.match(/method\s*:\s*["'`](\w+)["'`]/)?.[1] ?? 'GET').toUpperCase();
+    found.push({
+      name: nameNear(src, m.index) ?? `fetch${subUrl.replace(/[^\w]+/g, '_')}`,
+      method,
+      subUrl,
+      usesParams: false,
+      usesData: ['POST', 'PUT', 'PATCH'].includes(method),
+      absolute: /^https?:\/\//.test(subUrl),
+      chunk,
+      index: m.index,
+    });
+  }
+  return found;
+}
+
+/**
+ * The text belonging to one call, and not to the one after it.
+ *
+ * A fixed window would read `method: "POST"` out of the *next* fetch on the
+ * following line and label this call a POST, so the window stops at whichever
+ * comes first: the next call site, or a sensible cap.
+ */
+function chunkFrom(src, index, nextIndex, cap = 400) {
+  return src.slice(index, Math.min(nextIndex ?? src.length, index + cap));
+}
+
+/** the nearest enclosing function or const above this point, for a readable name */
+function nameNear(src, index) {
+  const before = src.slice(0, index);
+  const names = [...before.matchAll(/(?:function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=|(\w+)\s*[:(]\s*(?:async\s*)?\()/g)];
+  const last = names[names.length - 1];
+  return last ? last[1] ?? last[2] ?? last[3] : null;
+}
+
+export const PRESETS = {
+  'service-object': serviceObject,
+  'axios-direct': axiosDirect,
+  'fetch-direct': fetchDirect,
+};
+
+export const PRESET_NAMES = Object.keys(PRESETS);