@olives/devos 2.1.0

package/.agents/skills/vercel-optimize/scripts/collect-signals.mjs

@@ -0,0 +1,576 @@
+#!/usr/bin/env node
+// Emits signals.json: Vercel CLI capability probe + project config + plan +
+// usage + codebase stack + metric queries. Status → stderr, JSON → stdout.
+// Degrades gracefully when capabilities are missing.
+
+import {
+  checkCliVersion,
+  checkAuth,
+  resolveProjectId,
+  resolveCommandScope,
+  hasObservabilityPlus,
+  checkObservabilityPlusConfiguration,
+  getMetricsSchema,
+  getProjectConfig,
+  getAccountPlan,
+  getContract,
+  getUsage,
+  filterUsageByProject,
+  inferPlan,
+  queryMetric,
+  detectStack,
+  redactSensitiveText,
+} from '../lib/vercel.mjs';
+import { classifyFrameworkSupport } from '../lib/framework-support.mjs';
+import { QUERIES, TIME_WINDOW, normalizerFor } from '../lib/queries.mjs';
+
+const SCHEMA_VERSION = '1.2';
+
+const log = (...args) => console.error('[collect-signals]', ...args);
+
+function parseArgs(argv) {
+  let explicitProjectId = null;
+  let continueWithoutObservability = process.env.VERCEL_OPTIMIZE_CONTINUE_WITHOUT_OBSERVABILITY === '1';
+  let continueUnsupportedFramework = process.env.VERCEL_OPTIMIZE_CONTINUE_UNSUPPORTED_FRAMEWORK === '1';
+
+  for (const arg of argv) {
+    if (arg === '--continue-without-observability') {
+      continueWithoutObservability = true;
+      continue;
+    }
+    if (arg === '--continue-unsupported-framework') {
+      continueUnsupportedFramework = true;
+      continue;
+    }
+    if (arg.startsWith('--')) {
+      throw new Error(`UNKNOWN_ARG: ${arg}`);
+    }
+    if (!explicitProjectId) {
+      explicitProjectId = arg;
+      continue;
+    }
+    throw new Error(`UNKNOWN_ARG: ${arg}`);
+  }
+
+  return { explicitProjectId, continueWithoutObservability, continueUnsupportedFramework };
+}
+
+async function main() {
+  const { explicitProjectId, continueWithoutObservability, continueUnsupportedFramework } = parseArgs(process.argv.slice(2));
+
+  log('checking Vercel CLI version…');
+  const cli = await checkCliVersion();
+  log(`vercel CLI v${cli.join('.')} OK`);
+
+  log('checking auth…');
+  await checkAuth();
+  log('auth OK');
+
+  log('resolving project id…');
+  const project = await resolveProjectId(explicitProjectId);
+  if (!project) {
+    throw new Error(
+      'NO_PROJECT_ID: pass one as argv, set VERCEL_PROJECT_ID, or run `vercel link` in this directory.'
+    );
+  }
+  log(`project link resolved (source=${project.source}; teamScope=${project.orgId ? 'yes' : 'no'})`);
+
+  if (!project.orgId) {
+    throw new Error('PROJECT_SCOPE_UNRESOLVED: the project was resolved without an owner account. Ask the user which Vercel team or personal scope owns the project, then rerun from a linked app directory or set VERCEL_PROJECT_ID with VERCEL_ORG_ID for that scope.');
+  }
+
+  log('checking framework support…');
+  const stack = await detectStack();
+  const frameworkSupport = classifyFrameworkSupport(stack);
+  log(`framework=${stack.framework}@${stack.frameworkVersion ?? '?'} support=${frameworkSupport.status}`);
+
+  if (!frameworkSupport.ok && !continueUnsupportedFramework) {
+    writeOutput({
+      schemaVersion: SCHEMA_VERSION,
+      collectedAt: new Date().toISOString(),
+      timeWindow: TIME_WINDOW,
+      projectId: project.projectId,
+      orgId: project.orgId,
+      projectIdSource: project.source,
+      commandScope: null,
+      frameworkSupport,
+      frameworkSupportBlocker: frameworkSupport.blocker,
+      frameworkSupportDetail: frameworkSupport.detail,
+      observabilityPlus: null,
+      observabilityPlusPreflight: null,
+      observabilityPlusUsable: null,
+      observabilityPlusBlocker: null,
+      observabilityPlusBlockerDetail: null,
+      plan: {
+        plan: 'uncertain',
+        reason: 'not collected before unsupported-framework confirmation',
+      },
+      project: null,
+      contract: null,
+      usage: null,
+      usageScope: null,
+      usageTeamTotal: null,
+      usageError: 'NOT_COLLECTED_UNSUPPORTED_FRAMEWORK',
+      stack,
+      metrics: {},
+      metricsSchema: null,
+    }, { usable: true, blocker: null, detail: 'Observability Plus was not checked.' }, frameworkSupport);
+    return;
+  }
+
+  if (!frameworkSupport.ok && continueUnsupportedFramework) {
+    log('continuing after unsupported framework blocker because --continue-unsupported-framework was set');
+  }
+
+  log('resolving Vercel CLI command scope…');
+  const commandScope = await resolveCommandScope(project);
+  if (!commandScope.ok) {
+    throw new Error(`SCOPE_UNRESOLVED: ${commandScope.detail} Run \`vercel switch <team>\` or re-link with \`vercel link --yes --project <project-name-or-id> --team <team-slug>\`.`);
+  }
+  const scope = commandScope.cliScope || undefined;
+  log(`command scope resolved (source=${commandScope.source}; scoped=${scope ? 'yes' : 'no'})`);
+
+  log('validating linked project belongs to the resolved scope…');
+  const projectCfg = await getProjectConfig(project.projectId, project.orgId);
+  const projectScope = validateProjectScope(projectCfg, project);
+  if (!projectScope.ok) {
+    throw new Error(`PROJECT_SCOPE_MISMATCH: ${projectScope.detail} Ask the user to confirm the exact Vercel project and team/personal scope, then rerun after \`vercel link --yes --project <project-name-or-id> --team <team-slug>\` or after setting both VERCEL_PROJECT_ID and VERCEL_ORG_ID for the intended scope.`);
+  }
+  log(`project scope verified (source=${projectScope.source})`);
+
+  log('checking Observability Plus configuration…');
+  const observabilityPlusConfig = await checkObservabilityPlusConfiguration({
+    orgId: project.orgId,
+    projectId: project.projectId,
+  });
+  log(`observabilityPlusPreflight=${observabilityPlusConfig.access === true ? 'enabled' : observabilityPlusConfig.blocker ?? 'unknown'} (${observabilityPlusConfig.source})`);
+
+  let oplus = observabilityPlusConfig.access === true;
+  if (observabilityPlusConfig.access == null) {
+    log('Observability Plus configuration preflight inconclusive; falling back to metrics schema probe…');
+    oplus = await hasObservabilityPlus(scope);
+  }
+  log(`observabilityPlus=${oplus}`);
+
+  const schema = oplus ? await getMetricsSchema(scope) : null;
+  if (oplus && schema) {
+    const count = Array.isArray(schema) ? schema.length : (schema.metrics?.length ?? 0);
+    log(`metric catalog: ${count} metrics available`);
+  }
+
+  // Check one cheap metric before pulling slower project context. If this fails,
+  // the orchestrator can ask the user immediately instead of waiting on billing.
+  let metrics = {};
+  let metricsCanaryOk = false;
+  if (oplus) {
+    log(`checking Observability Plus metrics access (window=${TIME_WINDOW})…`);
+    const t0 = Date.now();
+    const canary = await queryMetric('vercel.request.count', {
+      aggregation: 'sum',
+      since: TIME_WINDOW,
+      limit: 1,
+      scope,
+    });
+    metricsCanaryOk = !!canary?.ok;
+    if (!metricsCanaryOk) {
+      metrics = {
+        observabilityPlusCanary: {
+          ...canary,
+          metricId: 'vercel.request.count',
+          aggregation: 'sum',
+        },
+      };
+      log(`metrics access check failed: ${canary?.code ?? 'unknown'} — skipping full metrics fan-out`);
+    } else {
+      log(`metrics access check passed in ${Date.now() - t0}ms`);
+    }
+  } else {
+    log('skipping metric queries (Observability Plus preflight did not confirm access)');
+  }
+
+  let oplusDiag = observabilityPlusConfig.access === false
+    ? {
+        usable: false,
+        blocker: observabilityPlusConfig.blocker,
+        detail: observabilityPlusConfig.detail,
+      }
+    : (metricsCanaryOk
+        ? { usable: true, blocker: null, detail: 'Observability Plus metrics access check passed.' }
+        : diagnoseObservabilityPlus(metrics, oplus));
+
+  if (!oplusDiag.usable && !continueWithoutObservability) {
+    writeOutput({
+      schemaVersion: SCHEMA_VERSION,
+      collectedAt: new Date().toISOString(),
+      timeWindow: TIME_WINDOW,
+      projectId: project.projectId,
+      orgId: project.orgId,
+      projectIdSource: project.source,
+      commandScope,
+      observabilityPlus: oplus,
+      observabilityPlusPreflight: observabilityPlusConfig,
+      observabilityPlusUsable: oplusDiag.usable,
+      observabilityPlusBlocker: oplusDiag.blocker,
+      observabilityPlusBlockerDetail: oplusDiag.detail,
+      frameworkSupport,
+      frameworkSupportBlocker: frameworkSupport.blocker,
+      frameworkSupportDetail: frameworkSupport.detail,
+      plan: {
+        plan: 'uncertain',
+        reason: 'not collected before Observability Plus blocker confirmation',
+      },
+      project: projectCfg,
+      contract: null,
+      usage: null,
+      usageScope: null,
+      usageTeamTotal: null,
+      usageError: 'NOT_COLLECTED_OBSERVABILITY_BLOCKED',
+      stack: null,
+      metrics,
+      metricsSchema: schema,
+    }, oplusDiag);
+    return;
+  }
+
+  if (!oplusDiag.usable && continueWithoutObservability) {
+    log('continuing after Observability Plus blocker because --continue-without-observability was set');
+  }
+
+  log('pulling account plan + contract + usage in parallel…');
+  const [accountPlan, contract, usageResult] = await Promise.all([
+    getAccountPlan(project.orgId || scope),
+    getContract(scope),
+    getUsage({ days: 14, scope }),
+  ]);
+
+  let usage = null;
+  let usageContextMismatch = false;
+  let usageTotalCost = null;
+  let usageScope = 'team';
+  let usageTeamTotal = null;
+  if (usageResult?.ok) {
+    usage = usageResult.data;
+    const contractContext = contract?.context;
+    if (usage?.context && contractContext && usage.context !== contractContext) {
+      usageContextMismatch = true;
+      log(`usage: WARNING context mismatch — returned context=${usage.context} but project team=${contractContext}; treating usage as unavailable for this project`);
+      usage = null;
+    } else {
+      // Capture team total pre-filter so the report can label "this project vs team-wide" honestly.
+      usageTeamTotal = sumUsageCosts(usage);
+      const filterResult = filterUsageByProject(usage, project.projectId, projectCfg?.name);
+      if (filterResult.matched) {
+        usage = filterResult.filtered;
+        usageScope = 'project';
+        usageTotalCost = sumUsageCosts(usage);
+        log(`usage: filtered to project — ~$${usageTotalCost.toFixed(2)} (team-wide ~$${usageTeamTotal.toFixed(2)}; unattributed ~$${filterResult.unattributedTotal.toFixed(2)})`);
+      } else {
+        usageTotalCost = usageTeamTotal;
+        log(`usage: ~$${usageTotalCost.toFixed(2)} billed across services (team-wide — no per-project usage rows matched the linked project; report will label this team-wide)`);
+      }
+    }
+  } else {
+    log(`usage: unavailable (${usageResult?.code ?? 'unknown'}) — degrading to scanner+metrics-only mode`);
+  }
+
+  const planInfo = inferPlan(contract, { accountPlan, usageTotalCost });
+  log(`plan=${planInfo.plan} (${planInfo.reason})`);
+
+  if (projectCfg?.error) {
+    log(`project config: failed (${projectCfg.error}) — gates that need it will skip`);
+  }
+
+  log(`stack: ${stack.framework}@${stack.frameworkVersion ?? '?'} ${stack.hasAppRouter ? 'app-router' : ''}${stack.hasPagesRouter ? ' pages-router' : ''}${stack.orm !== 'none' ? ` orm=${stack.orm}` : ''}`);
+
+  // Each query is wrapped; one failure degrades only that metric.
+  if (oplus && metricsCanaryOk) {
+    log(`querying observability metrics (${QUERIES.length} metrics in parallel)…`);
+    const t0 = Date.now();
+    metrics = await collectMetrics(scope);
+    const wallMs = Date.now() - t0;
+    const counts = Object.fromEntries(
+      Object.entries(metrics).map(([k, v]) => {
+        if (!v) return [k, 'null'];
+        if (!v.ok) return [k, `err:${v.code}`];
+        const rows = Array.isArray(v.rows) ? v.rows.length : 0;
+        return [k, `${rows} rows`];
+      })
+    );
+    log(`metrics collected in ${wallMs}ms: ${JSON.stringify(counts)}`);
+  }
+
+  // The `vercel metrics schema` probe alone is NOT a reliable usability signal:
+  // it can return OK while per-route queries fail with payment_required (metrics
+  // unavailable for the team) or FORBIDDEN (auth-scope mismatch). Diagnose AFTER
+  // running queries by counting failure codes so the orchestrator can PAUSE and
+  // surface the choice before falling back to scanner-only mode.
+  oplusDiag = observabilityPlusConfig.access === false
+    ? {
+        usable: false,
+        blocker: observabilityPlusConfig.blocker,
+        detail: observabilityPlusConfig.detail,
+      }
+    : diagnoseObservabilityPlus(metrics, oplus);
+
+
+  const output = {
+    schemaVersion: SCHEMA_VERSION,
+    collectedAt: new Date().toISOString(),
+    timeWindow: TIME_WINDOW,
+    projectId: project.projectId,
+    orgId: project.orgId,
+    projectIdSource: project.source,
+    commandScope,
+    observabilityPlus: oplus,
+    observabilityPlusPreflight: observabilityPlusConfig,
+    observabilityPlusUsable: oplusDiag.usable,
+    observabilityPlusBlocker: oplusDiag.blocker,
+    observabilityPlusBlockerDetail: oplusDiag.detail,
+    frameworkSupport,
+    frameworkSupportBlocker: frameworkSupport.blocker,
+    frameworkSupportDetail: frameworkSupport.detail,
+    plan: planInfo,
+    project: projectCfg,
+    contract,
+    usage,
+    usageScope,
+    usageTeamTotal,
+    usageError: usageResult?.ok
+      ? (usageContextMismatch ? 'USAGE_CONTEXT_MISMATCH' : null)
+      : (usageResult?.code ?? 'UNKNOWN'),
+    stack,
+    metrics,
+    metricsSchema: schema,
+  };
+
+  writeOutput(output, oplusDiag);
+}
+
+function writeOutput(output, oplusDiag, frameworkSupport = output.frameworkSupport) {
+  if (frameworkSupport?.blocker) {
+    log(`⚠ Framework is not supported for metric-backed route-to-file optimization: ${frameworkSupport.detail}`);
+    log('   The orchestrator should PAUSE and ask whether to continue with a limited platform/scanner audit.');
+  }
+  if (!oplusDiag.usable) {
+    log(`⚠ Observability Plus is NOT usable on this project: blocker=${oplusDiag.blocker} (${oplusDiag.detail})`);
+    log('   The orchestrator should PAUSE and follow the blocker-specific remediation before proceeding.');
+  }
+
+  process.stdout.write(JSON.stringify(output, null, 2) + '\n');
+  log('done');
+}
+
+function validateProjectScope(projectCfg, project) {
+  if (!projectCfg || projectCfg.error) {
+    return {
+      ok: false,
+      source: 'project-api',
+      detail: `The resolved account could not read the resolved project (project API error=${projectCfg?.error ?? 'unknown'}).`,
+    };
+  }
+
+  if (projectCfg.id && String(projectCfg.id) !== String(project.projectId)) {
+    return {
+      ok: false,
+      source: 'project-api',
+      detail: 'The project API returned a different project than the collector resolved from the link or environment.',
+    };
+  }
+
+  const ownerId = firstString(
+    projectCfg.accountId,
+    projectCfg.orgId,
+    projectCfg.ownerId,
+    projectCfg.teamId,
+    projectCfg.team?.id,
+    projectCfg.account?.id,
+    projectCfg.owner?.id,
+  );
+  if (ownerId && project.orgId && String(ownerId) !== String(project.orgId)) {
+    return {
+      ok: false,
+      source: 'project-api',
+      detail: 'The project API returned an owner account that differs from the collector-resolved account.',
+    };
+  }
+
+  return {
+    ok: true,
+    source: ownerId ? 'project-api-owner' : 'project-api-readable',
+  };
+}
+
+function firstString(...values) {
+  return values.find((value) => typeof value === 'string' && value.trim() !== '') ?? null;
+}
+
+async function collectMetrics(scope) {
+  const results = await Promise.all(
+    QUERIES.map(async (entry) => {
+      const r = await queryMetric(entry.metricId, {
+        aggregation: entry.aggregation,
+        groupBy: entry.groupBy,
+        filter: entry.filter,
+        since: TIME_WINDOW,
+        limit: entry.limit,
+        scope,
+      });
+      return [entry, r];
+    })
+  );
+
+  const out = {};
+  for (const [entry, result] of results) {
+    out[entry.id] = enrichEntry(entry, result);
+  }
+  return out;
+}
+
+function enrichEntry(entry, result) {
+  if (!result?.ok) {
+    return {
+      ...result,
+      metricId: entry.metricId,
+      aggregation: entry.aggregation,
+      groupBy: entry.groupBy,
+    };
+  }
+  const normalize = normalizerFor(entry);
+  const { rows } = normalize(result.data);
+  return {
+    ...result,
+    rows,
+    metricId: entry.metricId,
+    aggregation: entry.aggregation,
+    groupBy: entry.groupBy,
+  };
+}
+
+// `vercel usage --format json` shape is documented but not stable across CLI
+// versions; try several roots, return null if none match.
+function sumUsageCosts(usage) {
+  if (!usage) return null;
+  if (typeof usage.totalCost === 'number') return usage.totalCost;
+  if (typeof usage.totals?.billedCost === 'number') return usage.totals.billedCost;
+  if (Array.isArray(usage.services)) {
+    return usage.services.reduce((s, x) => s + (x.billedCost ?? x.cost ?? 0), 0);
+  }
+  if (Array.isArray(usage.breakdown?.data)) {
+    return usage.breakdown.data.reduce((s, d) => {
+      if (Array.isArray(d.services)) {
+        return s + d.services.reduce((ss, x) => ss + (x.billedCost ?? x.cost ?? 0), 0);
+      }
+      return s + (d.billedCost ?? d.cost ?? 0);
+    }, 0);
+  }
+  return null;
+}
+
+// Returns { usable, blocker, detail }. `blocker` enum:
+//   null | 'no_oplus_probe' | 'project_disabled' | 'payment_required' |
+//   'forbidden' | 'daily_quota_exceeded' | 'project_not_found' |
+//   'not_linked' | 'all_failed_other' | 'no_traffic'
+export function diagnoseObservabilityPlus(metrics, oplusProbe) {
+  if (!oplusProbe) {
+    return {
+      usable: false,
+      blocker: 'no_oplus_probe',
+      detail: 'vercel metrics schema returned non-OK; the team does not have Observability Plus enabled.',
+    };
+  }
+
+  const entries = Object.values(metrics);
+  if (entries.length === 0) {
+    return { usable: false, blocker: 'no_oplus_probe', detail: 'No metrics were attempted.' };
+  }
+
+  const failures = entries.filter((m) => m && m.ok === false);
+  const successes = entries.filter((m) => m && m.ok !== false);
+
+  if (successes.length === 0) {
+    const codeCounts = new Map();
+    for (const f of failures) {
+      const code = String(f.code ?? 'unknown').toLowerCase();
+      codeCounts.set(code, (codeCounts.get(code) ?? 0) + 1);
+    }
+    const top = [...codeCounts.entries()].sort((a, b) => b[1] - a[1])[0];
+    const topCode = top?.[0] ?? 'unknown';
+    if (/daily_quota_exceeded/.test(topCode)) {
+      return {
+        usable: false,
+        blocker: 'daily_quota_exceeded',
+        detail: `${top[1]}/${entries.length} metric queries hit the daily Observability query limit. Retry after the next UTC midnight reset.`,
+      };
+    }
+    if (/payment_required/.test(topCode)) {
+      const text = failures
+        .map((f) => `${f.message ?? ''}\n${f.stderr ?? ''}`)
+        .join('\n')
+        .toLowerCase();
+      if (
+        /subscription to observability plus[\s\S]{0,160}required/.test(text) ||
+        /observability plus[\s\S]{0,160}not enabled/.test(text)
+      ) {
+        return {
+          usable: false,
+          blocker: 'no_oplus_probe',
+          detail: `${top[1]}/${entries.length} metric queries need route-level Observability Plus data. Enable Observability Plus, then re-run the metric-backed audit.`,
+        };
+      }
+      return {
+        usable: false,
+        blocker: 'payment_required',
+        detail: `${top[1]}/${entries.length} metric queries returned payment_required. Route-level metrics were recognized for this team, but these queries are not usable. Check the team's Observability Plus subscription or event quota.`,
+      };
+    }
+    if (/forbidden|not_authorized|403/.test(topCode)) {
+      return {
+        usable: false,
+        blocker: 'forbidden',
+        detail: `${top[1]}/${entries.length} metric queries returned FORBIDDEN. Auth-scope mismatch — likely logged in to the wrong team (run \`vercel switch\`).`,
+      };
+    }
+    if (/project_not_found/.test(topCode)) {
+      return {
+        usable: false,
+        blocker: 'project_not_found',
+        detail: `Project ID not visible to the auth'd team. Run \`vercel switch\` or verify the project ID.`,
+      };
+    }
+    if (/not_linked/.test(topCode)) {
+      return {
+        usable: false,
+        blocker: 'not_linked',
+        detail: `${top[1]}/${entries.length} metric queries returned NOT_LINKED. Link the app directory first: \`vercel link --yes --project <project-name-or-id> --cwd <project-dir>\`; add \`--team <team-id-or-slug>\` when the team is known.`,
+      };
+    }
+    return {
+      usable: false,
+      blocker: 'all_failed_other',
+      detail: `Every metric query failed; top error code was \`${topCode}\` (${top?.[1]}/${entries.length}).`,
+    };
+  }
+
+  // Some queries succeeded; zero rows across the board = "no traffic in window",
+  // NOT an Observability Plus billing issue.
+  const totalRows = successes.reduce((s, m) => s + (Array.isArray(m.rows) ? m.rows.length : 0), 0);
+  if (totalRows === 0) {
+    return {
+      usable: true,
+      blocker: 'no_traffic',
+      detail: 'Observability Plus queries succeeded but every metric returned 0 rows. Either the project has no traffic in the 14-day window, or Observability Plus retention is limited (free tier = 1 day on Pro).',
+    };
+  }
+
+  return { usable: true, blocker: null, detail: 'Observability Plus is usable; queries returned data.' };
+}
+
+// Run main() only as a CLI; the test suite imports diagnoseObservabilityPlus directly.
+import { fileURLToPath } from 'node:url';
+import { realpathSync } from 'node:fs';
+if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) {
+  main().catch((err) => {
+    console.error('[collect-signals] FAILED:', redactSensitiveText(err.message));
+    process.exit(1);
+  });
+}