@everystack/cli 0.4.2 → 0.4.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/aws.ts

@@ -5,7 +5,7 @@
  * Authentication uses the developer's IAM credentials (default credential chain).
  */
 
-import { isOpsHandlerCrash, extractCrashDiagnosis, formatOpsCrashReport } from './ops-diagnostics.js';
+import { isOpsHandlerCrash, formatOpsCrashReport, fetchDiagnosisWithRetry } from './ops-diagnostics.js';
 import { tagInvokeTransport } from './ops-advice.js';
 
 let s3Client: InstanceType<typeof import('@aws-sdk/client-s3').S3Client> | null = null;
@@ -325,50 +325,94 @@ async function fetchRecentLogText(region: string, logGroupName: string, windowMs
   return (res.events ?? []).map((e) => e.message ?? '').join('\n');
 }
 
+/** Raw CloudWatch events for a function's log group (last `windowMs`, capped). */
+export async function fetchRecentLogEvents(
+  region: string,
+  logGroupName: string,
+  windowMs: number,
+): Promise<Array<{ timestamp?: number; message?: string }>> {
+  const { CloudWatchLogsClient, FilterLogEventsCommand } = await import('@aws-sdk/client-cloudwatch-logs');
+  const client = new CloudWatchLogsClient({ region });
+  const res = await client.send(new FilterLogEventsCommand({
+    logGroupName,
+    startTime: Date.now() - windowMs,
+    limit: 500,
+  }));
+  return res.events ?? [];
+}
+
 /**
  * Turn an opaque ops-handler runtime crash into a report naming the real cause: resolve the ops
- * function's log group, read the recent lines, and extract the failing import. Best-effort — any
- * failure (no log access, no group) falls back to the where-to-look hint.
+ * function's log group, read the recent lines (retrying through CloudWatch ingest lag — a crash
+ * that JUST happened takes a few seconds to land), and extract the failing import. Best-effort —
+ * any failure (no log access, no group) falls back to the everystack-native next step.
  */
 export async function diagnoseOpsCrash(region: string, functionName: string, baseMessage: string): Promise<string> {
+  let logGroup: string | undefined;
   try {
-    const logGroup = await resolveLogGroup(region, functionName);
-    const text = await fetchRecentLogText(region, logGroup);
-    return formatOpsCrashReport(baseMessage, logGroup, extractCrashDiagnosis(text));
+    logGroup = await resolveLogGroup(region, functionName);
+    const group = logGroup;
+    const diagnosis = await fetchDiagnosisWithRetry(() => fetchRecentLogText(region, group));
+    return formatOpsCrashReport(baseMessage, logGroup, diagnosis);
   } catch {
-    return formatOpsCrashReport(baseMessage);
+    return formatOpsCrashReport(baseMessage, logGroup);
   }
 }
 
+/** IO seam for resolveLogGroup — injectable so the resolution order tests without the SDK. */
+export interface LogGroupIo {
+  /** The log group the function's own LoggingConfig names, if readable. */
+  getFunctionLogGroup: (functionName: string) => Promise<string | undefined>;
+  /** Log groups matching a name prefix. */
+  describeLogGroups: (prefix: string, limit: number) => Promise<Array<{ logGroupName?: string; creationTime?: number }>>;
+}
+
+async function defaultLogGroupIo(region: string): Promise<LogGroupIo> {
+  return {
+    getFunctionLogGroup: async (functionName) => {
+      const client = await getLambda(region);
+      const { GetFunctionConfigurationCommand } = await import('@aws-sdk/client-lambda');
+      const cfg = await client.send(new GetFunctionConfigurationCommand({ FunctionName: functionName }));
+      return cfg.LoggingConfig?.LogGroup ?? undefined;
+    },
+    describeLogGroups: async (prefix, limit) => {
+      const { CloudWatchLogsClient, DescribeLogGroupsCommand } = await import('@aws-sdk/client-cloudwatch-logs');
+      const client = new CloudWatchLogsClient({ region });
+      const res = await client.send(new DescribeLogGroupsCommand({ logGroupNamePrefix: prefix, limit }));
+      return res.logGroups ?? [];
+    },
+  };
+}
+
 /**
  * Resolve the CloudWatch log group for a Lambda function.
  *
- * First checks if the conventional `/aws/lambda/{functionName}` log group exists.
- * If not (e.g. SST replaced the function during deploy, changing the hash suffix),
- * searches for a log group matching the function's base name prefix.
+ * The function's own LoggingConfig is authoritative: SST Ion creates the log group as its OWN
+ * resource, so the function and its group carry INDEPENDENT random suffixes (`OpsFunction-vnmzcmed`
+ * writes to `/aws/lambda/...-OpsFunction-ubzfsutk`) — deriving the group from the function name is
+ * wrong for every SST function. Name heuristics (conventional `/aws/lambda/{name}`, then a prefix
+ * search for a rotated suffix) are fallbacks for when the config read is denied.
  */
 export async function resolveLogGroup(
   region: string,
   functionName: string,
+  io?: LogGroupIo,
 ): Promise<string> {
+  const ops = io ?? await defaultLogGroupIo(region);
   const conventional = `/aws/lambda/${functionName}`;
 
-  const {
-    CloudWatchLogsClient,
-    DescribeLogGroupsCommand,
-  } = await import('@aws-sdk/client-cloudwatch-logs');
-
-  const client = new CloudWatchLogsClient({ region });
+  // Authoritative: what the Lambda itself says it logs to.
+  try {
+    const configured = await ops.getFunctionLogGroup(functionName);
+    if (configured) return configured;
+  } catch {
+    // No lambda:GetFunctionConfiguration (or the function is gone) — fall back to name heuristics
+  }
 
-  // Fast path: conventional name exists
+  // Conventional name exists
   try {
-    const res = await client.send(
-      new DescribeLogGroupsCommand({
-        logGroupNamePrefix: conventional,
-        limit: 1,
-      }),
-    );
-    if (res.logGroups?.some(lg => lg.logGroupName === conventional)) {
+    const res = await ops.describeLogGroups(conventional, 1);
+    if (res.some(lg => lg.logGroupName === conventional)) {
       return conventional;
     }
   } catch {
@@ -382,20 +426,13 @@ export async function resolveLogGroup(
   if (lastDash > 0) {
     const basePrefix = `/aws/lambda/${functionName.slice(0, lastDash)}`;
     try {
-      const res = await client.send(
-        new DescribeLogGroupsCommand({
-          logGroupNamePrefix: basePrefix,
-          limit: 5,
-        }),
-      );
-      if (res.logGroups?.length) {
-        // Return the most recently created matching log group
-        const sorted = res.logGroups
-          .filter(lg => lg.logGroupName)
-          .sort((a, b) => (b.creationTime || 0) - (a.creationTime || 0));
-        if (sorted.length > 0) {
-          return sorted[0].logGroupName!;
-        }
+      const res = await ops.describeLogGroups(basePrefix, 5);
+      // Return the most recently created matching log group
+      const sorted = res
+        .filter(lg => lg.logGroupName)
+        .sort((a, b) => (b.creationTime || 0) - (a.creationTime || 0));
+      if (sorted.length > 0) {
+        return sorted[0].logGroupName!;
       }
     } catch {
       // Fall through to conventional name

package/src/cli/commands/deploy.ts

@@ -13,6 +13,8 @@
 import { spawn } from 'node:child_process';
 import { step, success, fail, info } from '../output.js';
 import { invalidateCachedConfig } from '../discover.js';
+import { resolveConfig } from '../config.js';
+import { probeDeployedFunctions } from '../deploy-probe.js';
 
 /** The `sst` argv `everystack deploy` runs. Pure, so the mapping is unit-tested. */
 export function buildDeployArgs(flags: Record<string, string>): string[] {
@@ -46,5 +48,22 @@ export async function deployCommand(flags: Record<string, string>): Promise<void
   // A deploy can change topology (a new Ops/Worker function, rotated Lambda hashes) that a
   // stale per-stage cache would shadow — bust it so the next stage command re-discovers live.
   await invalidateCachedConfig(stage);
+
+  // `sst deploy` exiting 0 only proves the code UPLOADED — a bundle that crashes at INIT still
+  // deploys "green" and takes the whole ops plane down until the next verb trips over it. Probe
+  // the deployed functions now so the operator learns here, with the real cause attached.
+  try {
+    const config = await resolveConfig(stage);
+    const { fatal } = await probeDeployedFunctions(stage, config);
+    if (fatal) {
+      fail('Deploy landed, but a deployed function cannot serve — every --stage verb will fail until this is fixed and redeployed.');
+      process.exit(1);
+    }
+  } catch (err: any) {
+    // The probe is a safety net, not a gate on discovery: a stack with nothing to probe
+    // (V1 static) or an operator without invoke rights still gets their deploy.
+    info(`Skipped the post-deploy probe: ${err?.message ?? err}`);
+  }
+
   info('Next: `everystack db:migrate --stage ' + stage + '` to apply migrations, `everystack update` to publish the app bundle.');
 }

package/src/cli/commands/logs.ts

@@ -11,7 +11,8 @@
  */
 
 import { resolveConfig } from '../config.js';
-import { resolveLogGroup } from '../aws.js';
+import { resolveLogGroup, fetchRecentLogEvents } from '../aws.js';
+import { summarizeRuntimeFailures } from '../ops-diagnostics.js';
 import { queryLogArchive, type ObservabilityQuery } from '../observability.js';
 import { step, success, fail, info } from '../output.js';
 
@@ -32,6 +33,46 @@ function parseDurationMs(duration: string): number {
   }
 }
 
+/**
+ * Sweep the stage's Lambda log groups for runtime-level failures — INIT crashes, import errors,
+ * timeouts. These kill the handler BEFORE it can ship anything to the S3 archive, so without this
+ * sweep `logs:errors` answers "No errors found!" while the ops plane is down on every invoke (a
+ * real consumer incident). Best-effort per function: no CloudWatch read access just means the S3
+ * results stand alone.
+ */
+async function sweepRuntimeFailures(
+  config: {
+    region: string;
+    opsFunctionName?: string;
+    apiFunctionName?: string;
+    workerFunctionName?: string;
+    imageFunctionName?: string;
+  },
+  windowMs: number,
+): Promise<Array<{ label: string; functionName: string; lines: string[] }>> {
+  const candidates: Array<[string, string | undefined]> = [
+    ['ops', config.opsFunctionName],
+    ['api', config.apiFunctionName],
+    ['worker', config.workerFunctionName],
+    ['media', config.imageFunctionName],
+  ];
+  const seen = new Set<string>();
+  const failures: Array<{ label: string; functionName: string; lines: string[] }> = [];
+  for (const [label, functionName] of candidates) {
+    if (!functionName || seen.has(functionName)) continue;
+    seen.add(functionName);
+    try {
+      const logGroup = await resolveLogGroup(config.region, functionName);
+      const events = await fetchRecentLogEvents(config.region, logGroup, windowMs);
+      const lines = summarizeRuntimeFailures(events);
+      if (lines.length) failures.push({ label, functionName, lines });
+    } catch {
+      // No logs:FilterLogEvents (or no group yet) — skip; the S3 results stand alone.
+    }
+  }
+  return failures;
+}
+
 export async function logsErrorsCommand(flags: Record<string, string>): Promise<void> {
   step('Resolving deployed config...');
   let config;
@@ -60,12 +101,30 @@ export async function logsErrorsCommand(flags: Record<string, string>): Promise<
   try {
     const errors = await queryLogArchive(config.region, config.updatesBucket, prefix, query);
 
-    if (errors.length === 0) {
+    // Runtime-level failures never reach the S3 archive — sweep CloudWatch in the same window.
+    const windowMs = (flags.since && parseDurationMs(flags.since)) || 60 * 60 * 1000;
+    const runtimeFailures = await sweepRuntimeFailures(config, windowMs);
+
+    if (errors.length === 0 && runtimeFailures.length === 0) {
       success('No errors found!');
       return;
     }
 
-    success(`Found ${errors.length} ${level} log(s) (s3):\n`);
+    if (errors.length === 0) {
+      info('No app-level errors in the S3 archive.');
+    } else {
+      success(`Found ${errors.length} ${level} log(s) (s3):\n`);
+    }
+    if (runtimeFailures.length > 0) {
+      fail(`Lambda runtime failures (CloudWatch) — these crash the handler before it can log to S3:\n`);
+      for (const failure of runtimeFailures) {
+        console.log(`${failure.label} (${failure.functionName}):`);
+        for (const line of failure.lines) console.log(`  ${line}`);
+        console.log('');
+      }
+      info(`Inspect further: everystack logs:tail --origin ${runtimeFailures[0].label}${flags.stage ? ` --stage ${flags.stage}` : ''}\n`);
+    }
+    if (errors.length === 0) return;
 
     for (const error of errors) {
       const ts = new Date(error.timestamp).toISOString();
@@ -97,14 +156,14 @@ export async function logsErrorsCommand(flags: Record<string, string>): Promise<
   }
 }
 
-const VALID_LOG_ORIGINS = ['api', 'media', 'worker'] as const;
+const VALID_LOG_ORIGINS = ['api', 'media', 'worker', 'ops'] as const;
 
 /**
  * Resolve a Lambda function name from the --origin flag.
  * Defaults to 'api' if no origin is specified.
  */
 function resolveFunctionForOrigin(
-  config: { apiFunctionName: string; imageFunctionName?: string; workerFunctionName?: string },
+  config: { apiFunctionName: string; imageFunctionName?: string; workerFunctionName?: string; opsFunctionName?: string },
   origin?: string,
 ): string {
   const resolved = origin || 'api';
@@ -128,6 +187,13 @@ function resolveFunctionForOrigin(
         );
       }
       return config.workerFunctionName;
+    case 'ops':
+      if (!config.opsFunctionName) {
+        throw new Error(
+          'No ops function found. Add opsFunctionName to sst.config.ts outputs or use --stage for auto-discovery.',
+        );
+      }
+      return config.opsFunctionName;
     default:
       return config.apiFunctionName;
   }

package/src/cli/deploy-probe.ts

@@ -0,0 +1,142 @@
+/**
+ * Post-deploy ops-plane probe — deploy must not exit green when the functions it just shipped
+ * cannot boot.
+ *
+ * A deploy that lands a Lambda whose bundle crashes at INIT (a failing top-level import, an
+ * import-time side effect) reports success from `sst deploy` — CloudFormation only proves the
+ * code UPLOADED, not that it runs. The operator then learns at the next db verb, from an opaque
+ * "Runtime exited with error: exit status 1", with the whole ops plane already down. The probe
+ * invokes each deployed function with `_action: '_health'` right after deploy, so the failure is
+ * caught at deploy time with the real cause attached.
+ *
+ * The probe proves BOOT, not features: an older @everystack/server without a `_health` action
+ * answers `{ error: 'Unknown action: _health' }` — the runtime evaluated the bundle, initialized,
+ * and dispatched, which is exactly what deploy needs to verify. Only a FunctionError (the handler
+ * never answered) can fail the probe; a transport/IAM error means the probe itself couldn't run
+ * and must never fail the deploy.
+ */
+
+import { isOpsHandlerCrash } from './ops-diagnostics.js';
+import { diagnoseOpsCrash } from './aws.js';
+import { step, success, fail, info } from './output.js';
+
+/** The raw outcome of one probe invoke, before classification. */
+export interface ProbeOutcome {
+  /** Lambda FunctionError message — the handler threw uncaught or the runtime crashed. */
+  functionError?: string;
+  /** Parsed response payload of a successful invoke. */
+  payload?: unknown;
+  /** The invoke call itself failed (transport/IAM) — the probe could not run. */
+  invokeError?: { name?: string; message?: string };
+}
+
+export interface ProbeVerdict {
+  verdict: 'healthy' | 'boot-ok' | 'crashed' | 'failed' | 'unreachable';
+  detail?: string;
+}
+
+/** Pure classification of a probe outcome — the decision table this feature IS. */
+export function classifyProbe(outcome: ProbeOutcome): ProbeVerdict {
+  if (outcome.invokeError) {
+    return {
+      verdict: 'unreachable',
+      detail: outcome.invokeError.message ?? outcome.invokeError.name ?? 'invoke failed',
+    };
+  }
+  if (outcome.functionError) {
+    return {
+      verdict: isOpsHandlerCrash(outcome.functionError) ? 'crashed' : 'failed',
+      detail: outcome.functionError,
+    };
+  }
+  const payload = outcome.payload;
+  if (payload && typeof payload === 'object') {
+    const p = payload as { ok?: unknown; error?: unknown };
+    if (p.ok === true) return { verdict: 'healthy' };
+    if (p.ok === false) return { verdict: 'failed', detail: String(p.error ?? 'handler reported unhealthy') };
+    if (typeof p.error === 'string') {
+      // Pre-_health servers answer "Unknown action: _health" — boot proven, feature absent.
+      if (p.error.startsWith('Unknown action')) return { verdict: 'boot-ok' };
+      return { verdict: 'failed', detail: p.error };
+    }
+  }
+  // Any other answer (null, {}, a string) still came from a booted handler.
+  return { verdict: 'boot-ok' };
+}
+
+/** Which deployed functions to probe. Deduped: older apps route ops actions to the api function. */
+export function probeTargets(config: {
+  opsFunctionName?: string;
+  apiFunctionName?: string;
+}): Array<{ label: string; functionName: string }> {
+  const targets: Array<{ label: string; functionName: string }> = [];
+  if (config.opsFunctionName) targets.push({ label: 'ops', functionName: config.opsFunctionName });
+  if (config.apiFunctionName && config.apiFunctionName !== config.opsFunctionName) {
+    targets.push({ label: 'api', functionName: config.apiFunctionName });
+  }
+  return targets;
+}
+
+/** One raw `_health` invoke → outcome. Never throws; every failure mode lands in the outcome. */
+async function probeFunction(region: string, functionName: string): Promise<ProbeOutcome> {
+  try {
+    const { LambdaClient, InvokeCommand } = await import('@aws-sdk/client-lambda');
+    const client = new LambdaClient({ region });
+    const response = await client.send(new InvokeCommand({
+      FunctionName: functionName,
+      Payload: new TextEncoder().encode(JSON.stringify({ _action: '_health', _payload: {} })),
+    }));
+    const payload = response.Payload
+      ? JSON.parse(new TextDecoder().decode(response.Payload))
+      : null;
+    if (response.FunctionError) {
+      return { functionError: String((payload as { errorMessage?: string })?.errorMessage ?? response.FunctionError) };
+    }
+    return { payload };
+  } catch (err: unknown) {
+    const e = err as { name?: string; message?: string };
+    return { invokeError: { name: e?.name, message: e?.message } };
+  }
+}
+
+/**
+ * Probe the stage's deployed functions and report. Returns whether deploy must fail: a function
+ * that cannot boot (or answers unhealthy) is fatal; a probe that couldn't run is a warning.
+ */
+export async function probeDeployedFunctions(
+  stage: string,
+  config: { region: string; opsFunctionName?: string; apiFunctionName?: string },
+): Promise<{ fatal: boolean }> {
+  const targets = probeTargets(config);
+  if (targets.length === 0) return { fatal: false };
+
+  step('Probing deployed functions...');
+  let fatal = false;
+  for (const target of targets) {
+    const outcome = await probeFunction(config.region, target.functionName);
+    const result = classifyProbe(outcome);
+    switch (result.verdict) {
+      case 'healthy':
+        success(`${target.label} function healthy (${target.functionName})`);
+        break;
+      case 'boot-ok':
+        success(`${target.label} function boots (${target.functionName})`);
+        break;
+      case 'crashed': {
+        fatal = true;
+        const report = await diagnoseOpsCrash(config.region, target.functionName, result.detail ?? 'Runtime crash');
+        fail(`${target.label} function crashed at startup (${target.functionName})`);
+        console.error(report);
+        break;
+      }
+      case 'failed':
+        fatal = true;
+        fail(`${target.label} function is unhealthy (${target.functionName}): ${result.detail}`);
+        break;
+      case 'unreachable':
+        info(`Could not probe ${target.label} function (${target.functionName}): ${result.detail}`);
+        break;
+    }
+  }
+  return { fatal };
+}

package/src/cli/ops-diagnostics.ts

@@ -43,8 +43,11 @@ export function extractCrashDiagnosis(logText: string): string | null {
 
 /**
  * Assemble the operator-facing report for an ops-handler crash: the raw message, the "it's the
- * handler, not your command" framing, and either the real cause from CloudWatch or a where-to-look
- * hint when the logs couldn't be read.
+ * handler, not your command" framing, and either the real cause from CloudWatch or an
+ * everystack-native next step when the crash lines couldn't be read. The fallback must never
+ * send the operator to the AWS console — a consumer operator may have no console access at all,
+ * and an agent told to "check CloudWatch" will reach for raw `aws` with whatever credentials the
+ * shell holds.
  */
 export function formatOpsCrashReport(
   baseMessage: string,
@@ -54,7 +57,7 @@ export function formatOpsCrashReport(
   const lines = [
     baseMessage,
     '',
-    'The ops handler crashed at startup — usually a missing dependency in the deployed bundle, not a problem with this command.',
+    'The ops handler crashed at startup — a failing top-level import or import-time side effect in the deployed bundle, not a problem with this command.',
   ];
   if (diagnosis) {
     lines.push(
@@ -62,10 +65,67 @@ export function formatOpsCrashReport(
       `From CloudWatch${logGroup ? ` (${logGroup})` : ''}:`,
       diagnosis,
       '',
-      'Fix the import/dependency and redeploy.',
+      'Fix the failing import in the ops handler and redeploy.',
     );
   } else {
-    lines.push('Check its CloudWatch logs for the failing top-level import, then redeploy.');
+    lines.push(
+      '',
+      logGroup
+        ? `No diagnostic lines in ${logGroup} yet — CloudWatch ingest can lag by a few seconds.`
+        : 'The crash lines could not be read from CloudWatch.',
+      'See the startup error with: everystack logs:tail --origin ops --stage <stage>',
+      'Fix the failing import in the ops handler and redeploy.',
+    );
   }
   return lines.join('\n');
 }
+
+/**
+ * Fetch-and-extract with retry: CloudWatch ingest lags a crash by a few seconds, so a single
+ * fetch right after the invoke routinely reads an empty window and the diagnosis degrades to
+ * the where-to-look fallback (a real consumer incident). Retries until lines land or the
+ * attempt budget runs out. Takes the fetch and sleep as thunks so it tests without the SDK.
+ */
+export async function fetchDiagnosisWithRetry(
+  fetchText: () => Promise<string>,
+  opts?: { attempts?: number; sleep?: () => Promise<void> },
+): Promise<string | null> {
+  const attempts = opts?.attempts ?? 4;
+  const sleep = opts?.sleep ?? (() => new Promise<void>((r) => setTimeout(r, 2500)));
+  for (let i = 0; i < attempts; i++) {
+    if (i > 0) await sleep();
+    const diagnosis = extractCrashDiagnosis(await fetchText());
+    if (diagnosis) return diagnosis;
+  }
+  return null;
+}
+
+/**
+ * Runtime-level failure lines: the Lambda platform's own signals that a handler never ran —
+ * INIT crashes, import errors, timeouts. These never reach the app-level S3 log archive (the
+ * handler dies before it can ship them), so `logs:errors` reads them from CloudWatch and this
+ * decides which lines are worth showing.
+ */
+const RUNTIME_FAILURE = /Runtime exited|exited with error|INIT_REPORT.*Status: (error|timeout)|Runtime\.\w+(Error|Exit)|Cannot find (module|package)|ERR_MODULE_NOT_FOUND|\bMODULE_NOT_FOUND\b|Task timed out|Process exited before completing request|(?:Reference|Type|Syntax|Range)?Error:/i;
+
+/**
+ * Reduce raw CloudWatch events to the runtime-failure lines an operator needs: filter to the
+ * platform failure signals, de-duplicate (a crash-looping Lambda emits the same line on every
+ * invoke), stamp each with its event time, and cap the output.
+ */
+export function summarizeRuntimeFailures(
+  events: Array<{ timestamp?: number; message?: string }>,
+  cap = 8,
+): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const event of events) {
+    const line = (event.message ?? '').trim();
+    if (!line || !RUNTIME_FAILURE.test(line) || seen.has(line)) continue;
+    seen.add(line);
+    const stamp = event.timestamp ? `[${new Date(event.timestamp).toISOString()}] ` : '';
+    out.push(`${stamp}${line}`);
+    if (out.length >= cap) break;
+  }
+  return out;
+}