@everystack/cli 0.2.35 → 0.2.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.2.35",
+  "version": "0.2.36",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "publishConfig": {

package/src/cli/aws.ts

@@ -141,6 +141,48 @@ export async function deleteS3(
   }
 }
 
+/**
+ * Resolve a live Lambda function name, healing a stale hash suffix.
+ *
+ * SST names functions `{app}-{stage}-{Resource}{Random}-{hash}`; a redeploy can
+ * rotate the trailing `-{hash}` while the base prefix stays stable. Given a name
+ * that may be stale, returns the live function matching the base prefix (most
+ * recently modified), or the original name if no better match is found.
+ *
+ * Symmetric with resolveLogGroup — needs only lambda:ListFunctions.
+ */
+export async function resolveFunctionName(
+  region: string,
+  functionName: string,
+): Promise<string> {
+  const lastDash = functionName.lastIndexOf('-');
+  if (lastDash <= 0) return functionName;
+  // Include the trailing dash so the prefix can't match a longer resource name.
+  const basePrefix = functionName.slice(0, lastDash + 1).toLowerCase();
+
+  const { LambdaClient, ListFunctionsCommand } = await import('@aws-sdk/client-lambda');
+  const client = new LambdaClient({ region });
+
+  const candidates: Array<{ name: string; modified: number }> = [];
+  let marker: string | undefined;
+  do {
+    const res = await client.send(new ListFunctionsCommand({ Marker: marker, MaxItems: 50 }));
+    for (const fn of res.Functions || []) {
+      if (fn.FunctionName && fn.FunctionName.toLowerCase().startsWith(basePrefix)) {
+        candidates.push({
+          name: fn.FunctionName,
+          modified: fn.LastModified ? Date.parse(fn.LastModified) : 0,
+        });
+      }
+    }
+    marker = res.NextMarker;
+  } while (marker);
+
+  if (candidates.length === 0) return functionName;
+  candidates.sort((a, b) => b.modified - a.modified);
+  return candidates[0].name;
+}
+
 export async function invokeAction(
   region: string,
   functionName: string,
@@ -149,13 +191,35 @@ export async function invokeAction(
 ): Promise<unknown> {
   const client = await getLambda(region);
   const { InvokeCommand } = await import('@aws-sdk/client-lambda');
-  const response = await client.send(new InvokeCommand({
-    FunctionName: functionName,
-    Payload: new TextEncoder().encode(JSON.stringify({
-      _action: action,
-      _payload: payload,
-    })),
+
+  const encoded = new TextEncoder().encode(JSON.stringify({
+    _action: action,
+    _payload: payload,
   }));
+  const invoke = (fnName: string) =>
+    client.send(new InvokeCommand({ FunctionName: fnName, Payload: encoded }));
+
+  let response;
+  try {
+    response = await invoke(functionName);
+  } catch (err: unknown) {
+    // Stale function hash (e.g. the discovery cache outlived a redeploy that
+    // rotated the Lambda name). Re-resolve the live function and retry once,
+    // then bust the stale cache so the next command discovers fresh.
+    if (err && typeof err === 'object' && (err as { name?: string }).name === 'ResourceNotFoundException') {
+      const live = await resolveFunctionName(region, functionName);
+      if (live === functionName) throw err;
+      try {
+        const { invalidateCacheByFunctionName } = await import('./discover.js');
+        await invalidateCacheByFunctionName(functionName);
+      } catch {
+        // Cache invalidation is best-effort
+      }
+      response = await invoke(live);
+    } else {
+      throw err;
+    }
+  }
 
   if (response.FunctionError) {
     const errorBody = response.Payload

package/src/cli/commands/logs.ts

@@ -1,15 +1,37 @@
 /**
- * Logs commands — query application logs and tail CloudWatch logs.
+ * Logs commands — query the observability log archive and tail CloudWatch logs.
  *
- * These commands help debug production issues by accessing:
- * - Application logs stored in the database (logs:errors)
- * - Lambda container logs from CloudWatch (logs:tail)
+ * All three commands read their store DIRECTLY — no API Lambda invoke:
+ * - logs:errors / logs:query → S3 NDJSON event archive (s3:ListBucket + s3:GetObject)
+ * - logs:tail               → Lambda container logs from CloudWatch
+ *
+ * Reading the store directly (rather than invoking the function being diagnosed)
+ * means a read-only observability role works uniformly, and the commands are
+ * immune to the Lambda hash rotating on redeploy.
  */
 
 import { resolveConfig } from '../config.js';
-import { invokeAction, resolveLogGroup } from '../aws.js';
+import { resolveLogGroup } from '../aws.js';
+import { queryLogArchive, type ObservabilityQuery } from '../observability.js';
 import { step, success, fail, info } from '../output.js';
 
+/** Default S3 key prefix for the observability archive (loggingPlugin default). */
+const DEFAULT_PREFIX = '_observability';
+
+/** Parse a `5m` / `1h` / `2d` duration into milliseconds; 0 if unparseable. */
+function parseDurationMs(duration: string): number {
+  const match = duration.match(/^(\d+)([smhd])$/);
+  if (!match) return 0;
+  const value = parseInt(match[1], 10);
+  switch (match[2]) {
+    case 's': return value * 1000;
+    case 'm': return value * 60 * 1000;
+    case 'h': return value * 60 * 60 * 1000;
+    case 'd': return value * 24 * 60 * 60 * 1000;
+    default: return 0;
+  }
+}
+
 export async function logsErrorsCommand(flags: Record<string, string>): Promise<void> {
   step('Resolving deployed config...');
   let config;
@@ -20,50 +42,40 @@ export async function logsErrorsCommand(flags: Record<string, string>): Promise<
     process.exit(1);
   }
 
-  info(`Region: ${config.region}, Function: ${config.apiFunctionName}`);
+  const prefix = flags.prefix || DEFAULT_PREFIX;
+  info(`Region: ${config.region}, Bucket: ${config.updatesBucket}/${prefix}`);
 
   const limit = flags.limit ? parseInt(flags.limit, 10) : 20;
   const source = flags.source;
-  const level = flags.level || 'error'; // error or fatal
+  const level = flags.level || 'error'; // error (includes fatal) or fatal
+
+  const query: ObservabilityQuery = { level, source, limit };
+  if (flags.since) {
+    const sinceMs = parseDurationMs(flags.since);
+    if (sinceMs > 0) query.startTime = new Date(Date.now() - sinceMs);
+  }
 
   step(`Fetching last ${limit} ${level} logs...`);
 
   try {
-    const result: any = await invokeAction(
-      config.region,
-      config.apiFunctionName,
-      'logs:errors',
-      { limit, source, level },
-    );
-
-    if (result?.error) {
-      fail(`Failed to fetch errors: ${result.error}`);
-      process.exit(1);
-    }
-
-    const errors = result.errors || [];
+    const errors = await queryLogArchive(config.region, config.updatesBucket, prefix, query);
 
     if (errors.length === 0) {
       success('No errors found!');
       return;
     }
 
-    const sourceLabel = result.source === 's3' ? ' (s3)' : '';
-    success(`Found ${errors.length} ${level} log(s)${sourceLabel}:\n`);
+    success(`Found ${errors.length} ${level} log(s) (s3):\n`);
 
-    // Format and display errors
     for (const error of errors) {
-      console.log(`[${error.timestamp}] ${error.level.toUpperCase()}`);
+      const ts = new Date(error.timestamp).toISOString();
+      console.log(`[${ts}] ${(error.level || 'error').toUpperCase()}`);
       console.log(`Source: ${error.source || 'unknown'}`);
       console.log(`Message: ${error.message}`);
 
-      if (error.traceId) {
-        console.log(`TraceID: ${error.traceId}`);
-      }
-
-      if (error.userId) {
-        console.log(`UserID: ${error.userId}`);
-      }
+      const userId = error.userId ?? error.data?.userId;
+      if (error.traceId) console.log(`TraceID: ${error.traceId}`);
+      if (userId) console.log(`UserID: ${userId}`);
 
       if (error.error) {
         console.log(`Error: ${error.error.name}: ${error.error.message}`);
@@ -80,7 +92,7 @@ export async function logsErrorsCommand(flags: Record<string, string>): Promise<
     }
   } catch (err: any) {
     fail(`Failed to fetch errors: ${err.message}`);
-    info('Ensure your IAM user/role has lambda:InvokeFunction permission.');
+    info('Ensure your IAM user/role has s3:ListBucket + s3:GetObject on the updates bucket.');
     process.exit(1);
   }
 }
@@ -156,8 +168,8 @@ export async function logsTailCommand(flags: Record<string, string>): Promise<vo
     // Resolve actual log group from Lambda config (handles SST name changes)
     const logGroupName = await resolveLogGroup(config.region, functionName);
 
-    // Parse since duration to milliseconds
-    const sinceMs = parseDuration(since);
+    // Parse since duration to milliseconds (default 5m if unparseable)
+    const sinceMs = parseDurationMs(since) || 5 * 60 * 1000;
     const startTime = Date.now() - sinceMs;
 
     success(`Streaming logs from ${logGroupName}...\n`);
@@ -227,10 +239,11 @@ export async function logsQueryCommand(flags: Record<string, string>): Promise<v
     process.exit(1);
   }
 
-  info(`Region: ${config.region}, Function: ${config.apiFunctionName}`);
+  const prefix = flags.prefix || DEFAULT_PREFIX;
+  info(`Region: ${config.region}, Bucket: ${config.updatesBucket}/${prefix}`);
 
   // Build query parameters
-  const query: Record<string, any> = {
+  const query: ObservabilityQuery = {
     limit: flags.limit ? parseInt(flags.limit, 10) : 50,
   };
 
@@ -243,32 +256,22 @@ export async function logsQueryCommand(flags: Record<string, string>): Promise<v
   if (flags.platform) query.platform = flags.platform;
   if (flags.environment) query.environment = flags.environment;
   if (flags.search) query.search = flags.search;
-  if (flags.since) query.since = flags.since;
+  if (flags.since) {
+    const sinceMs = parseDurationMs(flags.since);
+    if (sinceMs > 0) query.startTime = new Date(Date.now() - sinceMs);
+  }
 
-  step(`Querying logs with filters: ${JSON.stringify(query)}...`);
+  step(`Querying logs with filters: ${JSON.stringify(flags)}...`);
 
   try {
-    const result: any = await invokeAction(
-      config.region,
-      config.apiFunctionName,
-      'logs:query',
-      query,
-    );
-
-    if (result?.error) {
-      fail(`Failed to query logs: ${result.error}`);
-      process.exit(1);
-    }
-
-    const logs = result.logs || [];
+    const logs = await queryLogArchive(config.region, config.updatesBucket, prefix, query);
 
     if (logs.length === 0) {
       success('No logs found matching the criteria.');
       return;
     }
 
-    const sourceLabel = result.source === 's3' ? ' (s3)' : '';
-    success(`Found ${logs.length} log(s)${sourceLabel}:\n`);
+    success(`Found ${logs.length} log(s) (s3):\n`);
 
     // Format and display logs
     for (const log of logs) {
@@ -278,17 +281,11 @@ export async function logsQueryCommand(flags: Record<string, string>): Promise<v
       console.log(`[${timestamp}] ${level} ${log.source || 'unknown'}`);
       console.log(`  ${log.message || '(no message)'}`);
 
-      if (log.traceId) {
-        console.log(`  TraceID: ${log.traceId}`);
-      }
-
-      if (log.userId) {
-        console.log(`  UserID: ${log.userId}`);
-      }
-
-      if (log.deviceId) {
-        console.log(`  DeviceID: ${log.deviceId}`);
-      }
+      const userId = log.userId ?? log.data?.userId;
+      const deviceId = log.deviceId ?? log.data?.deviceId;
+      if (log.traceId) console.log(`  TraceID: ${log.traceId}`);
+      if (userId) console.log(`  UserID: ${userId}`);
+      if (deviceId) console.log(`  DeviceID: ${deviceId}`);
 
       if (log.error) {
         console.log(`  Error: ${log.error.name}: ${log.error.message}`);
@@ -302,25 +299,7 @@ export async function logsQueryCommand(flags: Record<string, string>): Promise<v
     }
   } catch (err: any) {
     fail(`Failed to query logs: ${err.message}`);
-    info('Ensure your IAM user/role has lambda:InvokeFunction permission.');
+    info('Ensure your IAM user/role has s3:ListBucket + s3:GetObject on the updates bucket.');
     process.exit(1);
   }
 }
-
-function parseDuration(duration: string): number {
-  const match = duration.match(/^(\d+)([smhd])$/);
-  if (!match) {
-    return 5 * 60 * 1000; // Default: 5 minutes
-  }
-
-  const value = parseInt(match[1], 10);
-  const unit = match[2];
-
-  switch (unit) {
-    case 's': return value * 1000;
-    case 'm': return value * 60 * 1000;
-    case 'h': return value * 60 * 60 * 1000;
-    case 'd': return value * 24 * 60 * 60 * 1000;
-    default: return 5 * 60 * 1000;
-  }
-}

package/src/cli/discover.ts

@@ -384,6 +384,45 @@ export async function getCachedConfig(stage: string): Promise<DiscoveredConfig |
   }
 }
 
+/**
+ * Delete any per-stage config cache that pins `staleName` as a function name.
+ *
+ * Called after an invoke hits ResourceNotFoundException on a cached name — the
+ * cache outlived a redeploy that rotated the Lambda hash. Removing the file
+ * forces the next stage command to re-discover live. Best-effort; only touches
+ * `outputs.{stage}.json` caches, never the deploy-written `outputs.json`.
+ */
+export async function invalidateCacheByFunctionName(staleName: string): Promise<void> {
+  const dir = path.resolve('.sst');
+  let entries: string[];
+  try {
+    entries = await fs.readdir(dir);
+  } catch {
+    return;
+  }
+
+  for (const entry of entries) {
+    if (!entry.startsWith('outputs.') || !entry.endsWith('.json') || entry === 'outputs.json') {
+      continue;
+    }
+    const filePath = path.join(dir, entry);
+    try {
+      const cached: CachedOutputs = JSON.parse(await fs.readFile(filePath, 'utf8'));
+      const names = [
+        cached.apiFunctionName,
+        cached.opsFunctionName,
+        cached.imageFunctionName,
+        cached.workerFunctionName,
+      ];
+      if (names.includes(staleName)) {
+        await fs.unlink(filePath);
+      }
+    } catch {
+      // Unreadable/non-matching cache — skip
+    }
+  }
+}
+
 export async function setCachedConfig(stage: string, config: DiscoveredConfig): Promise<void> {
   const data: CachedOutputs = {
     _cachedAt: Date.now(),

package/src/cli/index.ts

@@ -188,9 +188,9 @@ Usage:
   everystack diag              [url] [--path /about] [--channel <name>] [--stage <name>]  Diagnose page freshness
   everystack diag              [url] --hydration  Analyze SSR hydration issues (fetches full HTML)
   everystack analyze:ssr       [--app ./app]      Scan app code for SSR anti-patterns
-  everystack logs:errors       [--stage <name>] [--limit 20] [--level error|fatal] [--source <name>]
+  everystack logs:errors       [--stage <name>] [--limit 20] [--level error|fatal] [--source <name>] [--since 24h] [--prefix _observability]
   everystack logs:tail         [--stage <name>] [--origin api|media|worker] [--filter <pattern>] [--since 5m]
-  everystack logs:query        [--stage <name>] [--level <level>] [--source <name>] [--traceId <id>] [--search <text>]
+  everystack logs:query        [--stage <name>] [--level <level>] [--source <name>] [--traceId <id>] [--search <text>] [--since 1h] [--prefix _observability]
   everystack branches list
   everystack branches create   --name <name>
   everystack branches delete   --name <name>

package/src/cli/observability.ts

@@ -0,0 +1,196 @@
+/**
+ * Direct reader for the S3 observability log archive.
+ *
+ * Reads the NDJSON event archive written by @everystack/logging's `S3LogStorage`
+ * WITHOUT invoking the API Lambda — the read-only analog of how `logs:tail` reads
+ * CloudWatch directly. Needs only `s3:ListBucket` + `s3:GetObject` on the updates
+ * bucket, so an observability/read-only role can run `logs:errors` / `logs:query`.
+ *
+ * FORMAT CONTRACT — mirrors @everystack/logging `S3LogStorage`. Keep the two in sync:
+ *   {prefix}/events/YYYY/MM/DD/HH/{ts}-{id}.ndjson   (one JSON LogEvent per line)
+ * Hour partitions are UTC. Canonical writer + query semantics:
+ *   packages/logging/src/service/s3-log-storage.ts
+ *
+ * We read the format rather than import the class because @everystack/logging
+ * pulls React Native (admin/ui) through its peer graph — wrong for a standalone CLI.
+ */
+
+import { listS3, getFromS3 } from './aws.js';
+
+/** Severity ordering — mirrors LogLevel in @everystack/logging core/types. */
+const LOG_LEVELS: Record<string, number> = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+  fatal: 4,
+};
+
+/** Minimal shape of an archived event — matches @everystack/logging LogEvent. */
+export interface LogEventRecord {
+  id: string;
+  timestamp: number;
+  level: string;
+  message: string;
+  source?: string;
+  traceId?: string;
+  data?: Record<string, unknown>;
+  error?: { name: string; message: string; stack?: string };
+  [key: string]: unknown;
+}
+
+export interface ObservabilityQuery {
+  level?: string;
+  source?: string;
+  traceId?: string;
+  userId?: string;
+  deviceId?: string;
+  sessionId?: string;
+  platform?: string;
+  environment?: string;
+  search?: string;
+  startTime?: Date;
+  endTime?: Date;
+  limit?: number;
+  offset?: number;
+}
+
+/** Default lookback when no startTime is given — matches S3LogStorage. */
+const DEFAULT_LOOKBACK_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Max in-flight S3 requests. A busy hour partition can hold thousands of NDJSON
+ * files (S3LogStorage writes one per ingest flush); an unbounded `Promise.all`
+ * over all of them — run from a laptop, not in-Lambda — exhausts the SDK socket
+ * pool and trips throttling. 32 keeps the pipe full without stampeding.
+ */
+const MAX_S3_CONCURRENCY = 32;
+
+/**
+ * Async map with bounded concurrency. Preserves input order in the output.
+ * A small pool of workers drains a shared index — at most `limit` run at once.
+ */
+async function mapLimit<T, R>(
+  items: T[],
+  limit: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    while (next < items.length) {
+      const i = next;
+      next += 1;
+      results[i] = await fn(items[i]);
+    }
+  };
+  const pool = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+  await Promise.all(pool);
+  return results;
+}
+
+function padTwo(n: number): string {
+  return n.toString().padStart(2, '0');
+}
+
+function hourPath(date: Date): string {
+  return `${date.getUTCFullYear()}/${padTwo(date.getUTCMonth() + 1)}/${padTwo(
+    date.getUTCDate(),
+  )}/${padTwo(date.getUTCHours())}`;
+}
+
+/** All UTC hour partitions touched by [start, end], inclusive. */
+function getHourPrefixes(start: Date, end: Date): string[] {
+  const prefixes: string[] = [];
+  const current = new Date(start);
+  current.setUTCMinutes(0, 0, 0);
+  while (current <= end) {
+    prefixes.push(hourPath(current));
+    current.setUTCHours(current.getUTCHours() + 1);
+  }
+  return prefixes;
+}
+
+function parseNdjson(data: string): LogEventRecord[] {
+  const records: LogEventRecord[] = [];
+  for (const line of data.split('\n')) {
+    if (line.trim().length === 0) continue;
+    try {
+      records.push(JSON.parse(line) as LogEventRecord);
+    } catch {
+      // Skip a truncated/partial line (e.g. an interrupted multipart write)
+      // rather than failing the whole query over one malformed record.
+    }
+  }
+  return records;
+}
+
+/** Read a `data`-nested or top-level field (the app nests userId/platform under data). */
+function field(e: LogEventRecord, key: string): unknown {
+  if (e[key] !== undefined) return e[key];
+  return e.data?.[key];
+}
+
+function matches(e: LogEventRecord, q: ObservabilityQuery): boolean {
+  if (q.level) {
+    const threshold = LOG_LEVELS[q.level] ?? 0;
+    if ((LOG_LEVELS[e.level] ?? 0) < threshold) return false;
+  }
+  if (q.source && e.source !== q.source) return false;
+  if (q.traceId && e.traceId !== q.traceId) return false;
+  if (q.userId && String(field(e, 'userId') ?? '') !== String(q.userId)) return false;
+  if (q.deviceId && String(field(e, 'deviceId') ?? '') !== String(q.deviceId)) return false;
+  if (q.sessionId && String(field(e, 'sessionId') ?? '') !== String(q.sessionId)) return false;
+  if (q.platform && String(field(e, 'platform') ?? '') !== String(q.platform)) return false;
+  if (q.environment && String(field(e, 'environment') ?? '') !== String(q.environment)) return false;
+  if (q.startTime && e.timestamp < q.startTime.getTime()) return false;
+  if (q.endTime && e.timestamp > q.endTime.getTime()) return false;
+  if (q.search) {
+    const term = q.search.toLowerCase();
+    const hay = `${e.message ?? ''} ${JSON.stringify(e.data ?? {})}`.toLowerCase();
+    if (!hay.includes(term)) return false;
+  }
+  return true;
+}
+
+/**
+ * Query the S3 event archive directly. Returns newest-first, filtered + paginated.
+ */
+export async function queryLogArchive(
+  region: string,
+  bucket: string,
+  prefix: string,
+  query: ObservabilityQuery = {},
+): Promise<LogEventRecord[]> {
+  const endTime = query.endTime ?? new Date();
+  const startTime =
+    query.startTime ?? new Date(endTime.getTime() - DEFAULT_LOOKBACK_MS);
+
+  const hourPrefixes = getHourPrefixes(startTime, endTime);
+
+  // List every hour partition (bounded), then flatten to one key list. A wide
+  // window (`--since 7d` → 168 hours) lists concurrently instead of serially.
+  const keyLists = await mapLimit(hourPrefixes, MAX_S3_CONCURRENCY, hour =>
+    listS3(region, bucket, `${prefix}/events/${hour}`),
+  );
+  const keys = keyLists.flat().filter(k => k.endsWith('.ndjson'));
+
+  // Fetch + parse with the same bound — caps total in-flight GETs across the
+  // whole window, not just within a single hour.
+  const files = await mapLimit(keys, MAX_S3_CONCURRENCY, k =>
+    getFromS3(region, bucket, k),
+  );
+
+  const all: LogEventRecord[] = [];
+  for (const file of files) {
+    if (file) all.push(...parseNdjson(file.toString()));
+  }
+
+  let results = all.filter(e => matches(e, { ...query, startTime, endTime }));
+  results.sort((a, b) => b.timestamp - a.timestamp);
+
+  if (query.offset) results = results.slice(query.offset);
+  if (query.limit) results = results.slice(0, query.limit);
+
+  return results;
+}