@agent-native/core 0.134.1 → 0.135.1

package/src/jobs/background-automation-runner.ts

@@ -32,6 +32,11 @@ import {
   type RequestContext,
 } from "../server/request-context.js";
 import type { JobFrontmatter } from "./frontmatter.js";
+import {
+  attachAutomationRunThread,
+  finishAutomationRun,
+  startAutomationRun,
+} from "./run-history.js";
 
 const BACKGROUND_RUN_STUCK_MS = 10 * 60_000;
 const BACKGROUND_RUN_HARD_TIMEOUT_MS = 5 * 60_000;
@@ -248,6 +253,87 @@ function createRunId(prefix: string): string {
 export async function runBackgroundAutomation(
   options: BackgroundAutomationRunOptions,
   deps: BackgroundAutomationDeps,
+): Promise<BackgroundAutomationRunResult> {
+  const { automation } = options;
+  // Bookkeeping, so it must not gate the work it describes: a history table
+  // that cannot be written should cost us the record, not the automation.
+  // Everything downstream tolerates a null id by skipping its own write.
+  let historyId: string | null = null;
+  try {
+    historyId = await startAutomationRun({
+      owner: automation.resource.owner,
+      automation: automation.name,
+      path: automation.resource.path,
+      scope: organizationIdFromResourceOwner(automation.resource.owner)
+        ? "organization"
+        : "personal",
+      orgId: options.orgId ?? null,
+    });
+  } catch (err) {
+    console.error(
+      `[automations] Could not open a history record for "${automation.name}"; running anyway:`,
+      err,
+    );
+  }
+
+  let result: BackgroundAutomationRunResult;
+  try {
+    result = await executeBackgroundAutomation(options, deps, historyId);
+  } catch (err) {
+    await recordRunOutcome(
+      historyId,
+      "error",
+      err instanceof Error ? err.message : String(err),
+    );
+    throw err;
+  }
+  // Outside the try: history is bookkeeping about the run, so a failure to
+  // write it must not turn a completed automation into a reported failure.
+  await recordRunOutcome(historyId, "success");
+  return result;
+}
+
+/**
+ * Link the run to its agent thread. Bookkeeping again: the automation is
+ * already executing by this point, so a failed write costs the cross-reference
+ * in the history view, not the run.
+ */
+async function recordRunThread(
+  historyId: string | null,
+  threadId: string,
+  runId: string,
+): Promise<void> {
+  if (!historyId) return;
+  try {
+    await attachAutomationRunThread(historyId, threadId, runId);
+  } catch (err) {
+    console.error(
+      `[automations] Could not attach thread ${threadId} to run ${historyId}:`,
+      err,
+    );
+  }
+}
+
+async function recordRunOutcome(
+  historyId: string | null,
+  status: "success" | "error",
+  error?: string,
+): Promise<void> {
+  if (!historyId) return;
+  try {
+    await finishAutomationRun(historyId, status, error);
+  } catch (err) {
+    console.error(
+      `[automations] Could not record run ${historyId} as ${status}:`,
+      err,
+    );
+  }
+}
+
+async function executeBackgroundAutomation(
+  options: BackgroundAutomationRunOptions,
+  deps: BackgroundAutomationDeps,
+  historyId: string | null,
 ): Promise<BackgroundAutomationRunResult> {
   const { automation, ownerEmail, orgId, prompt, threadTitle, usageLabel } =
     options;
@@ -291,6 +377,8 @@ export async function runBackgroundAutomation(
       const systemPrompt = await deps.getSystemPrompt(ownerEmail);
       const thread = await createThread(ownerEmail, { title: threadTitle });
       const runId = createRunId(options.runIdPrefix);
+      await recordRunThread(historyId, thread.id, runId);
+
       // Scheduled work is background work: it has no synchronous serverless
       // caller waiting on it, so it must not inherit the interactive clamp
       // (40s soft timeout, a 30s no-progress backstop at 0.75x that, and 6

package/src/jobs/cron.ts

@@ -9,12 +9,50 @@ function normalize(cronExpr: string): string {
   return ALIAS_MAP[cronExpr.trim().toLowerCase()] ?? cronExpr;
 }
 
+/**
+ * IANA zone the host process evaluates wall-clock times in. Jobs stored before
+ * schedules carried a `timezone` are interpreted here, so their meaning does
+ * not silently shift when timezone support is added.
+ */
+export function serverTimezone(): string {
+  try {
+    return Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC";
+  } catch {
+    return "UTC";
+  }
+}
+
+export function isValidTimezone(timezone: string): boolean {
+  if (!timezone.trim()) return false;
+  try {
+    new Intl.DateTimeFormat("en-US", { timeZone: timezone });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Resolve the zone a schedule's wall-clock fields are read in. An absent
+ * `timezone` means the schedule predates timezone support, so it keeps its
+ * original host-relative meaning rather than being reinterpreted as UTC.
+ */
+export function effectiveTimezone(timezone?: string | null): string {
+  return timezone && isValidTimezone(timezone) ? timezone : serverTimezone();
+}
+
 /**
  * Compute the next occurrence of a cron expression after the given date.
+ * `timezone` is the IANA zone the expression's wall-clock fields refer to.
  */
-export function nextOccurrence(cronExpr: string, after?: Date): Date {
+export function nextOccurrence(
+  cronExpr: string,
+  after?: Date,
+  timezone?: string | null,
+): Date {
   const expr = CronExpressionParser.parse(normalize(cronExpr), {
     currentDate: after ?? new Date(),
+    tz: effectiveTimezone(timezone),
   });
   const next = expr.next();
   return next.toDate();
@@ -36,13 +74,21 @@ export function isValidCron(cronExpr: string): boolean {
  * Convert a cron expression to a human-readable description.
  * Handles common patterns; falls back to the raw expression for unusual ones.
  */
-export function describeCron(cronExpr: string): string {
+export function describeCron(
+  cronExpr: string,
+  timezone?: string | null,
+): string {
   const normalized = normalize(cronExpr);
   const parts = normalized.trim().split(/\s+/);
   if (parts.length !== 5) return cronExpr;
 
   const [minute, hour, dayOfMonth, month, dayOfWeek] = parts;
 
+  // A bare wall-clock time is ambiguous the moment the host and the reader sit
+  // in different zones, so every description carrying a time names its zone.
+  const zone = (text: string) =>
+    timezone ? text + " (" + timezone + ")" : text;
+
   // Every minute
   if (normalized === "* * * * *") return "Every minute";
 
@@ -98,7 +144,7 @@ export function describeCron(cronExpr: string): string {
 
   // Every day at specific time
   if (dayOfMonth === "*" && month === "*" && dayOfWeek === "*" && time) {
-    return `Every day at ${time}`;
+    return zone(`Every day at ${time}`);
   }
 
   // Weekdays
@@ -108,18 +154,18 @@ export function describeCron(cronExpr: string): string {
     (dayOfWeek === "1-5" || dayOfWeek === "MON-FRI") &&
     time
   ) {
-    return `Every weekday at ${time}`;
+    return zone(`Every weekday at ${time}`);
   }
 
   // Specific day of week
   if (dayOfMonth === "*" && month === "*" && dayOfWeek !== "*" && time) {
     const days = dayOfWeek.split(",").map((d) => dayNames[d] || d);
-    return `Every ${days.join(", ")} at ${time}`;
+    return zone(`Every ${days.join(", ")} at ${time}`);
   }
 
   // Specific day of month
   if (dayOfMonth !== "*" && month === "*" && dayOfWeek === "*" && time) {
-    return `On day ${dayOfMonth} of every month at ${time}`;
+    return zone(`On day ${dayOfMonth} of every month at ${time}`);
   }
 
   return cronExpr;

package/src/jobs/frontmatter.ts

@@ -11,10 +11,22 @@ export type JobExecutionMode = "agentic" | "deterministic";
 export interface JobFrontmatter {
   schedule: string;
   enabled: boolean;
+  /**
+   * IANA zone the cron fields are read in. Absent means the schedule predates
+   * timezone support and keeps its original host-relative meaning.
+   */
+  timezone?: string;
   createdBy?: string;
   orgId?: string;
   runAs?: "creator" | "shared";
+  /** Last time the automation actually started executing. */
   lastRun?: string;
+  /**
+   * Last time a tick evaluated this automation and declined to run it. Kept
+   * distinct from `lastRun` so a blocked automation cannot report a run it
+   * never performed.
+   */
+  lastCheck?: string;
   lastStatus?: JobLastStatus;
   lastError?: string;
   nextRun?: string;
@@ -132,6 +144,9 @@ function parseKnownField(
     case "enabled":
       meta.enabled = value !== "false";
       break;
+    case "timezone":
+      meta.timezone = value || undefined;
+      break;
     case "createdBy":
       meta.createdBy = value;
       break;
@@ -145,6 +160,9 @@ function parseKnownField(
     case "lastRun":
       meta.lastRun = value;
       break;
+    case "lastCheck":
+      meta.lastCheck = value;
+      break;
     case "lastStatus":
       meta.lastStatus = value as JobLastStatus;
       break;
@@ -290,7 +308,9 @@ export function buildJobResourceContent(
   pushString(lines, "createdBy", meta.createdBy, false);
   pushString(lines, "orgId", meta.orgId);
   if (meta.runAs) lines.push(`runAs: ${meta.runAs}`);
+  pushString(lines, "timezone", meta.timezone);
   pushString(lines, "lastRun", meta.lastRun);
+  pushString(lines, "lastCheck", meta.lastCheck);
   if (meta.lastStatus) lines.push(`lastStatus: ${meta.lastStatus}`);
   pushString(lines, "lastError", meta.lastError);
   pushString(lines, "nextRun", meta.nextRun);

package/src/jobs/run-history.ts

@@ -0,0 +1,242 @@
+import { randomUUID } from "node:crypto";
+
+import { getDbExec, intType, isPostgres } from "../db/client.js";
+import { ensureIndexExists, ensureTableExists } from "../db/ddl-guard.js";
+
+/**
+ * "interrupted" is derived at read time, never stored: a process killed
+ * mid-run cannot write its own outcome, so a row left running long past the
+ * point a run could still be alive is reported as interrupted rather than
+ * shown as permanently in-flight.
+ */
+export type AutomationRunStatus =
+  | "running"
+  | "success"
+  | "error"
+  | "interrupted";
+
+export interface AutomationRun {
+  id: string;
+  owner: string;
+  automation: string;
+  path: string;
+  scope: string | null;
+  orgId: string | null;
+  runId: string | null;
+  threadId: string | null;
+  status: AutomationRunStatus;
+  startedAt: number;
+  finishedAt: number | null;
+  error: string | null;
+}
+
+export interface StartAutomationRunInput {
+  owner: string;
+  automation: string;
+  path: string;
+  scope?: string | null;
+  orgId?: string | null;
+  runId?: string | null;
+  threadId?: string | null;
+}
+
+const TABLE = "automation_runs";
+const MAX_ERROR_LENGTH = 500;
+
+/**
+ * Generous multiple of the runner's own 5 minute hard abort
+ * (BACKGROUND_RUN_HARD_TIMEOUT_MS). Past this, no run is still alive.
+ */
+const RUN_LIVENESS_CEILING_MS = 15 * 60_000;
+
+/** Rows kept per automation, so a per-minute schedule cannot grow forever. */
+const RUNS_RETAINED_PER_AUTOMATION = 50;
+
+let _initPromise: Promise<void> | undefined;
+
+async function ensureTable(): Promise<void> {
+  if (!_initPromise) {
+    _initPromise = (async () => {
+      const client = getDbExec();
+      const createSql = `
+        CREATE TABLE IF NOT EXISTS ${TABLE} (
+          id TEXT PRIMARY KEY,
+          owner TEXT NOT NULL,
+          automation TEXT NOT NULL,
+          path TEXT NOT NULL,
+          scope TEXT,
+          org_id TEXT,
+          run_id TEXT,
+          thread_id TEXT,
+          status TEXT NOT NULL DEFAULT 'running',
+          started_at ${intType()} NOT NULL,
+          finished_at ${intType()},
+          error TEXT
+        )
+      `;
+      const indexSql = `CREATE INDEX IF NOT EXISTS idx_${TABLE}_owner_automation ON ${TABLE} (owner, automation, started_at)`;
+
+      if (isPostgres()) {
+        await ensureTableExists(TABLE, createSql);
+        await ensureIndexExists(`idx_${TABLE}_owner_automation`, indexSql);
+        return;
+      }
+
+      await client.execute(createSql);
+      await client.execute(indexSql);
+    })().catch((err) => {
+      _initPromise = undefined;
+      throw err;
+    });
+  }
+  return _initPromise;
+}
+
+function toRun(row: Record<string, unknown>, now: number): AutomationRun {
+  const stored = String(row.status) as AutomationRunStatus;
+  const startedAt = Number(row.started_at);
+  const status: AutomationRunStatus =
+    stored === "running" && now - startedAt > RUN_LIVENESS_CEILING_MS
+      ? "interrupted"
+      : stored;
+  return {
+    id: String(row.id),
+    owner: String(row.owner),
+    automation: String(row.automation),
+    path: String(row.path),
+    scope: row.scope == null ? null : String(row.scope),
+    orgId: row.org_id == null ? null : String(row.org_id),
+    runId: row.run_id == null ? null : String(row.run_id),
+    threadId: row.thread_id == null ? null : String(row.thread_id),
+    status,
+    startedAt,
+    finishedAt: row.finished_at == null ? null : Number(row.finished_at),
+    error: row.error == null ? null : String(row.error),
+  };
+}
+
+/**
+ * Record that an automation actually began executing. Returns the run id used
+ * to close the record out. Only real executions get a row — a tick that
+ * declined to run the automation must not appear in its history.
+ */
+export async function startAutomationRun(
+  input: StartAutomationRunInput,
+): Promise<string> {
+  await ensureTable();
+  const id = randomUUID();
+  await getDbExec().execute({
+    sql: `INSERT INTO ${TABLE} (id, owner, automation, path, scope, org_id, run_id, thread_id, status, started_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'running', ?)`,
+    args: [
+      id,
+      input.owner,
+      input.automation,
+      input.path,
+      input.scope ?? null,
+      input.orgId ?? null,
+      input.runId ?? null,
+      input.threadId ?? null,
+      Date.now(),
+    ],
+  });
+  await pruneAutomationRuns(input.owner, input.automation);
+  return id;
+}
+
+/**
+ * Drop the oldest rows for one automation once it exceeds the retention cap.
+ *
+ * Pruning on insert keeps the table bounded by the number of automations
+ * rather than by how often they run, and avoids a separate sweeper. The
+ * derived table is aliased because Postgres requires it.
+ */
+async function pruneAutomationRuns(
+  owner: string,
+  automation: string,
+): Promise<void> {
+  await getDbExec().execute({
+    sql: `DELETE FROM ${TABLE}
+          WHERE owner = ? AND automation = ? AND started_at < (
+            SELECT MIN(started_at) FROM (
+              SELECT started_at FROM ${TABLE}
+              WHERE owner = ? AND automation = ?
+              ORDER BY started_at DESC
+              LIMIT ${RUNS_RETAINED_PER_AUTOMATION}
+            ) recent
+          )`,
+    args: [owner, automation, owner, automation],
+  });
+}
+
+export async function finishAutomationRun(
+  id: string,
+  status: Exclude<AutomationRunStatus, "running">,
+  error?: string,
+): Promise<void> {
+  await ensureTable();
+  await getDbExec().execute({
+    sql: `UPDATE ${TABLE} SET status = ?, finished_at = ?, error = ? WHERE id = ?`,
+    args: [status, Date.now(), error?.slice(0, MAX_ERROR_LENGTH) ?? null, id],
+  });
+}
+
+/**
+ * Attach the agent thread once it exists. The thread is created after the run
+ * row so the history survives a crash between the two.
+ */
+export async function attachAutomationRunThread(
+  id: string,
+  threadId: string,
+  runId: string,
+): Promise<void> {
+  await ensureTable();
+  await getDbExec().execute({
+    sql: `UPDATE ${TABLE} SET thread_id = ?, run_id = ? WHERE id = ?`,
+    args: [threadId, runId, id],
+  });
+}
+
+/**
+ * Forget an automation's executions.
+ *
+ * History is keyed by the automation's name, which is reusable: deleting
+ * "digest" and creating a new "digest" would otherwise show the old
+ * definition's runs as the new one's history.
+ */
+export async function deleteAutomationRuns(
+  owner: string,
+  automation: string,
+): Promise<void> {
+  await ensureTable();
+  // Bounded to runs that had already started. Names are reusable, so if a new
+  // automation takes this name and starts running before the cleanup lands,
+  // the cutoff keeps that run's history from being swept up with the old
+  // definition's.
+  const cutoff = Date.now();
+  await getDbExec().execute({
+    sql: `DELETE FROM ${TABLE} WHERE owner = ? AND automation = ? AND started_at <= ?`,
+    args: [owner, automation, cutoff],
+  });
+}
+
+export async function listAutomationRuns(options: {
+  owners: string[];
+  automation: string;
+  limit?: number;
+}): Promise<AutomationRun[]> {
+  await ensureTable();
+  const owners = options.owners.filter(Boolean);
+  if (!owners.length) return [];
+  const limit = Math.min(Math.max(options.limit ?? 20, 1), 100);
+  const placeholders = owners.map(() => "?").join(", ");
+  const result = await getDbExec().execute({
+    sql: `SELECT * FROM ${TABLE} WHERE owner IN (${placeholders}) AND automation = ?
+          ORDER BY started_at DESC LIMIT ${limit}`,
+    args: [...owners, options.automation],
+  });
+  const now = Date.now();
+  return (result.rows ?? []).map((row) =>
+    toRun(row as Record<string, unknown>, now),
+  );
+}

package/src/jobs/scheduler.ts

@@ -1,4 +1,5 @@
 import {
+  resourceGetByPath,
   resourceListAllOwners,
   resourcePut,
   type Resource,
@@ -11,7 +12,12 @@ import {
   type BackgroundAutomationContext,
   type BackgroundAutomationDeps,
 } from "./background-automation-runner.js";
-import { nextOccurrence, isValidCron, describeCron } from "./cron.js";
+import {
+  nextOccurrence,
+  isValidCron,
+  describeCron,
+  effectiveTimezone,
+} from "./cron.js";
 import {
   buildJobResourceContent,
   parseJobResource,
@@ -138,7 +144,7 @@ export async function processRecurringJobs(deps: SchedulerDeps): Promise<void> {
         // Stuck — reset so the next check can re-run it
         meta.lastStatus = "error";
         meta.lastError = "Job timed out or server crashed mid-run";
-        const next = nextOccurrence(meta.schedule, now);
+        const next = nextOccurrence(meta.schedule, now, meta.timezone);
         meta.nextRun = next.toISOString();
         await updateResource(resource, meta, body);
         continue;
@@ -153,7 +159,7 @@ export async function processRecurringJobs(deps: SchedulerDeps): Promise<void> {
         // real next occurrence. Computing from new Date(0) (the epoch) always
         // returns a 1970 date, which is < now, so the job would fire
         // immediately on first sight regardless of its schedule.
-        const next = nextOccurrence(meta.schedule, now);
+        const next = nextOccurrence(meta.schedule, now, meta.timezone);
         meta.nextRun = next.toISOString();
         await updateResource(resource, meta, body);
         continue;
@@ -216,10 +222,16 @@ async function executeJob(
         `User/membership no longer valid — leaving cron entry for admin review.`,
     );
     // Mark as skipped without resetting nextRun so an admin can find it.
-    meta.lastRun = now.toISOString();
+    // `lastRun` is deliberately untouched: the job did not run, and stamping
+    // it here made a permanently blocked job look like it ran every minute.
+    // Re-writing an unchanged resource on every tick also churns the poll
+    // stream, so only persist when the failure state actually changed.
+    const alreadyRecorded =
+      meta.lastStatus === "skipped" && meta.lastError === identity.reason;
+    meta.lastCheck = now.toISOString();
     meta.lastStatus = "skipped";
     meta.lastError = identity.reason;
-    await updateResource(resource, meta, body);
+    if (!alreadyRecorded) await updateResource(resource, meta, body);
     return;
   }
   const jobUserEmail = identity.identity.userEmail;
@@ -263,7 +275,7 @@ async function executeJob(
         automation: jobContext,
         ownerEmail: jobUserEmail,
         orgId: jobOrgId,
-        prompt: `[Recurring Job: ${jobName}]\nSchedule: ${describeCron(meta.schedule)}\n\nExecute the following job instructions:\n\n${body}`,
+        prompt: `[Recurring Job: ${jobName}]\nSchedule: ${describeCron(meta.schedule, effectiveTimezone(meta.timezone))}\n\nExecute the following job instructions:\n\n${body}`,
         threadTitle: `Job: ${jobName} — ${now.toLocaleDateString()}`,
         runIdPrefix: `job-${jobName}`,
         usageLabel: `recurring-job:${jobName}`,
@@ -272,20 +284,21 @@ async function executeJob(
       deps,
     );
 
-    // Compute from completion time so long runs cannot immediately re-fire.
-    meta.lastStatus = "success";
-    meta.nextRun = nextOccurrence(meta.schedule, new Date()).toISOString();
-    await updateResource(resource, meta, body);
-    console.log(
-      `[recurring-jobs] Job "${jobName}" completed. Next run: ${meta.nextRun}`,
-    );
+    await recordExecutionOutcome(resource, {
+      lastRun: meta.lastRun,
+      lastStatus: "success",
+      lastError: undefined,
+    });
+    console.log(`[recurring-jobs] Job "${jobName}" completed.`);
   } catch (err) {
-    meta.lastStatus = "error";
-    meta.lastError =
+    const lastError =
       err instanceof Error ? err.message.slice(0, 200) : "Unknown error";
-    meta.nextRun = nextOccurrence(meta.schedule, new Date()).toISOString();
-    await updateResource(resource, meta, body);
-    console.error(`[recurring-jobs] Job "${jobName}" failed:`, meta.lastError);
+    await recordExecutionOutcome(resource, {
+      lastRun: meta.lastRun,
+      lastStatus: "error",
+      lastError,
+    });
+    console.error(`[recurring-jobs] Job "${jobName}" failed:`, lastError);
   }
 }
 
@@ -297,3 +310,45 @@ async function updateResource(
   const content = buildJobContent(meta, body);
   await resourcePut(resource.owner, resource.path, content);
 }
+
+/** Execution bookkeeping the scheduler owns; the rest belongs to the editor. */
+type ExecutionOutcome = Pick<
+  JobFrontmatter,
+  "lastRun" | "lastCheck" | "lastStatus" | "lastError"
+>;
+
+/**
+ * Persist the result of a run without clobbering a concurrent edit.
+ *
+ * A run holds its `meta` for as long as the job takes, so writing that whole
+ * snapshot back on completion would silently revert a schedule, timezone or
+ * instruction change made while it was running. Only the execution fields are
+ * ours to write, and `nextRun` is recomputed from whatever schedule is stored
+ * now, so an edit mid-run takes effect on the next tick.
+ */
+async function recordExecutionOutcome(
+  resource: Resource,
+  outcome: ExecutionOutcome,
+): Promise<void> {
+  const latest = await resourceGetByPath(resource.owner, resource.path);
+  if (!latest) {
+    // Deleted while it was running. Writing the pre-run snapshot back would
+    // resurrect the automation and schedule it again.
+    console.log(
+      `[recurring-jobs] "${resource.path}" was deleted mid-run; dropping its outcome.`,
+    );
+    return;
+  }
+  const current = parseJobResource(latest.content);
+
+  const meta: JobFrontmatter = { ...current.meta, ...outcome };
+  if (meta.schedule && isValidCron(meta.schedule)) {
+    // Measured from completion so a long run cannot immediately re-fire.
+    meta.nextRun = nextOccurrence(
+      meta.schedule,
+      new Date(),
+      meta.timezone,
+    ).toISOString();
+  }
+  await updateResource(resource, meta, current.body);
+}