@moretea-labs/forge 1.7.1 → 1.8.0

package/packages/kernel/scheduler/application/settlement.ts

@@ -0,0 +1,197 @@
+import type { ScheduleDecisionType, ScheduleOccurrence } from '../domain/schedule';
+import {
+  getOccurrence,
+  getSchedule,
+  recordScheduleOccurrenceHandoff,
+  saveOccurrence,
+  updateSchedule,
+  type ScheduleOccurrenceHandoffInput,
+} from '../infrastructure/schedule-store';
+
+export interface ScheduledExecutionReference {
+  type: string;
+  repoId: string;
+  jobId: string;
+  payload: Record<string, unknown>;
+}
+
+const TERMINAL_OCCURRENCE_STATUSES = new Set<ScheduleOccurrence['status']>(['dispatched', 'succeeded', 'failed', 'shadowed', 'skipped']);
+
+function computeScheduleBackoff(schedule: ReturnType<typeof getSchedule>, nextFailures: number): string {
+  const backoffBase = Math.max(1, schedule.policy.backoffBaseMinutes ?? schedule.policy.cooldownMinutes ?? 1);
+  const backoffMax = Math.max(backoffBase, schedule.policy.backoffMaxMinutes ?? 24 * 60);
+  // Persistent readiness failures may span hours or days. Cap the exponent so
+  // the calculation stays bounded even while the observable failure count grows.
+  const exponent = Math.min(16, Math.max(0, nextFailures - 1));
+  const backoffMinutes = Math.min(backoffMax, backoffBase * (2 ** exponent));
+  return new Date(Date.now() + backoffMinutes * 60_000).toISOString();
+}
+
+function computeScheduleFailureState(schedule: ReturnType<typeof getSchedule>, nextFailures: number): Pick<ReturnType<typeof getSchedule>, 'consecutiveFailures' | 'nextEligibleAt' | 'enabled' | 'pausedReason'> {
+  const shouldPause = nextFailures >= schedule.policy.maxFailures;
+  return {
+    consecutiveFailures: nextFailures,
+    nextEligibleAt: computeScheduleBackoff(schedule, nextFailures),
+    enabled: shouldPause ? false : schedule.enabled,
+    pausedReason: shouldPause ? 'Maximum consecutive failures reached.' : schedule.enabled ? undefined : schedule.pausedReason,
+  };
+}
+
+export function applyScheduleFailure(
+  controllerHome: string,
+  scheduleId: string,
+  repoId: string,
+  occurrenceId: string,
+  options: {
+    outcome: 'failed' | 'skipped';
+    decision?: ScheduleDecisionType;
+    reason: string;
+    countFailure?: boolean;
+    pauseReason?: string;
+    handoff?: ScheduleOccurrenceHandoffInput;
+  },
+): { schedule: ReturnType<typeof getSchedule>; occurrence?: ScheduleOccurrence } {
+  const schedule = getSchedule(controllerHome, repoId, scheduleId);
+  const occurrence = getOccurrence(controllerHome, repoId, occurrenceId);
+  let nextOccurrence = occurrence;
+  if (occurrence && !TERMINAL_OCCURRENCE_STATUSES.has(occurrence.status)) {
+    nextOccurrence = saveOccurrence(controllerHome, {
+      ...occurrence,
+      status: options.outcome,
+      decision: options.decision ?? occurrence.decision,
+      reason: options.reason,
+    });
+  }
+  if (nextOccurrence && options.handoff) {
+    nextOccurrence = recordScheduleOccurrenceHandoff(controllerHome, schedule, nextOccurrence, options.handoff);
+  }
+
+  const countFailure = options.countFailure !== false;
+  const nextSchedule = options.pauseReason
+    ? updateSchedule(controllerHome, repoId, scheduleId, (current) => ({
+      enabled: false,
+      pausedReason: options.pauseReason,
+      nextEligibleAt: undefined,
+      consecutiveFailures: countFailure ? current.consecutiveFailures + 1 : current.consecutiveFailures,
+    }))
+    : countFailure
+      ? updateSchedule(controllerHome, repoId, scheduleId, (current) => computeScheduleFailureState(current, current.consecutiveFailures + 1))
+      : getSchedule(controllerHome, repoId, scheduleId);
+  return { schedule: nextSchedule, occurrence: nextOccurrence };
+}
+
+export function applyScheduleRetryableFailure(
+  controllerHome: string,
+  scheduleId: string,
+  repoId: string,
+  occurrenceId: string,
+  options: {
+    outcome: 'failed' | 'skipped';
+    decision: ScheduleDecisionType;
+    reason: string;
+    countFailure?: boolean;
+  },
+): { schedule: ReturnType<typeof getSchedule>; occurrence?: ScheduleOccurrence } {
+  const occurrence = getOccurrence(controllerHome, repoId, occurrenceId);
+  let nextOccurrence = occurrence;
+  if (occurrence && !TERMINAL_OCCURRENCE_STATUSES.has(occurrence.status)) {
+    nextOccurrence = saveOccurrence(controllerHome, {
+      ...occurrence,
+      status: options.outcome,
+      decision: options.decision,
+      reason: options.reason,
+    });
+  }
+  const countFailure = options.countFailure !== false;
+  const nextSchedule = updateSchedule(controllerHome, repoId, scheduleId, (current) => {
+    const nextFailures = countFailure ? current.consecutiveFailures + 1 : current.consecutiveFailures;
+    const failureState = countFailure && current.enabled
+      ? computeScheduleFailureState(current, nextFailures)
+      : {
+        consecutiveFailures: nextFailures,
+        nextEligibleAt: countFailure
+          ? current.nextEligibleAt
+          : computeScheduleBackoff(current, Math.max(1, nextFailures)),
+        enabled: current.enabled,
+        pausedReason: current.enabled ? undefined : current.pausedReason,
+      };
+    // One transient fault backs off and re-arms. Repeated faults have crossed
+    // the already-declared failure budget and must stop before they consume
+    // further controller sessions or browser interactions.
+    return failureState;
+  });
+  return { schedule: nextSchedule, occurrence: nextOccurrence };
+}
+
+export function markScheduledExecutionRunning(controllerHome: string, job: ScheduledExecutionReference): void {
+  if (job.type !== 'scheduled-occurrence') return;
+  const occurrenceId = typeof job.payload.occurrenceId === 'string' ? job.payload.occurrenceId : undefined;
+  if (!occurrenceId) return;
+  try {
+    const occurrence = getOccurrence(controllerHome, job.repoId, occurrenceId);
+    if (occurrence && occurrence.status === 'queued') {
+      saveOccurrence(controllerHome, { ...occurrence, status: 'running', decision: 'execute', reason: 'Scheduled Worker started.' });
+    }
+  } catch {
+    // The durable Job remains authoritative if an old occurrence was removed.
+  }
+}
+
+/**
+ * Keep an occurrence and its owning schedule consistent with the terminal
+ * state of the durable ExecutionJob that implements it. This is deliberately
+ * idempotent so both the Worker and the Reconciler may call it safely.
+ */
+export function settleScheduledExecution(
+  controllerHome: string,
+  job: ScheduledExecutionReference,
+  outcome: 'succeeded' | 'failed',
+  reason: string,
+): void {
+  if (job.type !== 'scheduled-occurrence') return;
+  const scheduleId = typeof job.payload.scheduleId === 'string' ? job.payload.scheduleId : undefined;
+  const occurrenceId = typeof job.payload.occurrenceId === 'string' ? job.payload.occurrenceId : undefined;
+  if (!scheduleId || !occurrenceId) return;
+  try {
+    const occurrence = getOccurrence(controllerHome, job.repoId, occurrenceId);
+    if (occurrence && !['dispatched', 'succeeded', 'failed', 'shadowed', 'skipped'].includes(occurrence.status)) {
+      saveOccurrence(controllerHome, {
+        ...occurrence,
+        status: outcome,
+        decision: 'execute',
+        reason,
+      });
+    }
+    if (outcome === 'failed') {
+      applyScheduleFailure(controllerHome, scheduleId, job.repoId, occurrenceId, {
+        outcome,
+        decision: 'execute',
+        reason,
+        handoff: {
+          title: `Scheduled maintenance occurrence ${occurrenceId} failed`,
+          summary: 'A bounded live maintenance occurrence failed and requires review before the schedule continues unattended.',
+          reason,
+          creationReason: 'repeated_infrastructure_failure',
+          blockingDecision: 'Review the failed maintenance occurrence and decide whether the schedule should continue automatically.',
+          recommendedDecision: 'Inspect the failed occurrence, fix the runtime blocker, then re-enable or retrigger the schedule intentionally.',
+          recommendedPrompt: `Review schedule occurrence ${occurrenceId} for ${scheduleId}, inspect the failed runtime maintenance action, and decide whether to resume automatic maintenance.`,
+          statusSummary: 'Scheduled maintenance execution failed.',
+          blockedBy: ['scheduled_execution_failed'],
+          attemptedActions: [
+            `job:${job.jobId}`,
+            `operation:${String(job.payload.operation ?? 'unknown')}`,
+          ],
+        },
+      });
+      return;
+    }
+    updateSchedule(controllerHome, job.repoId, scheduleId, (current) => ({
+      consecutiveFailures: 0,
+      nextEligibleAt: undefined,
+      ...(current.enabled ? { pausedReason: undefined } : {}),
+    }));
+  } catch {
+    // Job terminal state remains authoritative even if an old schedule record
+    // has already been removed. Reconciliation must not be blocked by it.
+  }
+}

package/packages/kernel/scheduler/domain/continuation.ts

@@ -0,0 +1,38 @@
+export type ScheduledContinuationDispatchStatus =
+  | 'prepared'
+  | 'dispatching'
+  | 'dispatched'
+  | 'wait_for_user'
+  | 'semantic_wait'
+  | 'rejected'
+  | 'outcome_unknown';
+
+export interface ScheduledContinuationDispatch {
+  schemaVersion: 1;
+  repoId: string;
+  scheduleId: string;
+  occurrenceId: string;
+  workId: string;
+  /** Observed transport/session at initial dispatch; never part of semantic occurrence identity. */
+  controllerSessionId: string;
+  controllerBindingId: string;
+  relayScopeId: string;
+  controllerAuthorityId?: string;
+  status: ScheduledContinuationDispatchStatus;
+  /** Projection of ControllerRound.providerDispatchReceiptId for occurrence replay diagnostics. */
+  hostDispatchId?: string;
+  handoffId?: string;
+  reason?: string;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface ScheduledControllerContinuationInput {
+  scheduleId: string;
+  occurrenceId: string;
+  workId: string;
+  /** Observed transport/session at initial dispatch; never part of semantic occurrence identity. */
+  controllerBindingId: string;
+  relayScopeId?: string;
+  continuationHint?: string;
+}

package/packages/kernel/scheduler/domain/schedule.ts

@@ -0,0 +1,149 @@
+import type { ExecutionPlacement, ScopeRef } from '../../identity/api/index';
+
+export type ScheduleExecutionPriority = 'P0' | 'P1' | 'P2' | 'P3' | 'P4';
+
+export interface ScheduleResourceClaim {
+  resourceKey: string;
+  mode: 'read' | 'write' | 'exclusive';
+  quantity?: number;
+  /** Portable semantic claim scope when one exists. */
+  scopeRef?: ScopeRef;
+  /** Replaceable node-local execution placement. */
+  executionPlacement?: ExecutionPlacement;
+  /** @deprecated Node-local placement compatibility field. */
+  repoId?: string;
+  /** @deprecated Node-local placement compatibility field. */
+  checkoutId?: string;
+  workId?: string;
+}
+
+export type ScheduleTriggerType =
+  | 'interval'
+  | 'cron'
+  | 'calendar'
+  | 'condition'
+  | 'repository-event'
+  | 'dependency-checkpoint'
+  | 'manual';
+
+export interface ScheduleCondition {
+  kind: 'repository_clean' | 'job_succeeded' | 'job_terminal';
+  jobId?: string;
+}
+
+export interface ScheduleTrigger {
+  type: ScheduleTriggerType;
+  everyMinutes?: number;
+  cronExpression?: string;
+  timezone?: string;
+  catchUpMinutes?: number;
+  calendarAt?: string;
+  condition?: ScheduleCondition;
+  eventName?: string;
+  dependencyJobIds?: string[];
+}
+
+export interface ScheduleTriggerContext {
+  source?: 'timer' | 'manual' | 'repository-event' | 'dependency-checkpoint' | 'condition';
+  eventName?: string;
+  eventId?: string;
+  data?: Record<string, unknown>;
+}
+
+export interface SchedulePolicy {
+  maxActiveOccurrences: number;
+  maxFailures: number;
+  cooldownMinutes: number;
+  dailyBudgetMinutes: number;
+  shadowMode: boolean;
+  backoffBaseMinutes?: number;
+  backoffMaxMinutes?: number;
+}
+
+export interface ScheduleAction {
+  operation: string;
+  target?: 'repository' | 'runtime' | 'controller' | 'external';
+  arguments?: Record<string, unknown>;
+  priority?: ScheduleExecutionPriority;
+  resourceClaims?: ScheduleResourceClaim[];
+}
+
+export interface RepositorySchedule {
+  schemaVersion: 1;
+  revision: number;
+  scheduleId: string;
+  requestId: string;
+  /** Portable semantic scope for V2 records. Legacy repository schedules may omit it. */
+  scopeRef?: ScopeRef;
+  /** Node-local placement for execution. Legacy records are projected from repoId. */
+  executionPlacement?: ExecutionPlacement;
+  /** @deprecated Repository registration is placement, not portable schedule identity. */
+  repoId: string;
+  name: string;
+  enabled: boolean;
+  trigger: ScheduleTrigger;
+  policy: SchedulePolicy;
+  action: ScheduleAction;
+  stopConditions: string[];
+  createdAt: string;
+  updatedAt: string;
+  lastTriggeredAt?: string;
+  lastOccurrenceId?: string;
+  consecutiveFailures: number;
+  consecutiveNoops?: number;
+  nextEligibleAt?: string;
+  pausedReason?: string;
+  /** Last deterministic external observation for observation-driven schedules. */
+  lastObservationAt?: string;
+  lastObservationFingerprint?: string;
+  lastObservationChangedAt?: string;
+  lastObservationStatus?: 'baseline' | 'unchanged' | 'changed' | 'keepalive' | 'auth_required';
+}
+
+export type ScheduleDecisionType =
+  | 'nothing_to_do'
+  | 'would_execute'
+  | 'execute'
+  | 'cooldown'
+  | 'budget_exhausted'
+  | 'active_occurrence'
+  | 'stopped'
+  | 'operation_blocked'
+  | 'maintenance_not_ready'
+  | 'trigger_not_due'
+  | 'condition_not_met'
+  | 'dependency_not_ready'
+  | 'event_not_matched';
+
+export interface ScheduleDecision {
+  schemaVersion: 1;
+  revision: number;
+  decisionId: string;
+  occurrenceId: string;
+  scheduleId: string;
+  repoId: string;
+  requestId: string;
+  decision: ScheduleDecisionType;
+  reason?: string;
+  triggerContext?: ScheduleTriggerContext;
+  evidence?: Record<string, unknown>;
+  createdAt: string;
+}
+
+export interface ScheduleOccurrence {
+  schemaVersion: 1;
+  revision: number;
+  occurrenceId: string;
+  scheduleId: string;
+  repoId: string;
+  windowKey: string;
+  status: 'created' | 'shadowed' | 'queued' | 'running' | 'dispatched' | 'succeeded' | 'failed' | 'skipped';
+  decision: ScheduleDecisionType;
+  decisionId?: string;
+  triggerContext?: ScheduleTriggerContext;
+  createdAt: string;
+  updatedAt: string;
+  jobId?: string;
+  handoffId?: string;
+  reason?: string;
+}

package/packages/kernel/scheduler/index.ts

@@ -0,0 +1 @@
+export * from './api/index';

package/packages/kernel/scheduler/infrastructure/continuation-dispatch-store.ts

@@ -0,0 +1,38 @@
+import { withControllerLock } from '../../../../src/cli/repositories/locks';
+import { readControlPlaneRecord, writeControlPlaneRecord } from '../../../../src/runtime/control-plane/persistence/sqlite-store';
+import type { ScheduledContinuationDispatch } from '../domain/continuation';
+
+const NAMESPACE = 'scheduler_continuation_dispatch';
+
+export interface SchedulerContinuationStoreOptions {
+  controllerHome: string;
+  repoId: string;
+  now?: () => string;
+}
+
+export function getScheduledContinuationDispatch(
+  options: SchedulerContinuationStoreOptions,
+  occurrenceId: string,
+): ScheduledContinuationDispatch | undefined {
+  return readControlPlaneRecord<ScheduledContinuationDispatch>(
+    options.controllerHome, NAMESPACE, options.repoId, occurrenceId,
+  )?.value;
+}
+
+export function updateScheduledContinuationDispatch(
+  options: SchedulerContinuationStoreOptions,
+  occurrenceId: string,
+  actor: string,
+  update: (current: ScheduledContinuationDispatch | undefined, now: string) => ScheduledContinuationDispatch,
+): ScheduledContinuationDispatch {
+  return withControllerLock(options.controllerHome, { scope: 'task', repoId: options.repoId, taskId: `schedule-continuation-${occurrenceId}` }, actor, () => {
+    const current = readControlPlaneRecord<ScheduledContinuationDispatch>(options.controllerHome, NAMESPACE, options.repoId, occurrenceId);
+    const at = options.now?.() ?? new Date().toISOString();
+    const value = update(current?.value, at);
+    writeControlPlaneRecord(options.controllerHome, {
+      namespace: NAMESPACE, scope: options.repoId, key: occurrenceId, schemaVersion: 1, value,
+      action: actor, expectedRevision: current?.revision ?? null,
+    });
+    return value;
+  });
+}