deepline 0.3.104 → 0.3.106

package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts

@@ -18,6 +18,7 @@ export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
   'convex.monitor-deployment-job-tick',
   'convex.provider-monitor-global-audit-daily',
   'convex.recover-notification-delivery-work',
+  'convex.observe-operational-queue-health',
   'convex.cleanup-expired-notification-audit-records',
   'convex.fail-stale-play-deployments',
 ] as const;

package/dist/bundling-sources/shared_libs/observability/scheduled-work-v2.ts

@@ -0,0 +1,380 @@
+/**
+ * Scheduler-neutral scheduled-work contract.
+ *
+ * This is deliberately separate from scheduled-work.ts. That module is the
+ * v1 wire contract and is consumed by existing adapters. New adapters should
+ * use this module: all domain counters and declared condition codes are
+ * required, and no result-shape inference is performed.
+ */
+
+export const SCHEDULED_WORK_CONTRACT_VERSION = 2 as const;
+
+const SCHEDULED_WORK_BOILERPLATE = [
+  'publish its declared domain outcome',
+  'Declared work completes with valid accounting',
+  'the trigger is missed, execution is stuck, reporting is invalid',
+] as const;
+
+export type ScheduledCatalogEntry = {
+  readonly id: string;
+  readonly name: string;
+  readonly scheduler: string;
+  readonly cadence: string;
+  readonly timezone: string;
+  readonly source: string;
+  readonly locator: string;
+  readonly owner: string;
+  readonly criticality: string;
+  readonly purpose: string;
+  readonly successDefinition: string;
+  readonly failureDefinition: string;
+  /** Every root must report a concrete domain result, not only scheduler liveness. */
+  readonly domainReportRequired: true;
+  readonly startGraceMs: number;
+  readonly runTimeoutMs: number;
+  readonly failureAlert: string;
+};
+
+export function validateScheduledWorkCatalog(
+  catalog: readonly ScheduledCatalogEntry[],
+): void {
+  const ids = new Set<string>();
+  for (const entry of catalog) {
+    for (const field of [
+      'id',
+      'name',
+      'scheduler',
+      'cadence',
+      'timezone',
+      'source',
+      'locator',
+      'owner',
+      'criticality',
+      'purpose',
+      'successDefinition',
+      'failureDefinition',
+      'failureAlert',
+    ] as const)
+      requireText(
+        entry[field],
+        `catalog entry ${entry.id || '<unknown>'}: ${field}`,
+      );
+    if (entry.domainReportRequired !== true) {
+      throw new Error(`${entry.id}: domainReportRequired must be true`);
+    }
+    for (const field of [
+      'purpose',
+      'successDefinition',
+      'failureDefinition',
+    ] as const) {
+      if (
+        SCHEDULED_WORK_BOILERPLATE.some((phrase) =>
+          entry[field].includes(phrase),
+        )
+      ) {
+        throw new Error(
+          `${entry.id}: ${field} contains scheduled-work boilerplate`,
+        );
+      }
+    }
+    if (ids.has(entry.id))
+      throw new Error(`duplicate scheduled-work id: ${entry.id}`);
+    ids.add(entry.id);
+    if (!Number.isFinite(entry.startGraceMs) || entry.startGraceMs < 0) {
+      throw new Error(
+        `${entry.id}: startGraceMs must be finite and non-negative`,
+      );
+    }
+    if (!Number.isFinite(entry.runTimeoutMs) || entry.runTimeoutMs <= 0) {
+      throw new Error(`${entry.id}: runTimeoutMs must be finite and positive`);
+    }
+  }
+}
+
+export type Measured<T> =
+  | { readonly kind: 'exact'; readonly value: T }
+  | { readonly kind: 'lower_bound'; readonly value: T }
+  | { readonly kind: 'not_applicable'; readonly reason: string }
+  | { readonly kind: 'unavailable'; readonly reason: string };
+
+export function exact<T>(value: T): Measured<T> {
+  return { kind: 'exact', value };
+}
+
+export function lowerBound<T>(value: T): Measured<T> {
+  return { kind: 'lower_bound', value };
+}
+
+export function notApplicable<T = never>(reason: string): Measured<T> {
+  return { kind: 'not_applicable', reason };
+}
+
+export function unavailable<T = never>(reason: string): Measured<T> {
+  return { kind: 'unavailable', reason };
+}
+
+export function measuredValue<T>(measurement: Measured<T>): T | undefined {
+  return measurement.kind === 'exact' || measurement.kind === 'lower_bound'
+    ? measurement.value
+    : undefined;
+}
+
+export type ScheduledConditionSeverity = 'warning' | 'failure';
+export type ScheduledConditionOwner = 'customer' | 'provider' | 'deepline';
+
+export type ScheduledCondition = {
+  readonly severity: ScheduledConditionSeverity;
+  readonly owner: ScheduledConditionOwner;
+  readonly meaning: string;
+};
+
+export type ScheduledTrigger =
+  | {
+      readonly kind: 'periodic';
+      readonly cadence: string;
+      readonly startGraceMs: number;
+    }
+  | { readonly kind: 'on_demand' };
+
+export type ScheduledWorkDefinition<
+  W extends string = string,
+  F extends string = string,
+> = {
+  readonly id: string;
+  readonly owner: string;
+  readonly purpose: string;
+  readonly unit: string;
+  readonly trigger: ScheduledTrigger;
+  readonly runTimeoutMs: number;
+  readonly conditions: Readonly<Record<W | F, ScheduledCondition>>;
+  /** Type-only witnesses make the intended condition sets visible to TS. */
+  readonly warningCodes?: readonly W[];
+  readonly failureCodes?: readonly F[];
+};
+
+export type ScheduledWorkReport<
+  W extends string = string,
+  F extends string = string,
+> = {
+  readonly examined: number;
+  readonly actionable: number;
+  readonly dispositions: {
+    readonly succeeded: number;
+    readonly skipped: number;
+    readonly deferred: number;
+    readonly failures: Readonly<Record<F, number>>;
+  };
+  readonly warnings: Readonly<Record<W, number>>;
+  readonly failures: Readonly<Record<F, number>>;
+  readonly remaining: Measured<number>;
+  readonly noop?: { readonly reason: string };
+};
+
+type WarningCodes<C extends Record<string, ScheduledCondition>> = {
+  [K in keyof C]: C[K] extends { readonly severity: 'warning' } ? K : never;
+}[keyof C] &
+  string;
+
+type FailureCodes<C extends Record<string, ScheduledCondition>> = {
+  [K in keyof C]: C[K] extends { readonly severity: 'failure' } ? K : never;
+}[keyof C] &
+  string;
+
+export type ScheduledWorkOccurrence = {
+  readonly occurrenceId: string;
+  readonly invocationId?: string;
+  readonly expectedAt: number;
+  readonly startedAt?: number;
+  readonly heartbeatAt?: number;
+  readonly completedAt?: number;
+  readonly outcome?: 'succeeded' | 'warning' | 'failed';
+};
+
+export type ScheduledWorkHealthStatus =
+  | 'waiting'
+  | 'running'
+  | 'healthy'
+  | 'warning'
+  | 'failed'
+  | 'missed_trigger'
+  | 'stuck'
+  | 'on_demand'
+  | 'unobserved';
+
+export type ScheduledWorkHealth = {
+  readonly status: ScheduledWorkHealthStatus;
+  readonly reason:
+    | 'before_start_deadline'
+    | 'started'
+    | 'completed'
+    | 'completed_with_warnings'
+    | 'completed_with_failures'
+    | 'missed_trigger'
+    | 'runtime_timeout'
+    | 'heartbeat_timeout'
+    | 'on_demand_idle'
+    | 'missing_observation';
+};
+
+export function defineScheduledWork<
+  const C extends Readonly<Record<string, ScheduledCondition>>,
+>(
+  definition: Omit<
+    ScheduledWorkDefinition<WarningCodes<C>, FailureCodes<C>>,
+    'conditions'
+  > & { readonly conditions: C },
+) {
+  validateScheduledWorkDefinition(definition);
+  return {
+    definition,
+    report(
+      report: ScheduledWorkReport<WarningCodes<C>, FailureCodes<C>>,
+    ): ScheduledWorkReport<WarningCodes<C>, FailureCodes<C>> {
+      validateScheduledWorkReport(definition, report);
+      return report;
+    },
+  } as const;
+}
+
+export function validateScheduledWorkDefinition(
+  definition: ScheduledWorkDefinition,
+): void {
+  requireText(definition.id, 'id');
+  requireText(definition.owner, 'owner');
+  requireText(definition.purpose, 'purpose');
+  requireText(definition.unit, 'unit');
+  if (
+    !Number.isFinite(definition.runTimeoutMs) ||
+    definition.runTimeoutMs <= 0
+  ) {
+    throw new Error(`${definition.id}: runTimeoutMs must be positive`);
+  }
+  if (definition.trigger.kind === 'periodic') {
+    requireText(definition.trigger.cadence, `${definition.id}: cadence`);
+    if (
+      !Number.isFinite(definition.trigger.startGraceMs) ||
+      definition.trigger.startGraceMs < 0
+    ) {
+      throw new Error(`${definition.id}: startGraceMs must be non-negative`);
+    }
+  }
+  const codes = Object.entries(definition.conditions);
+  if (codes.length === 0) {
+    throw new Error(`${definition.id}: declare at least one condition code`);
+  }
+  const codeSet = new Set<string>();
+  for (const [code, condition] of codes) {
+    requireText(code, `${definition.id}: condition code`);
+    if (codeSet.has(code))
+      throw new Error(`${definition.id}: duplicate condition ${code}`);
+    codeSet.add(code);
+    requireText(condition.meaning, `${definition.id}: ${code}.meaning`);
+  }
+}
+
+export function validateScheduledWorkReport<W extends string, F extends string>(
+  definition: ScheduledWorkDefinition<W, F>,
+  report: ScheduledWorkReport<W, F>,
+): void {
+  for (const [name, value] of [
+    ['examined', report.examined],
+    ['actionable', report.actionable],
+    ['succeeded', report.dispositions.succeeded],
+    ['skipped', report.dispositions.skipped],
+    ['deferred', report.dispositions.deferred],
+  ] as const)
+    requireCount(definition.id, name, value);
+  if (report.actionable > report.examined) {
+    throw new Error(`${definition.id}: actionable cannot exceed examined`);
+  }
+  const conditionCodes = Object.keys(definition.conditions);
+  for (const code of conditionCodes) {
+    const condition = (
+      definition.conditions as Readonly<Record<string, ScheduledCondition>>
+    )[code]!;
+    const value =
+      condition.severity === 'warning'
+        ? report.warnings[code as W]
+        : report.failures[code as F];
+    requireCount(definition.id, `${code} condition`, value);
+  }
+  for (const [code, value] of Object.entries(report.dispositions.failures)) {
+    requireCount(definition.id, `disposition ${code}`, value as number);
+  }
+  if (
+    report.remaining.kind === 'exact' ||
+    report.remaining.kind === 'lower_bound'
+  ) {
+    requireCount(definition.id, 'remaining', report.remaining.value);
+  } else
+    requireText(report.remaining.reason, `${definition.id}: remaining.reason`);
+  const totals =
+    report.dispositions.succeeded +
+    report.dispositions.skipped +
+    report.dispositions.deferred +
+    (Object.values(report.dispositions.failures) as number[]).reduce(
+      (sum, n) => sum + n,
+      0,
+    );
+  if (totals > report.actionable) {
+    throw new Error(`${definition.id}: dispositions exceed actionable work`);
+  }
+  if (report.actionable === 0 && !report.noop) {
+    throw new Error(
+      `${definition.id}: zero actionable work requires noop.reason`,
+    );
+  }
+  if (report.noop)
+    requireText(report.noop.reason, `${definition.id}: noop.reason`);
+}
+
+export function deriveScheduledWorkHealth(
+  definition: ScheduledWorkDefinition,
+  occurrence: ScheduledWorkOccurrence | undefined,
+  now = Date.now(),
+): ScheduledWorkHealth {
+  if (definition.trigger.kind === 'on_demand' && !occurrence) {
+    return { status: 'on_demand', reason: 'on_demand_idle' };
+  }
+  if (!occurrence)
+    return { status: 'unobserved', reason: 'missing_observation' };
+  const startDeadline =
+    occurrence.expectedAt +
+    (definition.trigger.kind === 'periodic'
+      ? definition.trigger.startGraceMs
+      : 0);
+  if (!occurrence.startedAt) {
+    return now <= startDeadline
+      ? { status: 'waiting', reason: 'before_start_deadline' }
+      : { status: 'missed_trigger', reason: 'missed_trigger' };
+  }
+  if (occurrence.completedAt !== undefined) {
+    return occurrence.outcome === 'failed'
+      ? { status: 'failed', reason: 'completed_with_failures' }
+      : occurrence.outcome === 'warning'
+        ? { status: 'warning', reason: 'completed_with_warnings' }
+        : { status: 'healthy', reason: 'completed' };
+  }
+  const lastAlive = occurrence.heartbeatAt ?? occurrence.startedAt;
+  const timeoutAt = occurrence.startedAt + definition.runTimeoutMs;
+  if (now > timeoutAt) return { status: 'stuck', reason: 'runtime_timeout' };
+  if (now - lastAlive > Math.min(definition.runTimeoutMs, 5 * 60_000)) {
+    return { status: 'stuck', reason: 'heartbeat_timeout' };
+  }
+  return { status: 'running', reason: 'started' };
+}
+
+function requireText(value: string | undefined, label: string): void {
+  if (typeof value !== 'string' || value.trim() === '')
+    throw new Error(`${label} is required`);
+}
+
+function requireCount(
+  id: string,
+  label: string,
+  value: number | undefined,
+): void {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
+    throw new Error(`${id}: ${label} must be a finite non-negative number`);
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts

@@ -32,6 +32,13 @@ export type PlayLaunchQueuePolicy = {
   sandboxCapacityMaxWaitSeconds: number | null;
 };
 
+/**
+ * CLI/API launches are user-visible requests. Give a short capacity window so
+ * a temporary spike can clear, but never let an interactive run appear to
+ * succeed and then start charging much later without an explicit retry.
+ */
+export const API_LAUNCH_CAPACITY_WAIT_SECONDS = 60;
+
 export function shouldTerminalizeSandboxCapacityRejection(input: {
   triggerSource?: PlayLaunchTriggerSource | null;
   submittedAtMs?: number | null;
@@ -68,13 +75,15 @@ export function resolvePlayLaunchQueuePolicy(
     case 'api':
       return {
         triggerSource: 'api',
-        // A person waiting in the CLI/UI should get the next slot in their
-        // own organization rather than sitting behind that org's automation.
-        // The bounded deadline gives the caller a typed terminal outcome
-        // instead of an indefinite watch.
+        // API/CLI callers need a bounded, user-visible outcome. A brief
+        // capacity window absorbs normal spikes; after that the run is
+        // terminalized with a typed queue-expired/capacity result rather than
+        // starting unexpectedly much later. The priority still lets manual
+        // work win the next slot within its organization without bypassing
+        // cross-org fairness.
         priority: 100,
-        maxDelaySeconds: 10,
-        sandboxCapacityMaxWaitSeconds: 0,
+        maxDelaySeconds: API_LAUNCH_CAPACITY_WAIT_SECONDS,
+        sandboxCapacityMaxWaitSeconds: API_LAUNCH_CAPACITY_WAIT_SECONDS,
       };
     case 'webhook':
       return {
@@ -82,8 +91,7 @@ export function resolvePlayLaunchQueuePolicy(
         priority: 0,
         // A 202 webhook response is a durable delivery promise. Capacity is
         // not a delivery failure: retain it until it executes, is explicitly
-        // cancelled, or an operator terminalizes it. Interactive API launches
-        // remain separately bounded above.
+        // cancelled, or an operator terminalizes it.
         maxDelaySeconds: null,
         sandboxCapacityMaxWaitSeconds: null,
       };
@@ -91,15 +99,21 @@ export function resolvePlayLaunchQueuePolicy(
       return {
         triggerSource: 'cron',
         priority: -10,
-        maxDelaySeconds: 30 * 60,
-        sandboxCapacityMaxWaitSeconds: 30 * 60,
+        // Scheduled automation is a durable customer intent. A capacity SEV
+        // must not silently discard a scheduled run; recovery/alerting owns
+        // platform-wide exhaustion and operators can cancel explicitly.
+        maxDelaySeconds: null,
+        sandboxCapacityMaxWaitSeconds: null,
       };
     case 'sql_listener':
       return {
         triggerSource: 'sql_listener',
         priority: -10,
-        maxDelaySeconds: 30 * 60,
-        sandboxCapacityMaxWaitSeconds: 30 * 60,
+        // SQL-listener automation has the same durable intent contract as
+        // cron. Capacity pressure is recoverable state, not a user-visible
+        // queue expiry.
+        maxDelaySeconds: null,
+        sandboxCapacityMaxWaitSeconds: null,
       };
     case 'async_child':
       return {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -322,9 +322,9 @@ const HEARTBEAT_CLAIM_TIMEOUT_SECONDS = RUNTIME_CLAIM_LEASE_SECONDS;
 export const ABSURD_IDLE_POLL_INTERVAL_SECONDS = 30;
 /**
  * A direct LISTEN session is an optional latency optimization. When it is
- * unavailable, durable polling must still discover manual/API work before its
- * 10-second admission deadline. Keep enough margin for a pooled query, claim,
- * and scheduler receipt write.
+ * unavailable, durable polling should still discover newly accepted work
+ * promptly. Queue admission has source-specific deadlines, so a missed poll
+ * may add latency but must not invent a terminal result outside that policy.
  */
 export const ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS = 1_000;
 /** Heartbeat throttle: keep the lease well ahead of expiry (leaseTimeout / 3). */