create-windy 0.1.0

package/template/apps/server/src/scheduler/service.test.ts

@@ -0,0 +1,330 @@
+import { describe, expect, test } from "bun:test";
+import { entitlementKey, licenseVersionKey } from "@windy/shared";
+import { createFoundationSnapshot } from "../foundation.js";
+import { createServerRuntime } from "../runtime.js";
+import { InMemorySchedulerRepository } from "./repository.js";
+import { SchedulerService } from "./service.js";
+import type { ScheduledTaskDefinition, SchedulerClock } from "./types.js";
+
+class FakeClock implements SchedulerClock {
+  constructor(private current: Date) {}
+
+  now() {
+    return new Date(this.current);
+  }
+
+  advance(seconds: number) {
+    this.current = new Date(this.current.getTime() + seconds * 1_000);
+  }
+}
+
+describe("SchedulerService", () => {
+  test("重复 tick 不会并发执行同一任务", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    let release: () => void = () => undefined;
+    const barrier = new Promise<void>((resolve) => (release = resolve));
+    let executions = 0;
+    const fixture = await createFixture(clock, async () => {
+      executions += 1;
+      await barrier;
+      return { execution: executions };
+    });
+    clock.advance(100);
+
+    const firstTick = fixture.scheduler.tick();
+    await until(() => executions === 1);
+    const duplicate = await fixture.scheduler.tick();
+    release();
+    const first = await firstTick;
+
+    expect(first.scheduled).toBe(1);
+    expect(duplicate.scheduled).toBe(0);
+    expect(executions).toBe(1);
+    expect(await fixture.scheduler.listRuns()).toEqual([
+      expect.objectContaining({ status: "succeeded", attempt: 1 }),
+    ]);
+  });
+
+  test("失败按 fake clock 延迟重试并保留每次历史", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    let executions = 0;
+    const fixture = await createFixture(clock, () => {
+      executions += 1;
+      if (executions < 3) throw new Error(`第 ${executions} 次失败`);
+      return { recovered: true };
+    });
+    clock.advance(100);
+
+    expect(await fixture.scheduler.tick()).toMatchObject({
+      scheduled: 1,
+      failed: 1,
+    });
+    expect((await fixture.scheduler.listRuns())[0]).toMatchObject({
+      status: "retry-pending",
+      attempt: 1,
+      error: "第 1 次失败",
+    });
+
+    clock.advance(9);
+    expect((await fixture.scheduler.tick()).retried).toBe(0);
+    clock.advance(1);
+    expect(await fixture.scheduler.tick()).toMatchObject({
+      retried: 1,
+      failed: 1,
+    });
+    clock.advance(10);
+    expect((await fixture.scheduler.tick()).retried).toBe(1);
+
+    expect(executions).toBe(3);
+    expect(
+      (await fixture.scheduler.listRuns()).map(({ status, attempt }) => ({
+        status,
+        attempt,
+      })),
+    ).toEqual([
+      { status: "succeeded", attempt: 3 },
+      { status: "failed", attempt: 2 },
+      { status: "failed", attempt: 1 },
+    ]);
+  });
+
+  test("动态 Feature 停用后调度与手动触发都不执行，恢复无需重启", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    let featureEnabled = false;
+    let executions = 0;
+    const fixture = await createFixture(
+      clock,
+      () => {
+        executions += 1;
+      },
+      {
+        featureResolver: {
+          resolve: (key) => ({
+            key,
+            enabled: featureEnabled,
+            visible: "visible",
+          }),
+        },
+      },
+    );
+    clock.advance(100);
+
+    expect(await fixture.scheduler.tick()).toMatchObject({
+      scheduled: 0,
+      blocked: 0,
+    });
+    expect(await fixture.scheduler.listRuns()).toHaveLength(0);
+    expect(fixture.runtime.auditSink.list()).toHaveLength(0);
+    const deniedContext = await fixture.context();
+    expect(
+      await fixture.scheduler.triggerManual("test.task", deniedContext),
+    ).toMatchObject({
+      ok: false,
+      reason: "feature-disabled",
+    });
+    expect(executions).toBe(0);
+
+    featureEnabled = true;
+    const allowedContext = await fixture.context();
+    expect(
+      await fixture.scheduler.triggerManual("test.task", allowedContext),
+    ).toMatchObject({
+      ok: true,
+      run: { status: "succeeded" },
+    });
+    expect(executions).toBe(1);
+  });
+
+  test("自动调度持续被同一策略拒绝时只记录一次审计，恢复后再次拒绝会重新记录", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    let restricted = true;
+    const fixture = await createFixture(clock, () => undefined, {
+      licenseResolver: {
+        resolve: () => ({
+          versionKey: licenseVersionKey("standard"),
+          entitlements: [entitlementKey("capability.scheduler")],
+          status: restricted ? "expired" : "valid",
+          restricted,
+        }),
+      },
+    });
+
+    clock.advance(100);
+    expect(await fixture.scheduler.tick()).toMatchObject({ blocked: 1 });
+    expect(fixture.runtime.auditSink.list()).toEqual([
+      expect.objectContaining({ action: "task.execute", outcome: "denied" }),
+    ]);
+
+    clock.advance(100);
+    expect(await fixture.scheduler.tick()).toMatchObject({ blocked: 1 });
+    expect(fixture.runtime.auditSink.list()).toHaveLength(1);
+
+    restricted = false;
+    clock.advance(100);
+    expect(await fixture.scheduler.tick()).toMatchObject({ scheduled: 1 });
+
+    restricted = true;
+    clock.advance(100);
+    expect(await fixture.scheduler.tick()).toMatchObject({ blocked: 1 });
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(
+          ({ action, outcome }) =>
+            action === "task.execute" && outcome === "denied",
+        ),
+    ).toHaveLength(2);
+  });
+
+  test("缺少 Entitlement 时手动触发在创建执行记录前被拒绝", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    const fixture = await createFixture(clock, () => undefined, {}, false);
+    const result = await fixture.scheduler.triggerManual(
+      "test.task",
+      await fixture.context(),
+    );
+
+    expect(result).toMatchObject({
+      ok: false,
+      reason: "entitlement-not-granted",
+    });
+    expect(await fixture.scheduler.listRuns()).toHaveLength(0);
+    expect(fixture.runtime.auditSink.list()[0]?.action).toBe("security.denied");
+  });
+
+  test("License 限制模式先于 Entitlement、Feature 与 RBAC 阻断调度", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    let featureReads = 0;
+    let executions = 0;
+    const fixture = await createFixture(
+      clock,
+      () => {
+        executions += 1;
+      },
+      {
+        licenseResolver: {
+          resolve: () => ({
+            entitlements: [entitlementKey("capability.scheduler")],
+            status: "expired",
+            restricted: true,
+          }),
+        },
+        featureResolver: {
+          resolve: (key) => {
+            featureReads += 1;
+            return { key, enabled: true, visible: "visible" };
+          },
+        },
+      },
+    );
+
+    expect(
+      await fixture.scheduler.triggerManual(
+        "test.task",
+        await fixture.context(),
+      ),
+    ).toMatchObject({ ok: false, reason: "license-restricted" });
+    expect(await fixture.scheduler.listRuns()).toHaveLength(0);
+
+    clock.advance(100);
+    expect(await fixture.scheduler.tick()).toMatchObject({
+      scheduled: 1,
+      blocked: 1,
+    });
+    expect(await fixture.scheduler.listRuns()).toEqual([
+      expect.objectContaining({
+        status: "blocked",
+        error: "license-restricted",
+      }),
+    ]);
+    expect(featureReads).toBe(0);
+    expect(executions).toBe(0);
+  });
+
+  test("后台 License 解析异常会形成可重试失败而不是遗留 running", async () => {
+    const clock = new FakeClock(new Date("2026-07-10T00:00:00.000Z"));
+    const fixture = await createFixture(clock, () => undefined, {
+      licenseResolver: {
+        resolve() {
+          throw new Error("License 存储暂不可用");
+        },
+      },
+    });
+    clock.advance(100);
+
+    expect(await fixture.scheduler.tick()).toMatchObject({
+      scheduled: 1,
+      failed: 1,
+    });
+    expect(await fixture.scheduler.listRuns()).toEqual([
+      expect.objectContaining({
+        status: "retry-pending",
+        error: "License 存储暂不可用",
+      }),
+    ]);
+  });
+
+});
+
+async function createFixture(
+  clock: FakeClock,
+  handler: ScheduledTaskDefinition["handler"],
+  runtimeOptions: Parameters<typeof createServerRuntime>[1] = {},
+  licensed = true,
+) {
+  const feature = createFoundationSnapshot().features.find(
+    ({ key }) => key === "system.scheduler",
+  );
+  if (!feature) throw new Error("测试缺少 system.scheduler Feature");
+  const runtime = createServerRuntime(
+    {
+      WINDY_DEV_ADMIN_TOKEN: "token",
+      WINDY_LICENSE_ENTITLEMENTS: licensed ? "capability.scheduler" : undefined,
+    },
+    runtimeOptions,
+  );
+  const repository = new InMemorySchedulerRepository();
+  const scheduler = new SchedulerService(
+    repository,
+    [
+      {
+        key: "test.task",
+        label: "测试任务",
+        intervalSeconds: 100,
+        maxAttempts: 3,
+        retryDelaySeconds: 10,
+        permissionKey: "system.scheduler.manage",
+        feature: {
+          ...feature,
+          requiredEntitlement: entitlementKey("capability.scheduler"),
+        },
+        handler,
+      },
+    ],
+    runtime,
+    clock,
+  );
+  await scheduler.initialize();
+  return {
+    runtime,
+    repository,
+    scheduler,
+    context: () =>
+      runtime.createGuardContext(
+        new Request(
+          "http://localhost/api/system/scheduled-tasks/test.task/trigger",
+          {
+            headers: { authorization: "Bearer token" },
+          },
+        ),
+      ),
+  };
+}
+
+async function until(predicate: () => boolean) {
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    if (predicate()) return;
+    await Bun.sleep(1);
+  }
+  throw new Error("测试等待条件超时");
+}

package/template/apps/server/src/scheduler/service.ts

@@ -0,0 +1,339 @@
+import { randomUUIDv7 } from "bun";
+import type { SchedulerOperationsSnapshot } from "@windy/shared";
+import { runGuardedBackgroundTask } from "../background/task-guard.js";
+import type { RequestGuardContext } from "../guards.js";
+import type { LegacyDurableAdapter } from "../jobs/legacy-adapter.js";
+import type { createServerRuntime } from "../runtime.js";
+import type { SchedulerRepository } from "./repository.js";
+import { validateTaskDefinitions } from "./definitions.js";
+import { SchedulerExecutionRecorder } from "./execution-recorder.js";
+import {
+  schedulerActor,
+  schedulerPolicy,
+  type ManualTriggerResult,
+} from "./execution-context.js";
+import { isScheduledFeatureActive, isTaskDue } from "./feature-preflight.js";
+import { listSchedulerRunHistory } from "./history.js";
+import { buildSchedulerOperationsSnapshot } from "./overview.js";
+import {
+  recoverStaleScheduledRun,
+  type StaleRecoveryServiceResult,
+} from "./recovery-service.js";
+import {
+  SCHEDULER_STALE_THRESHOLD_SECONDS,
+  systemClock,
+  type ScheduledTaskDefinition,
+  type ScheduledTaskRun,
+  type SchedulerClock,
+  type SchedulerTickResult,
+  type StoredScheduledTask,
+} from "./types.js";
+
+type ServerRuntime = ReturnType<typeof createServerRuntime>;
+
+export class SchedulerService {
+  private readonly definitions = new Map<string, ScheduledTaskDefinition>();
+  private readonly recorder: SchedulerExecutionRecorder;
+
+  constructor(
+    private readonly repository: SchedulerRepository,
+    definitions: ScheduledTaskDefinition[],
+    private readonly runtime: ServerRuntime,
+    private readonly clock: SchedulerClock = systemClock,
+    private readonly durable?: LegacyDurableAdapter,
+  ) {
+    this.recorder = new SchedulerExecutionRecorder(repository, runtime, clock);
+    validateTaskDefinitions(definitions);
+    definitions.forEach((definition) =>
+      this.definitions.set(definition.key, definition),
+    );
+  }
+
+  async initialize() {
+    await this.repository.syncDefinitions(
+      [...this.definitions.values()],
+      this.clock.now(),
+    );
+  }
+
+  listTasks(): Promise<StoredScheduledTask[]> {
+    return this.repository.listTasks();
+  }
+
+  listRuns(limit?: number): Promise<ScheduledTaskRun[]> {
+    return this.repository.listRuns(limit);
+  }
+
+  async listRunHistory(page: number, pageSize: number) {
+    return listSchedulerRunHistory(this.repository, page, pageSize);
+  }
+
+  async operationsSnapshot(
+    staleThresholdSeconds = SCHEDULER_STALE_THRESHOLD_SECONDS,
+    runLimit = 50,
+  ): Promise<SchedulerOperationsSnapshot> {
+    const [tasks, runs] = await Promise.all([
+      this.repository.listTasks(),
+      this.repository.listRuns(runLimit),
+    ]);
+    return buildSchedulerOperationsSnapshot(
+      tasks,
+      runs,
+      new Set(this.definitions.keys()),
+      this.clock.now(),
+      staleThresholdSeconds,
+    );
+  }
+
+  async recoverStaleRun(
+    taskKey: string,
+    runId: string,
+    expectedStartedAt: string,
+    rawReason: string,
+    context: RequestGuardContext,
+  ): Promise<StaleRecoveryServiceResult> {
+    return recoverStaleScheduledRun({
+      repository: this.repository,
+      runtime: this.runtime,
+      clock: this.clock,
+      definition: this.definitions.get(taskKey),
+      taskKey,
+      runId,
+      expectedStartedAt,
+      rawReason,
+      context,
+    });
+  }
+
+  async triggerManual(
+    taskKey: string,
+    context: RequestGuardContext,
+  ): Promise<ManualTriggerResult> {
+    const definition = this.definitions.get(taskKey);
+    if (!definition) return { ok: false, reason: "not-found" };
+    let auditUnavailable = false;
+    const guarded = await runGuardedBackgroundTask(
+      this.runtime,
+      context,
+      schedulerPolicy(definition),
+      async () => {
+        const run = await this.repository.claimManual(
+          taskKey,
+          context.actor.id,
+          this.clock.now(),
+        );
+        if (!run) return undefined;
+        try {
+          await this.recorder.trigger(context, run);
+        } catch {
+          auditUnavailable = true;
+          await this.repository.complete(run.id, {
+            status: "blocked",
+            finishedAt: this.clock.now(),
+            error: "关键审计不可用，手动执行已阻断",
+          });
+          return undefined;
+        }
+        return this.execute(run, definition, context, false);
+      },
+    );
+    if (!guarded.ok) return { ok: false, reason: guarded.reason };
+    if (auditUnavailable) return { ok: false, reason: "audit-unavailable" };
+    const run = await guarded.value;
+    return run ? { ok: true, run } : { ok: false, reason: "busy" };
+  }
+
+  async tick(): Promise<SchedulerTickResult> {
+    const result: SchedulerTickResult = {
+      scheduled: 0,
+      retried: 0,
+      blocked: 0,
+      failed: 0,
+    };
+    await this.resumeDurableWork(result);
+    const now = this.clock.now();
+    for (const pending of await this.repository.listRetryable(now)) {
+      const definition = this.definitions.get(pending.taskKey);
+      if (
+        definition &&
+        !(await isScheduledFeatureActive(this.runtime, definition))
+      )
+        continue;
+      const run = await this.repository.claimRetry(pending.id, now);
+      if (!run) continue;
+      result.retried += 1;
+      this.recorder.count(await this.executeScheduled(run), result);
+    }
+    for (const task of await this.repository.listTasks()) {
+      if (!isTaskDue(task, now)) continue;
+      const definition = this.definitions.get(task.key);
+      if (
+        definition &&
+        !(await isScheduledFeatureActive(this.runtime, definition))
+      )
+        continue;
+      const run = await this.repository.claimScheduled(task.key, now);
+      if (!run) continue;
+      result.scheduled += 1;
+      this.recorder.count(await this.executeScheduled(run), result);
+    }
+    return result;
+  }
+
+  async resumeDurable(runId: string, generation: number) {
+    const run = await this.repository.getRun(runId);
+    if (!run || run.status !== "running" || run.attempt !== generation) return;
+    const definition = this.definitions.get(run.taskKey);
+    if (!definition) {
+      await this.repository.complete(run.id, {
+        status: "failed",
+        finishedAt: this.clock.now(),
+        error: "任务定义已不存在",
+      });
+      return;
+    }
+    if (run.trigger !== "manual") {
+      await this.executeScheduled(run, false);
+      return;
+    }
+    await this.execute(
+      run,
+      definition,
+      manualResumeContext(run, definition),
+      false,
+      false,
+    );
+  }
+
+  private async executeScheduled(run: ScheduledTaskRun, useDurable = true) {
+    const definition = this.definitions.get(run.taskKey);
+    if (!definition) {
+      return this.repository.complete(run.id, {
+        status: "failed",
+        finishedAt: this.clock.now(),
+        error: "任务定义已不存在",
+      });
+    }
+    const actor = schedulerActor(definition.permissionKey);
+    const fallbackContext: RequestGuardContext = {
+      requestId: randomUUIDv7(),
+      actor,
+      startedAt: this.clock.now().toISOString(),
+    };
+    try {
+      const context = await this.runtime.createGuardContextForActor(
+        new Request(
+          `http://scheduler.local/tasks/${encodeURIComponent(run.taskKey)}`,
+        ),
+        actor,
+      );
+      return this.execute(run, definition, context, true, useDurable);
+    } catch (error) {
+      return this.recorder.failure(run, definition, fallbackContext, error);
+    }
+  }
+
+  private async execute(
+    run: ScheduledTaskRun,
+    definition: ScheduledTaskDefinition,
+    context: RequestGuardContext,
+    applyGuard: boolean,
+    useDurable = true,
+  ) {
+    if (this.durable && useDurable) {
+      return this.dispatchDurable(run, definition);
+    }
+    if (applyGuard) {
+      let guarded;
+      try {
+        guarded = await runGuardedBackgroundTask(
+          this.runtime,
+          context,
+          schedulerPolicy(definition),
+          definition.handler,
+          { recordDecision: false },
+        );
+      } catch (error) {
+        return this.recorder.failure(run, definition, context, error);
+      }
+      if (!guarded.ok) {
+        const blocked = await this.repository.complete(run.id, {
+          status: "blocked",
+          finishedAt: this.clock.now(),
+          error: guarded.reason,
+        });
+        this.recorder.denied(context, blocked, guarded.reason);
+        return blocked;
+      }
+      return this.recorder.success(run, definition, context, guarded.value);
+    }
+
+    try {
+      return this.recorder.success(
+        run,
+        definition,
+        context,
+        await definition.handler(),
+      );
+    } catch (error) {
+      return this.recorder.failure(run, definition, context, error);
+    }
+  }
+
+  private async dispatchDurable(
+    run: ScheduledTaskRun,
+    definition: ScheduledTaskDefinition,
+  ) {
+    await this.durable!.dispatch({
+      legacyId: run.id,
+      generation: run.attempt,
+      requestedBy: run.requestedBy,
+      maxAttempts: definition.maxAttempts,
+      timeoutSeconds: Math.max(1, definition.intervalSeconds),
+      link: (durableJobId) =>
+        this.repository.linkDurableJob(run.id, run.attempt, durableJobId),
+    });
+    return (
+      (await this.repository.getRun(run.id)) || {
+        ...run,
+        status: "succeeded" as const,
+        finishedAt: this.clock.now().toISOString(),
+      }
+    );
+  }
+
+  private async resumeDurableWork(result: SchedulerTickResult) {
+    if (!this.durable) return;
+    const resumed = await this.durable.runNext();
+    if (resumed.status !== "idle" && resumed.legacyId) {
+      const run = await this.repository.getRun(resumed.legacyId);
+      if (run) this.recorder.count(run, result);
+      return;
+    }
+    const [orphan] = await this.repository.listUnlinkedRunning(1);
+    const definition = orphan && this.definitions.get(orphan.taskKey);
+    if (orphan && definition) {
+      this.recorder.count(
+        await this.dispatchDurable(orphan, definition),
+        result,
+      );
+    }
+  }
+}
+
+function manualResumeContext(
+  run: ScheduledTaskRun,
+  definition: ScheduledTaskDefinition,
+): RequestGuardContext {
+  return {
+    requestId: randomUUIDv7(),
+    actor: {
+      id: run.requestedBy,
+      type: "user",
+      name: run.requestedBy,
+      roleCodes: [],
+      permissionKeys: [definition.permissionKey],
+    },
+    startedAt: new Date().toISOString(),
+  };
+}