create-windy 0.1.0

package/template/apps/server/src/jobs/legacy-adapter.integration.test.ts

@@ -0,0 +1,177 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { resolve } from "node:path";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { migrate } from "drizzle-orm/node-postgres/migrator";
+import { Pool } from "pg";
+import { DrizzleBulkDataRepository } from "../bulk-data/drizzle-repository.js";
+import type { BulkResourceHandler } from "../bulk-data/resource-handler.js";
+import { BulkDataService } from "../bulk-data/service.js";
+import { DrizzleDurableExecutionRepository } from "./drizzle-execution-repository.js";
+import { DrizzleDurableJobRepository } from "./drizzle-repository.js";
+import { DurableHandlerRegistry } from "./handler-registry.js";
+import {
+  DeferredLegacyExecutor,
+  LegacyDurableAdapter,
+} from "./legacy-adapter.js";
+import { DurableJobService } from "./service.js";
+import { DurableJobWorker } from "./worker.js";
+
+const databaseUrl = process.env.JOBS_TEST_DATABASE_URL;
+const describePostgres = databaseUrl ? describe : describe.skip;
+
+describePostgres("PostgreSQL legacy durable adapter", () => {
+  let pool: Pool;
+
+  beforeAll(async () => {
+    pool = new Pool({ connectionString: databaseUrl });
+    await migrate(drizzle(pool), {
+      migrationsFolder: resolve(
+        import.meta.dir,
+        "../../../../packages/database/drizzle",
+      ),
+      migrationsSchema: "public",
+      migrationsTable: "__jobs_adapter_test_migrations",
+    });
+    await pool.query(`
+      truncate table
+        governed_export_requests,
+        bulk_data_artifacts,
+        bulk_data_jobs,
+        scheduled_task_runs,
+        durable_job_recoveries,
+        durable_job_failures,
+        durable_job_artifacts,
+        durable_job_progress,
+        durable_job_leases,
+        durable_job_attempts,
+        durable_job_stages,
+        durable_jobs
+    `);
+  });
+
+  afterAll(async () => {
+    await pool?.end();
+  });
+
+  test("重建服务回收 running Bulk，重复 dispatch 不产生额外 attempt", async () => {
+    let now = new Date("2026-07-16T00:00:00.000Z");
+    const db = drizzle(pool);
+    const legacy = new DrizzleBulkDataRepository(db);
+    const original = new BulkDataService(
+      legacy,
+      [fixtureHandler()],
+      {},
+      () => now,
+    );
+    const created = await original.createExport(
+      "fixtures",
+      "pg:durable:bulk:restart",
+      actor("user_pg_adapter"),
+    );
+    const running = await legacy.claimNext(now);
+    if (!running) throw new Error("应创建 running Bulk 任务");
+
+    const facts = new DrizzleDurableJobRepository(db);
+    const durable = new DurableJobService(facts, () => now);
+    const submitted = await durable.submit({
+      handlerKey: "compat.bulk-data.pg",
+      queue: "compat.bulk-data.pg",
+      idempotencyKey: `${running.id}:${running.attempt}`,
+      payload: { legacyId: running.id, generation: running.attempt },
+      requestedBy: running.requestedBy,
+      maxAttempts: 3,
+    });
+    await legacy.linkDurableJob(running.id, running.attempt, submitted.job.id);
+    const crashedExecution = new DrizzleDurableExecutionRepository(db);
+    expect(
+      await crashedExecution.claim({
+        queue: "compat.bulk-data.pg",
+        workerId: "crashed-pg-worker",
+        now,
+        leaseDurationMs: 1_000,
+      }),
+    ).toMatchObject({ jobId: submitted.job.id });
+
+    now = new Date(now.getTime() + 2_000);
+    const restartedDb = drizzle(pool);
+    const restartedLegacy = new DrizzleBulkDataRepository(restartedDb);
+    const restartedFacts = new DrizzleDurableJobRepository(restartedDb);
+    const restartedDurable = new DurableJobService(restartedFacts, () => now);
+    const registry = new DurableHandlerRegistry();
+    const worker = new DurableJobWorker(
+      new DrizzleDurableExecutionRepository(restartedDb),
+      registry,
+      { now: () => now },
+    );
+    const executor = new DeferredLegacyExecutor();
+    const adapter = new LegacyDurableAdapter(
+      restartedDurable,
+      worker,
+      registry,
+      {
+        handlerKey: "compat.bulk-data.pg",
+        queue: "compat.bulk-data.pg",
+        workerId: "restarted-pg-worker",
+        execute: (payload) => executor.execute(payload),
+      },
+    );
+    const restarted = new BulkDataService(
+      restartedLegacy,
+      [fixtureHandler()],
+      {},
+      () => now,
+      undefined,
+      adapter,
+    );
+    executor.attach(({ legacyId, generation }) =>
+      restarted.resumeDurable(legacyId, generation),
+    );
+
+    expect(await restarted.processNext()).toEqual({ claimed: false });
+    now = new Date(now.getTime() + 1_000);
+    expect(await restarted.processNext()).toMatchObject({
+      claimed: true,
+      jobId: created.job.id,
+      processedRows: 1,
+    });
+    expect(await restartedLegacy.getJob(created.job.id)).toMatchObject({
+      status: "succeeded",
+      attempt: 1,
+      durableJobId: submitted.job.id,
+    });
+    expect(
+      (await restartedDurable.get(submitted.job.id))?.attempts,
+    ).toMatchObject([{ status: "lease-expired" }, { status: "succeeded" }]);
+
+    await adapter.dispatch({
+      legacyId: running.id,
+      generation: running.attempt,
+      requestedBy: running.requestedBy,
+      maxAttempts: 3,
+      link: (jobId) =>
+        restartedLegacy.linkDurableJob(running.id, running.attempt, jobId),
+    });
+    expect(
+      (await restartedDurable.get(submitted.job.id))?.attempts,
+    ).toHaveLength(2);
+  });
+});
+
+function fixtureHandler(): BulkResourceHandler {
+  return {
+    scopeMode: "platform",
+    resource: "fixtures",
+    importRow: async () => undefined,
+    exportRows: async () => [{ id: "pg-row" }],
+  };
+}
+
+function actor(id: string) {
+  return {
+    id,
+    type: "user" as const,
+    name: id,
+    roleCodes: [],
+    permissionKeys: [],
+  };
+}

package/template/apps/server/src/jobs/legacy-adapter.test.ts

@@ -0,0 +1,199 @@
+import { describe, expect, test } from "bun:test";
+import { createFoundationSnapshot } from "../foundation.js";
+import { createServerRuntime } from "../runtime.js";
+import { InMemorySchedulerRepository } from "../scheduler/repository.js";
+import { SchedulerService } from "../scheduler/service.js";
+import type { SchedulerClock } from "../scheduler/types.js";
+import { InMemoryBulkDataRepository } from "../bulk-data/repository.js";
+import { BulkDataService } from "../bulk-data/service.js";
+import type { BulkResourceHandler } from "../bulk-data/resource-handler.js";
+import { DurableHandlerRegistry } from "./handler-registry.js";
+import { InMemoryDurableExecutionRepository } from "./in-memory-execution.js";
+import {
+  DeferredLegacyExecutor,
+  LegacyDurableAdapter,
+} from "./legacy-adapter.js";
+import { DurableJobService } from "./service.js";
+import { DurableJobWorker } from "./worker.js";
+
+describe("legacy durable adapters", () => {
+  test("running Bulk 在进程重启后回收 lease，重复启动与完成不重复执行", async () => {
+    let now = new Date("2026-07-16T00:00:00.000Z");
+    const legacy = new InMemoryBulkDataRepository();
+    const original = new BulkDataService(
+      legacy,
+      [testBulkHandler()],
+      {},
+      () => now,
+    );
+    const created = await original.createExport(
+      "fixtures",
+      "durable:bulk:restart",
+      actor("user_bulk"),
+    );
+    const running = await legacy.claimNext(now);
+    if (!running) throw new Error("应创建 running Bulk 任务");
+
+    const facts = new InMemoryDurableExecutionRepository();
+    const durable = new DurableJobService(facts, () => now);
+    const submitted = await durable.submit({
+      handlerKey: "compat.bulk-data",
+      queue: "compat.bulk-data",
+      idempotencyKey: `${running.id}:${running.attempt}`,
+      payload: { legacyId: running.id, generation: running.attempt },
+      requestedBy: running.requestedBy,
+      maxAttempts: 3,
+    });
+    await legacy.linkDurableJob(running.id, running.attempt, submitted.job.id);
+    const crashed = await facts.claim({
+      queue: "compat.bulk-data",
+      workerId: "crashed-worker",
+      now,
+      leaseDurationMs: 1_000,
+    });
+    expect(crashed?.jobId).toBe(submitted.job.id);
+
+    now = new Date(now.getTime() + 2_000);
+    const registry = new DurableHandlerRegistry();
+    const worker = new DurableJobWorker(facts, registry, { now: () => now });
+    const executor = new DeferredLegacyExecutor();
+    const adapter = new LegacyDurableAdapter(durable, worker, registry, {
+      handlerKey: "compat.bulk-data",
+      queue: "compat.bulk-data",
+      workerId: "restarted-worker",
+      execute: (payload) => executor.execute(payload),
+    });
+    const restarted = new BulkDataService(
+      legacy,
+      [testBulkHandler()],
+      {},
+      () => now,
+      undefined,
+      adapter,
+    );
+    executor.attach(({ legacyId, generation }) =>
+      restarted.resumeDurable(legacyId, generation),
+    );
+
+    expect(await restarted.processNext()).toEqual({ claimed: false });
+    now = new Date(now.getTime() + 1_000);
+    expect(await restarted.processNext()).toMatchObject({
+      claimed: true,
+      jobId: created.job.id,
+      processedRows: 1,
+    });
+    expect(await legacy.getJob(created.job.id)).toMatchObject({
+      status: "succeeded",
+      attempt: 1,
+      durableJobId: submitted.job.id,
+    });
+    expect((await durable.get(submitted.job.id))?.attempts).toMatchObject([
+      { status: "lease-expired" },
+      { status: "succeeded" },
+    ]);
+
+    await adapter.dispatch({
+      legacyId: running.id,
+      generation: running.attempt,
+      requestedBy: running.requestedBy,
+      maxAttempts: 3,
+      link: (jobId) =>
+        legacy.linkDurableJob(running.id, running.attempt, jobId),
+    });
+    await restarted.resumeDurable(running.id, running.attempt);
+    expect((await durable.get(submitted.job.id))?.attempts).toHaveLength(2);
+    expect(await legacy.getJob(created.job.id)).toMatchObject({
+      status: "succeeded",
+      processedRows: 1,
+    });
+  });
+
+  test("Scheduler 采用未关联 running 记录且重放已完成记录为空操作", async () => {
+    const now = new Date("2026-07-16T00:00:00.000Z");
+    const clock: SchedulerClock = { now: () => new Date(now) };
+    const feature = createFoundationSnapshot().features.find(
+      ({ key }) => key === "system.scheduler",
+    );
+    if (!feature) throw new Error("测试缺少 Scheduler Feature");
+    const legacy = new InMemorySchedulerRepository();
+    const facts = new InMemoryDurableExecutionRepository();
+    const durable = new DurableJobService(facts, clock.now);
+    const registry = new DurableHandlerRegistry();
+    const worker = new DurableJobWorker(facts, registry, { now: clock.now });
+    let executions = 0;
+    const executor = new DeferredLegacyExecutor();
+    const adapter = new LegacyDurableAdapter(durable, worker, registry, {
+      handlerKey: "compat.scheduler",
+      queue: "compat.scheduler",
+      workerId: "scheduler-worker",
+      execute: (payload) => executor.execute(payload),
+    });
+    const scheduler = new SchedulerService(
+      legacy,
+      [
+        {
+          key: "test.durable",
+          label: "持久执行测试",
+          intervalSeconds: 3_600,
+          maxAttempts: 3,
+          retryDelaySeconds: 10,
+          permissionKey: "system.scheduler.manage",
+          feature,
+          handler: () => {
+            executions += 1;
+            return { executions };
+          },
+        },
+      ],
+      createServerRuntime(),
+      clock,
+      adapter,
+    );
+    executor.attach(({ legacyId, generation }) =>
+      scheduler.resumeDurable(legacyId, generation),
+    );
+    await scheduler.initialize();
+    const run = await legacy.claimManual("test.durable", "user_scheduler", now);
+    if (!run) throw new Error("应创建未关联的 Scheduler run");
+
+    await scheduler.tick();
+    const completed = await legacy.getRun(run.id);
+    expect(completed).toMatchObject({
+      status: "succeeded",
+    });
+    expect(typeof completed?.durableJobId).toBe("string");
+    if (!completed?.durableJobId) throw new Error("应关联 durable job");
+    expect(executions).toBe(1);
+    await scheduler.resumeDurable(run.id, run.attempt);
+    await adapter.dispatch({
+      legacyId: run.id,
+      generation: run.attempt,
+      requestedBy: run.requestedBy,
+      maxAttempts: 3,
+      link: (jobId) => legacy.linkDurableJob(run.id, run.attempt, jobId),
+    });
+    expect(executions).toBe(1);
+    const [durableJob] = await durable.list(run.requestedBy);
+    expect(durableJob?.id).toBe(completed.durableJobId);
+    expect((await durable.get(durableJob!.id))?.attempts).toHaveLength(1);
+  });
+});
+
+function testBulkHandler(): BulkResourceHandler {
+  return {
+    scopeMode: "platform",
+    resource: "fixtures",
+    importRow: async () => undefined,
+    exportRows: async () => [{ id: "row-1" }],
+  };
+}
+
+function actor(id: string) {
+  return {
+    id,
+    type: "user" as const,
+    name: id,
+    roleCodes: [],
+    permissionKeys: [],
+  };
+}

package/template/apps/server/src/jobs/legacy-adapter.ts

@@ -0,0 +1,109 @@
+import type { DurableRunResult } from "./worker.js";
+import type { DurableHandlerRegistry } from "./handler-registry.js";
+import type { DurableJobService } from "./service.js";
+import type { DurableJobWorker } from "./worker.js";
+
+export interface LegacyExecutionPayload {
+  legacyId: string;
+  generation: number;
+}
+
+export interface LegacyDispatchInput extends LegacyExecutionPayload {
+  requestedBy: string;
+  maxAttempts: number;
+  timeoutSeconds?: number;
+  link(durableJobId: string): Promise<boolean>;
+}
+
+export type LegacyDurableRunResult = DurableRunResult & {
+  legacyId?: string;
+  generation?: number;
+};
+
+export interface LegacyDurableAdapterOptions {
+  handlerKey: string;
+  queue: string;
+  workerId: string;
+  execute(payload: LegacyExecutionPayload): Promise<void>;
+}
+
+export class DeferredLegacyExecutor {
+  private target?: (payload: LegacyExecutionPayload) => Promise<void>;
+
+  attach(target: (payload: LegacyExecutionPayload) => Promise<void>) {
+    if (this.target) throw new Error("兼容任务 executor 已绑定");
+    this.target = target;
+  }
+
+  execute(payload: LegacyExecutionPayload) {
+    if (!this.target) throw new Error("兼容任务 executor 尚未绑定");
+    return this.target(payload);
+  }
+}
+
+/**
+ * Bridges an existing persisted execution row to the durable lease runtime.
+ * The handler only stores legacy identity, so replay after restart never relies
+ * on an in-memory callback captured by the process that submitted the job.
+ */
+export class LegacyDurableAdapter {
+  constructor(
+    private readonly service: DurableJobService,
+    private readonly worker: DurableJobWorker,
+    registry: DurableHandlerRegistry,
+    private readonly options: LegacyDurableAdapterOptions,
+  ) {
+    registry.register({
+      key: options.handlerKey,
+      payload: { parse: parseLegacyPayload },
+      execute: options.execute,
+    });
+  }
+
+  async dispatch(input: LegacyDispatchInput) {
+    const submitted = await this.service.submit({
+      handlerKey: this.options.handlerKey,
+      queue: this.options.queue,
+      idempotencyKey: `${input.legacyId}:${input.generation}`,
+      payload: {
+        legacyId: input.legacyId,
+        generation: input.generation,
+      },
+      requestedBy: input.requestedBy,
+      maxAttempts: Math.max(2, input.maxAttempts),
+      timeoutSeconds: input.timeoutSeconds,
+    });
+    await input.link(submitted.job.id);
+    return this.runNext();
+  }
+
+  async runNext(): Promise<LegacyDurableRunResult> {
+    const result = await this.worker.runNext(
+      this.options.queue,
+      this.options.workerId,
+    );
+    if (result.status === "idle") return result;
+    const snapshot = await this.service.get(result.jobId);
+    const payload = snapshot
+      ? parseLegacyPayload(snapshot.job.payload)
+      : undefined;
+    return { ...result, ...payload };
+  }
+}
+
+function parseLegacyPayload(input: unknown): LegacyExecutionPayload {
+  if (!input || typeof input !== "object") {
+    throw new Error("兼容任务 payload 必须是对象");
+  }
+  const value = input as Record<string, unknown>;
+  if (typeof value.legacyId !== "string" || !value.legacyId.trim()) {
+    throw new Error("兼容任务 legacyId 不能为空");
+  }
+  if (!Number.isSafeInteger(value.generation) || Number(value.generation) < 0) {
+    throw new Error("兼容任务 generation 必须是非负整数");
+  }
+  return {
+    legacyId: value.legacyId,
+    generation: Number(value.generation),
+  };
+}

package/template/apps/server/src/jobs/legacy-service-runtime.ts

@@ -0,0 +1,95 @@
+import type { BlobStore } from "@windy/storage";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { BulkDataService } from "../bulk-data/service.js";
+import type { BulkDataLimits } from "../bulk-data/input-validation.js";
+import type { BulkDataRepository } from "../bulk-data/repository.js";
+import type { BulkResourceHandler } from "../bulk-data/resource-handler.js";
+import type { createServerRuntime } from "../runtime.js";
+import type { SchedulerRepository } from "../scheduler/repository.js";
+import { SchedulerService } from "../scheduler/service.js";
+import type { ScheduledTaskDefinition } from "../scheduler/types.js";
+import { DrizzleDurableExecutionRepository } from "./drizzle-execution-repository.js";
+import { DrizzleDurableJobRepository } from "./drizzle-repository.js";
+import { DurableHandlerRegistry } from "./handler-registry.js";
+import { InMemoryDurableExecutionRepository } from "./in-memory-execution.js";
+import {
+  DeferredLegacyExecutor,
+  LegacyDurableAdapter,
+} from "./legacy-adapter.js";
+import { DurableJobService } from "./service.js";
+import { DurableJobWorker } from "./worker.js";
+import type { WorkflowService } from "../workflow/service.js";
+import { WorkflowTimeoutRuntime } from "../workflow/timeout-runtime.js";
+
+interface LegacyServiceRuntimeInput {
+  db?: NodePgDatabase;
+  bulkRepository: BulkDataRepository;
+  bulkHandlers: BulkResourceHandler[];
+  bulkLimits: Partial<BulkDataLimits>;
+  blobStore: BlobStore;
+  schedulerRepository: SchedulerRepository;
+  taskDefinitions: ScheduledTaskDefinition[];
+  runtime: ReturnType<typeof createServerRuntime>;
+  workflow: WorkflowService;
+}
+
+export function createLegacyServiceRuntime(input: LegacyServiceRuntimeInput) {
+  const inMemory = new InMemoryDurableExecutionRepository();
+  const facts = input.db ? new DrizzleDurableJobRepository(input.db) : inMemory;
+  const execution = input.db
+    ? new DrizzleDurableExecutionRepository(input.db)
+    : inMemory;
+  const service = new DurableJobService(facts);
+  const registry = new DurableHandlerRegistry();
+  const worker = new DurableJobWorker(execution, registry);
+  const workflowTimeout = new WorkflowTimeoutRuntime(
+    service,
+    worker,
+    registry,
+    input.workflow,
+  );
+
+  const bulkExecutor = new DeferredLegacyExecutor();
+  const bulkDataService = new BulkDataService(
+    input.bulkRepository,
+    input.bulkHandlers,
+    input.bulkLimits,
+    () => new Date(),
+    input.blobStore,
+    new LegacyDurableAdapter(service, worker, registry, {
+      handlerKey: "compat.bulk-data",
+      queue: "compat.bulk-data",
+      workerId: `server-${process.pid}-bulk`,
+      execute: (payload) => bulkExecutor.execute(payload),
+    }),
+  );
+  bulkExecutor.attach(({ legacyId, generation }) =>
+    bulkDataService.resumeDurable(legacyId, generation),
+  );
+
+  const schedulerExecutor = new DeferredLegacyExecutor();
+  const taskDefinitions = input.taskDefinitions;
+  const scheduler = new SchedulerService(
+    input.schedulerRepository,
+    taskDefinitions,
+    input.runtime,
+    undefined,
+    new LegacyDurableAdapter(service, worker, registry, {
+      handlerKey: "compat.scheduler",
+      queue: "compat.scheduler",
+      workerId: `server-${process.pid}-scheduler`,
+      execute: (payload) => schedulerExecutor.execute(payload),
+    }),
+  );
+  schedulerExecutor.attach(({ legacyId, generation }) =>
+    scheduler.resumeDurable(legacyId, generation),
+  );
+
+  return {
+    bulkDataService,
+    scheduler,
+    taskDefinitions,
+    workflowTimeout,
+    durable: { facts, service, registry, worker },
+  };
+}