create-windy 0.1.0

package/template/apps/server/src/bulk-data/service.test.ts

@@ -0,0 +1,347 @@
+import { describe, expect, test } from "bun:test";
+import type { RequestActor } from "../guards.js";
+import { MemoryBlobStore } from "@windy/storage";
+import { InMemoryEntityRepository } from "../system/repository.js";
+import type { ManagedDictionary } from "../system/entities.js";
+import { DictionaryBulkResourceHandler } from "./dictionary-handler.js";
+import { InMemoryBulkDataRepository } from "./repository.js";
+import { BulkDataInputError, BulkDataService } from "./service.js";
+import type { BulkResourceHandler } from "./resource-handler.js";
+import { isServerScopeContext } from "../data-access/context.js";
+
+const actor: RequestActor = {
+  id: "user_1",
+  type: "user",
+  name: "测试用户",
+  roleCodes: ["developer"],
+  permissionKeys: ["system.dict.read", "system.dict.manage"],
+};
+
+describe("BulkDataService", () => {
+  test("字典 NDJSON 导入保留行级失败文件且重试输入幂等", async () => {
+    const clock = new FakeClock("2026-07-10T00:00:00.000Z");
+    const repository = new InMemoryBulkDataRepository();
+    const dictionaries = new InMemoryEntityRepository<ManagedDictionary>();
+    const service = new BulkDataService(
+      repository,
+      [new DictionaryBulkResourceHandler(dictionaries)],
+      {},
+      clock.now,
+    );
+    const content = [
+      JSON.stringify({ id: "dict_1", code: "color", name: "颜色", items: [] }),
+      "{broken",
+      JSON.stringify({ id: "bad id", code: "bad", name: "错误", items: [] }),
+    ].join("\n");
+
+    const first = await service.createImport(
+      "dictionaries",
+      content,
+      "upsert",
+      "import:20260710:001",
+      actor,
+    );
+    const duplicate = await service.createImport(
+      "dictionaries",
+      content,
+      "upsert",
+      "import:20260710:001",
+      actor,
+    );
+    expect(first.created).toBe(true);
+    expect(duplicate.created).toBe(false);
+
+    expect(await service.processNext()).toMatchObject({
+      claimed: true,
+      succeededRows: 1,
+      failedRows: 2,
+    });
+    expect(await service.getOwnedJob(first.job.id, actor.id)).toMatchObject({
+      status: "completed-with-errors",
+      processedRows: 3,
+    });
+    expect((await dictionaries.get("dict_1"))?.name).toBe("颜色");
+
+    const failures = await service.getOwnedArtifact(
+      first.job.id,
+      "failures",
+      actor.id,
+    );
+    const failureContent = failures?.content;
+    expect(failureContent).toBeDefined();
+    if (!failureContent) throw new Error("缺少失败文件内容");
+    const rows = failureContent
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line));
+    expect(rows).toEqual([
+      expect.objectContaining({ line: 2, code: "INVALID_JSON" }),
+      expect.objectContaining({ line: 3, code: "INVALID_ROW" }),
+    ]);
+    expect(failureContent).not.toContain("{broken");
+    expect(rows?.[0].inputDigest).toHaveLength(64);
+  });
+
+  test("幂等键按操作者、类型和资源隔离，复用不同输入返回冲突", async () => {
+    const repository = new InMemoryBulkDataRepository();
+    const service = fixtureService(repository);
+    const content = validLine("dict_a");
+    await service.createImport(
+      "dictionaries",
+      content,
+      "create",
+      "scope:key:001",
+      actor,
+    );
+
+    await expect(
+      service.createImport(
+        "dictionaries",
+        validLine("dict_b"),
+        "create",
+        "scope:key:001",
+        actor,
+      ),
+    ).rejects.toMatchObject({ code: "IDEMPOTENCY_CONFLICT" });
+    expect(
+      (
+        await service.createImport(
+          "dictionaries",
+          validLine("dict_b"),
+          "create",
+          "scope:key:001",
+          { ...actor, id: "user_2" },
+        )
+      ).created,
+    ).toBe(true);
+    expect(
+      (await service.createExport("dictionaries", "scope:key:001", actor))
+        .created,
+    ).toBe(true);
+  });
+
+  test("系统性失败进入持久化重试，到期后幂等重放并成功", async () => {
+    const clock = new FakeClock("2026-07-10T00:00:00.000Z");
+    const repository = new InMemoryBulkDataRepository();
+    let attempts = 0;
+    const handler: BulkResourceHandler = {
+      scopeMode: "platform",
+      resource: "dictionaries",
+      async importRow() {
+        attempts += 1;
+        if (attempts === 1) throw new Error("数据库暂不可用");
+      },
+      async exportRows() {
+        return [];
+      },
+    };
+    const service = new BulkDataService(
+      repository,
+      [handler],
+      { retryDelaySeconds: 30 },
+      clock.now,
+    );
+    const created = await service.createImport(
+      "dictionaries",
+      validLine("dict_retry"),
+      "upsert",
+      "retry:key:001",
+      actor,
+    );
+
+    expect(await service.processNext()).toMatchObject({ failed: true });
+    expect(await service.getOwnedJob(created.job.id, actor.id)).toMatchObject({
+      status: "retry-pending",
+      attempt: 1,
+    });
+    expect(await service.processNext()).toEqual({ claimed: false });
+    clock.advanceSeconds(30);
+    expect(await service.processNext()).toMatchObject({
+      claimed: true,
+      succeededRows: 1,
+    });
+    expect(await service.getOwnedJob(created.job.id, actor.id)).toMatchObject({
+      status: "succeeded",
+      attempt: 2,
+    });
+  });
+
+  test("输入、行数、单行边界和 7 天 artifact 清理是固定行为", async () => {
+    const clock = new FakeClock("2026-07-10T00:00:00.000Z");
+    const repository = new InMemoryBulkDataRepository();
+    const service = new BulkDataService(
+      repository,
+      [emptyHandler()],
+      { maxInputBytes: 100, maxRows: 2, maxLineBytes: 60 },
+      clock.now,
+    );
+    await expectInputCode(service, "", "EMPTY_INPUT");
+    await expectInputCode(service, "x".repeat(101), "INPUT_TOO_LARGE");
+    await expectInputCode(service, "{}\n{}\n{}", "TOO_MANY_ROWS");
+    await expectInputCode(
+      service,
+      JSON.stringify({ value: "x".repeat(70) }),
+      "LINE_TOO_LARGE",
+    );
+
+    const created = await service.createImport(
+      "dictionaries",
+      "{}",
+      "create",
+      "retention:key:001",
+      actor,
+    );
+    await service.processNext();
+    expect(
+      await service.getOwnedArtifact(created.job.id, "result", actor.id),
+    ).toBeDefined();
+    clock.advanceSeconds(7 * 86_400 + 1);
+    expect(await service.cleanupExpired()).toEqual({ deletedArtifacts: 2 });
+  });
+
+  test("导出生成可下载 NDJSON 结果并按申请人隔离", async () => {
+    const repository = new InMemoryBulkDataRepository();
+    const service = fixtureService(repository);
+    const created = await service.createExport(
+      "dictionaries",
+      "export:key:001",
+      actor,
+    );
+    await service.processNext();
+    const artifact = await service.getOwnedArtifact(
+      created.job.id,
+      "result",
+      actor.id,
+    );
+    expect(artifact).toMatchObject({
+      contentType: "application/x-ndjson; charset=utf-8",
+    });
+    expect(artifact?.content).toContain('"code":"color"');
+    expect(
+      await service.getOwnedArtifact(created.job.id, "result", "user_2"),
+    ).toBeUndefined();
+  });
+
+  test("结果 artifact 通过 BlobStore 持久化、流式读取并随留存清理", async () => {
+    const clock = new FakeClock("2026-07-10T00:00:00.000Z");
+    const repository = new InMemoryBulkDataRepository();
+    const blobStore = new MemoryBlobStore();
+    const service = new BulkDataService(
+      repository,
+      [emptyHandler()],
+      {},
+      clock.now,
+      blobStore,
+    );
+    const created = await service.createExport(
+      "dictionaries",
+      "export:blob:001",
+      actor,
+    );
+    await service.processNext();
+
+    const metadata = await repository.getArtifact(created.job.id, "result");
+    expect(metadata?.content).toBeUndefined();
+    expect(metadata?.artifactRef).toMatch(/^blob_[a-f0-9]{32}$/);
+    if (!metadata?.artifactRef) throw new Error("批处理 Blob 引用未持久化");
+    expect(blobStore.size).toBe(1);
+    expect(await service.listActiveArtifactRefs(clock.now())).toEqual([
+      metadata.artifactRef,
+    ]);
+    const opened = await service.openOwnedArtifact(
+      created.job.id,
+      "result",
+      actor.id,
+    );
+    expect(await new Response(opened?.body).text()).toBe("");
+
+    clock.advanceSeconds(7 * 86_400 + 1);
+    expect(await service.listActiveArtifactRefs(clock.now())).toEqual([]);
+    expect(await service.cleanupExpired()).toEqual({ deletedArtifacts: 1 });
+    expect(blobStore.size).toBe(0);
+  });
+
+  test("异步批处理只能注册平台资源并由服务端恢复显式 all scope", async () => {
+    let receivedActor: RequestActor | undefined;
+    const handler: BulkResourceHandler = {
+      scopeMode: "platform",
+      resource: "platform-fixture",
+      async importRow() {},
+      async exportRows(jobActor) {
+        receivedActor = jobActor;
+        return [];
+      },
+    };
+    const service = new BulkDataService(new InMemoryBulkDataRepository(), [
+      handler,
+    ]);
+    await service.createExport("platform-fixture", "platform:scope:001", actor);
+    await service.processNext();
+
+    expect(isServerScopeContext(receivedActor?.scopeContext)).toBe(true);
+    expect(receivedActor?.scopeContext?.grants).toContainEqual({
+      scope: "all",
+    });
+  });
+});
+
+function fixtureService(repository: InMemoryBulkDataRepository) {
+  const dictionaries = new InMemoryEntityRepository<ManagedDictionary>([
+    {
+      id: "dict_color",
+      code: "color",
+      name: "颜色",
+      status: "active",
+      items: [],
+      createdAt: "2026-07-10T00:00:00.000Z",
+      createdBy: "system",
+      updatedAt: "2026-07-10T00:00:00.000Z",
+      updatedBy: "system",
+    },
+  ]);
+  return new BulkDataService(repository, [
+    new DictionaryBulkResourceHandler(dictionaries),
+  ]);
+}
+
+function emptyHandler(): BulkResourceHandler {
+  return {
+    scopeMode: "platform",
+    resource: "dictionaries",
+    async importRow() {},
+    async exportRows() {
+      return [];
+    },
+  };
+}
+
+function validLine(id: string) {
+  return JSON.stringify({ id, code: id, name: id, items: [] });
+}
+
+async function expectInputCode(
+  service: BulkDataService,
+  content: string,
+  code: BulkDataInputError["code"],
+) {
+  await expect(
+    service.createImport(
+      "dictionaries",
+      content,
+      "create",
+      `boundary:${code}`,
+      actor,
+    ),
+  ).rejects.toMatchObject({ code });
+}
+
+class FakeClock {
+  private current: Date;
+  constructor(iso: string) {
+    this.current = new Date(iso);
+  }
+  readonly now = () => new Date(this.current);
+  advanceSeconds(seconds: number) {
+    this.current = new Date(this.current.getTime() + seconds * 1_000);
+  }
+}

package/template/apps/server/src/bulk-data/service.ts

@@ -0,0 +1,319 @@
+import type { RequestActor } from "../guards.js";
+import type { BlobStore } from "@windy/storage";
+import type { LegacyDurableAdapter } from "../jobs/legacy-adapter.js";
+import { BulkArtifactStore } from "./artifact-store.js";
+import {
+  bulkJobActor,
+  createTextArtifact,
+  safeBulkError,
+  sha256,
+} from "./helpers.js";
+import {
+  BulkDataInputError,
+  defaultBulkDataLimits,
+  isExpectedRowError,
+  nonEmptyLines,
+  rowFailure,
+  validateContent,
+  validateIdempotencyKey,
+  type BulkDataLimits,
+} from "./input-validation.js";
+import type { BulkDataRepository } from "./repository.js";
+import type { BulkResourceHandler } from "./resource-handler.js";
+import type {
+  BulkArtifactKind,
+  BulkDataJob,
+  BulkRowFailure,
+  JobProgress,
+} from "./types.js";
+export * from "./input-validation.js";
+
+export class BulkDataService {
+  private readonly handlers = new Map<string, BulkResourceHandler>();
+  private readonly limits: BulkDataLimits;
+  private readonly artifacts: BulkArtifactStore;
+
+  constructor(
+    private readonly repository: BulkDataRepository,
+    handlers: BulkResourceHandler[],
+    limits: Partial<BulkDataLimits> = {},
+    private readonly now: () => Date = () => new Date(),
+    blobStore?: BlobStore,
+    private readonly durable?: LegacyDurableAdapter,
+  ) {
+    this.limits = { ...defaultBulkDataLimits, ...limits };
+    this.artifacts = new BulkArtifactStore(repository, blobStore);
+    handlers.forEach((handler) => this.handlers.set(handler.resource, handler));
+  }
+
+  async createImport(
+    resource: string,
+    content: string,
+    mode: "create" | "upsert",
+    idempotencyKey: string,
+    actor: RequestActor,
+  ) {
+    this.requireHandler(resource);
+    validateIdempotencyKey(idempotencyKey);
+    const lines = validateContent(content, this.limits);
+    const digest = sha256(content);
+    const expiresAt = this.expiresAt();
+    const result = await this.repository.createJob(
+      {
+        kind: "import",
+        resource,
+        mode,
+        idempotencyKey,
+        requestDigest: digest,
+        requestedBy: actor.id,
+        totalRows: lines.length,
+        maxAttempts: this.limits.maxAttempts,
+      },
+      createTextArtifact(
+        "pending",
+        "source",
+        `${resource}-import.ndjson`,
+        content,
+        expiresAt,
+      ),
+    );
+    if (result.digestConflict) {
+      throw new BulkDataInputError(
+        "IDEMPOTENCY_CONFLICT",
+        "同一幂等键已用于不同输入内容",
+      );
+    }
+    return result;
+  }
+
+  async createExport(
+    resource: string,
+    idempotencyKey: string,
+    actor: RequestActor,
+  ) {
+    this.requireHandler(resource);
+    validateIdempotencyKey(idempotencyKey);
+    return this.repository.createJob({
+      kind: "export",
+      resource,
+      idempotencyKey,
+      requestDigest: sha256(`${resource}:export:v1`),
+      requestedBy: actor.id,
+      totalRows: 0,
+      maxAttempts: this.limits.maxAttempts,
+    });
+  }
+
+  listJobs(actorId: string, limit?: number) {
+    return this.repository.listJobs(actorId, limit);
+  }
+
+  async getOwnedJob(id: string, actorId: string) {
+    const job = await this.repository.getJob(id);
+    return job?.requestedBy === actorId ? job : undefined;
+  }
+
+  async getOwnedArtifact(
+    jobId: string,
+    kind: BulkArtifactKind,
+    actorId: string,
+  ) {
+    const job = await this.getOwnedJob(jobId, actorId);
+    if (!job) return undefined;
+    const result = await this.repository.getArtifact(jobId, kind);
+    return result && result.expiresAt > this.now().toISOString()
+      ? result
+      : undefined;
+  }
+
+  async openOwnedArtifact(
+    jobId: string,
+    kind: BulkArtifactKind,
+    actorId: string,
+  ) {
+    const artifact = await this.getOwnedArtifact(jobId, kind, actorId);
+    return artifact ? this.artifacts.open(artifact) : undefined;
+  }
+
+  async processNext() {
+    if (this.durable) {
+      const resumed = await this.durable.runNext();
+      if (resumed.status !== "idle" && resumed.legacyId) {
+        return this.resultFor(resumed.legacyId);
+      }
+      const [orphan] = await this.repository.listUnlinkedRunning(1);
+      if (orphan) return this.dispatchDurable(orphan);
+    }
+    const job = await this.repository.claimNext(this.now());
+    if (!job) return { claimed: false };
+    if (this.durable) return this.dispatchDurable(job);
+    return this.processClaimed(job);
+  }
+
+  async resumeDurable(jobId: string, generation: number) {
+    const job = await this.repository.getJob(jobId);
+    if (!job || job.status !== "running" || job.attempt !== generation) {
+      return;
+    }
+    await this.processClaimed(job);
+  }
+
+  async cleanupExpired() {
+    return {
+      deletedArtifacts: await this.artifacts.cleanupExpired(this.now()),
+    };
+  }
+
+  listActiveArtifactRefs(now: Date) {
+    return this.repository.listActiveArtifactRefs(now);
+  }
+
+  private async dispatchDurable(job: BulkDataJob) {
+    await this.durable!.dispatch({
+      legacyId: job.id,
+      generation: job.attempt,
+      requestedBy: job.requestedBy,
+      maxAttempts: job.maxAttempts,
+      link: (durableJobId) =>
+        this.repository.linkDurableJob(job.id, job.attempt, durableJobId),
+    });
+    return this.resultFor(job.id);
+  }
+
+  private async resultFor(jobId: string) {
+    const job = await this.repository.getJob(jobId);
+    if (!job) return { claimed: true, jobId };
+    if (["retry-pending", "failed"].includes(job.status)) {
+      return { claimed: true, jobId, failed: true };
+    }
+    return {
+      claimed: true,
+      jobId,
+      processedRows: job.processedRows,
+      succeededRows: job.succeededRows,
+      failedRows: job.failedRows,
+    };
+  }
+
+  private async processClaimed(job: BulkDataJob) {
+    try {
+      const progress =
+        job.kind === "import"
+          ? await this.processImport(job)
+          : await this.processExport(job);
+      return { claimed: true, jobId: job.id, ...progress };
+    } catch (error) {
+      const now = this.now();
+      const retryAt = new Date(
+        now.getTime() + this.limits.retryDelaySeconds * 1_000,
+      );
+      await this.repository.retryOrFail(
+        job.id,
+        safeBulkError(error),
+        now,
+        retryAt,
+      );
+      return { claimed: true, jobId: job.id, failed: true };
+    }
+  }
+
+  private async processImport(job: BulkDataJob) {
+    const source = await this.repository.getArtifact(job.id, "source");
+    if (!source) throw new Error("导入源文件不存在或已清理");
+    const handler = this.requireHandler(job.resource);
+    const actor = bulkJobActor(job.requestedBy);
+    const failures: BulkRowFailure[] = [];
+    const progress: JobProgress = {
+      processedRows: 0,
+      succeededRows: 0,
+      failedRows: 0,
+    };
+    const lines = nonEmptyLines(await this.artifacts.readText(source));
+    for (const [index, line] of lines.entries()) {
+      try {
+        const value = JSON.parse(line);
+        await handler.importRow(value, job.mode || "create", actor);
+        progress.succeededRows += 1;
+      } catch (error) {
+        if (!isExpectedRowError(error)) {
+          throw error;
+        }
+        progress.failedRows += 1;
+        failures.push(rowFailure(index + 1, line, error));
+      }
+      progress.processedRows += 1;
+      if (progress.processedRows % 100 === 0) {
+        await this.repository.updateProgress(job.id, progress, this.now());
+      }
+    }
+    const expiresAt = this.expiresAt();
+    await this.artifacts.put(
+      createTextArtifact(
+        job.id,
+        "result",
+        `${job.resource}-import-result.json`,
+        JSON.stringify({ jobId: job.id, ...progress }, null, 2),
+        expiresAt,
+        "application/json; charset=utf-8",
+      ),
+      true,
+    );
+    if (failures.length) {
+      await this.artifacts.put(
+        createTextArtifact(
+          job.id,
+          "failures",
+          `${job.resource}-import-failures.ndjson`,
+          failures.map((failure) => JSON.stringify(failure)).join("\n") + "\n",
+          expiresAt,
+        ),
+        true,
+      );
+    }
+    await this.repository.finish(
+      job.id,
+      failures.length ? "completed-with-errors" : "succeeded",
+      progress,
+      this.now(),
+    );
+    return progress;
+  }
+
+  private async processExport(job: BulkDataJob) {
+    const rows = await this.requireHandler(job.resource).exportRows(
+      bulkJobActor(job.requestedBy),
+    );
+    const content =
+      rows.map((row) => JSON.stringify(row)).join("\n") +
+      (rows.length ? "\n" : "");
+    const progress = {
+      processedRows: rows.length,
+      succeededRows: rows.length,
+      failedRows: 0,
+    };
+    await this.artifacts.put(
+      createTextArtifact(
+        job.id,
+        "result",
+        `${job.resource}-export.ndjson`,
+        content,
+        this.expiresAt(),
+      ),
+      true,
+    );
+    await this.repository.finish(job.id, "succeeded", progress, this.now());
+    return progress;
+  }
+
+  private requireHandler(resource: string) {
+    const handler = this.handlers.get(resource);
+    if (!handler) throw new Error(`不支持批处理资源：${resource}`);
+    return handler;
+  }
+
+  private expiresAt() {
+    return new Date(
+      this.now().getTime() + this.limits.retentionDays * 86_400_000,
+    );
+  }
+}