create-windy 0.1.0

package/template/apps/server/src/background/task-guard.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, test } from "bun:test";
+import { createFoundationSnapshot } from "../foundation.js";
+import { createServerRuntime } from "../runtime.js";
+import { createSystemRepositories } from "../system/seed.js";
+import { runGuardedBackgroundTask } from "./task-guard.js";
+
+describe("background task guard", () => {
+  test("每次执行读取动态 Feature 状态，恢复无需重启", async () => {
+    const snapshot = createFoundationSnapshot();
+    const repositories = createSystemRepositories(snapshot);
+    await repositories.features.syncManifest(snapshot.features);
+    const runtime = createServerRuntime(
+      { WINDY_DEV_ADMIN_TOKEN: "token" },
+      { featureResolver: repositories.features },
+    );
+    const feature = snapshot.features.find(
+      (item) => item.key === "system.profile",
+    );
+    if (!feature) throw new Error("缺少 system.profile Feature");
+    const context = await runtime.createGuardContext(
+      new Request("http://localhost/tasks/profile-refresh", {
+        headers: { authorization: "Bearer token" },
+      }),
+    );
+    const policy = {
+      taskKey: "system.profile.refresh",
+      permission: { permissionKey: "system.feature.read" },
+      feature,
+    };
+    let executions = 0;
+
+    await repositories.features.updateState(
+      feature.key,
+      { enabled: false },
+      context.actor,
+    );
+    const denied = await runGuardedBackgroundTask(
+      runtime,
+      context,
+      policy,
+      () => ++executions,
+    );
+
+    expect(denied).toMatchObject({ ok: false, reason: "feature-disabled" });
+    expect(executions).toBe(0);
+
+    await repositories.features.updateState(
+      feature.key,
+      { enabled: true, visible: "visible" },
+      context.actor,
+    );
+    const nextContext = await runtime.createGuardContext(
+      new Request("http://localhost/tasks/profile-refresh", {
+        headers: { authorization: "Bearer token" },
+      }),
+    );
+    const completed = await runGuardedBackgroundTask(
+      runtime,
+      nextContext,
+      policy,
+      () => ++executions,
+    );
+
+    expect(completed).toMatchObject({ ok: true, value: 1 });
+  });
+});

package/template/apps/server/src/background/task-guard.ts

@@ -0,0 +1,49 @@
+import type {
+  AccessGuardPolicy,
+  GuardPipelineResult,
+} from "../access-guards.js";
+import { evaluateAccessGuards } from "../access-guards.js";
+import type { RequestGuardContext } from "../guards.js";
+import type { createServerRuntime } from "../runtime.js";
+
+type ServerRuntime = Pick<
+  ReturnType<typeof createServerRuntime>,
+  "recordDecisions" | "resolveFeature"
+>;
+
+export interface BackgroundTaskPolicy extends AccessGuardPolicy {
+  taskKey: string;
+}
+
+export interface BackgroundTaskDenied {
+  ok: false;
+  taskKey: string;
+  reason: GuardPipelineResult["decision"]["reason"];
+}
+
+export interface BackgroundTaskCompleted<T> {
+  ok: true;
+  taskKey: string;
+  value: T;
+}
+
+export async function runGuardedBackgroundTask<T>(
+  runtime: ServerRuntime,
+  context: RequestGuardContext,
+  policy: BackgroundTaskPolicy,
+  execute: () => Promise<T> | T,
+  options: { recordDecision?: boolean } = {},
+): Promise<BackgroundTaskDenied | BackgroundTaskCompleted<T>> {
+  const pipeline = await evaluateAccessGuards(runtime, context, policy, {
+    record: options.recordDecision,
+  });
+  if (!pipeline.decision.allowed) {
+    return {
+      ok: false,
+      taskKey: policy.taskKey,
+      reason: pipeline.decision.reason,
+    };
+  }
+
+  return { ok: true, taskKey: policy.taskKey, value: await execute() };
+}

package/template/apps/server/src/bulk-data/artifact-store.ts

@@ -0,0 +1,82 @@
+import type { BlobStore } from "@windy/storage";
+import type { BulkDataRepository, NewArtifact } from "./repository.js";
+import type { BulkDataArtifact } from "./types.js";
+
+export interface OpenedBulkArtifact extends BulkDataArtifact {
+  body: ReadableStream<Uint8Array>;
+}
+
+export class BulkArtifactStore {
+  constructor(
+    private readonly repository: BulkDataRepository,
+    private readonly blobStore?: BlobStore,
+  ) {}
+
+  async put(input: NewArtifact, externalize = false) {
+    if (!externalize || !this.blobStore || input.content === undefined) {
+      return this.repository.putArtifact(input);
+    }
+    const bytes = new TextEncoder().encode(input.content);
+    const stored = await this.blobStore.put({
+      body: byteStream(bytes),
+      expectedByteSize: input.byteSize,
+      expectedContentHash: { algorithm: "sha256", value: input.sha256 },
+    });
+    try {
+      return await this.repository.putArtifact({
+        ...input,
+        content: undefined,
+        artifactRef: stored.ref,
+      });
+    } catch (error) {
+      await this.blobStore.delete(stored.ref);
+      throw error;
+    }
+  }
+
+  async open(artifact: BulkDataArtifact): Promise<OpenedBulkArtifact> {
+    if (artifact.artifactRef && this.blobStore) {
+      const blob = await this.blobStore.open(artifact.artifactRef);
+      if (
+        !blob ||
+        blob.byteSize !== artifact.byteSize ||
+        blob.contentHash.value !== artifact.sha256
+      ) {
+        throw new Error("批处理 artifact 内容不可用");
+      }
+      return { ...artifact, body: blob.body };
+    }
+    if (artifact.content === undefined) {
+      throw new Error("批处理 artifact 缺少内容存储 Adapter");
+    }
+    return {
+      ...artifact,
+      body: byteStream(new TextEncoder().encode(artifact.content)),
+    };
+  }
+
+  async readText(artifact: BulkDataArtifact) {
+    return new Response((await this.open(artifact)).body).text();
+  }
+
+  async cleanupExpired(now: Date) {
+    const deleted = await this.repository.cleanupExpired(now);
+    if (this.blobStore) {
+      for (const artifact of deleted) {
+        if (artifact.artifactRef) {
+          await this.blobStore.delete(artifact.artifactRef);
+        }
+      }
+    }
+    return deleted.length;
+  }
+}
+
+function byteStream(bytes: Uint8Array) {
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(bytes);
+      controller.close();
+    },
+  });
+}

package/template/apps/server/src/bulk-data/dictionary-handler.ts

@@ -0,0 +1,122 @@
+import type { LifecycleStatus } from "@windy/shared";
+import type { RequestActor } from "../guards.js";
+import type { ManagedDictionary } from "../system/entities.js";
+import type { EntityRepository } from "../system/repository.js";
+import {
+  BulkRowImportError,
+  BulkRowValidationError,
+  type BulkResourceHandler,
+} from "./resource-handler.js";
+
+export class DictionaryBulkResourceHandler implements BulkResourceHandler {
+  readonly scopeMode = "platform";
+  readonly resource = "dictionaries";
+
+  constructor(
+    private readonly repository: EntityRepository<ManagedDictionary>,
+  ) {}
+
+  async importRow(
+    value: unknown,
+    mode: "create" | "upsert",
+    actor: RequestActor,
+  ) {
+    const input = parseDictionary(value);
+    try {
+      await this.repository.importMany([input], actor, mode);
+    } catch (error) {
+      if (isUniqueViolation(error)) {
+        throw new BulkRowImportError(
+          "字典 id 或 code 已存在且不能按当前模式写入",
+        );
+      }
+      throw error;
+    }
+  }
+
+  async exportRows() {
+    return await this.repository.all({}, ["code", "name"]);
+  }
+}
+
+function isUniqueViolation(error: unknown) {
+  return Boolean(
+    error &&
+    typeof error === "object" &&
+    "code" in error &&
+    (error as { code?: unknown }).code === "23505",
+  );
+}
+
+function parseDictionary(value: unknown): Partial<ManagedDictionary> {
+  const row = record(value, "每行必须是 JSON 对象");
+  const id = requiredString(row.id, "id");
+  const code = requiredString(row.code, "code");
+  const name = requiredString(row.name, "name");
+  if (!/^[a-zA-Z0-9_.-]{1,64}$/.test(id)) {
+    throw new BulkRowValidationError(
+      "id 仅允许 1-64 位字母、数字、点、横线或下划线",
+    );
+  }
+  if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(code)) {
+    throw new BulkRowValidationError(
+      "code 仅允许 1-80 位字母、数字、点、横线或下划线",
+    );
+  }
+  return {
+    id,
+    code,
+    name,
+    description: optionalString(row.description, "description"),
+    status: lifecycle(row.status),
+    items: Array.isArray(row.items)
+      ? row.items.map((item, index) => parseItem(item, index + 1))
+      : [],
+  };
+}
+
+function parseItem(value: unknown, index: number) {
+  const item = record(value, `items[${index}] 必须是对象`);
+  const order = Number(item.order);
+  if (!Number.isSafeInteger(order) || order < 0) {
+    throw new BulkRowValidationError(`items[${index}].order 必须是非负整数`);
+  }
+  return {
+    label: requiredString(item.label, `items[${index}].label`),
+    value: requiredString(item.value, `items[${index}].value`),
+    status: lifecycle(item.status, `items[${index}].status`),
+    order,
+  };
+}
+
+function record(value: unknown, message: string): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new BulkRowValidationError(message);
+  }
+  return value as Record<string, unknown>;
+}
+
+function requiredString(value: unknown, field: string) {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new BulkRowValidationError(`${field} 必须是非空字符串`);
+  }
+  return value.trim();
+}
+
+function optionalString(value: unknown, field: string) {
+  if (value === undefined || value === null || value === "") return undefined;
+  if (typeof value !== "string") {
+    throw new BulkRowValidationError(`${field} 必须是字符串`);
+  }
+  return value;
+}
+
+function lifecycle(value: unknown, field = "status"): LifecycleStatus {
+  const status = value ?? "active";
+  if (!(["active", "disabled", "archived"] as unknown[]).includes(status)) {
+    throw new BulkRowValidationError(
+      `${field} 必须是 active、disabled 或 archived`,
+    );
+  }
+  return status as LifecycleStatus;
+}

package/template/apps/server/src/bulk-data/drizzle-repository.integration.test.ts

@@ -0,0 +1,156 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { resolve } from "node:path";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { migrate } from "drizzle-orm/node-postgres/migrator";
+import { Pool } from "pg";
+import { DrizzleBulkDataRepository } from "./drizzle-repository.js";
+
+const databaseUrl = process.env.BULK_DATA_TEST_DATABASE_URL;
+const describePostgres = databaseUrl ? describe : describe.skip;
+
+describePostgres("PostgreSQL bulk data repository", () => {
+  let pool: Pool;
+  let repository: DrizzleBulkDataRepository;
+  const baseTime = new Date("2026-07-10T00:00:00.000Z");
+
+  beforeAll(async () => {
+    pool = new Pool({ connectionString: databaseUrl });
+    const db = drizzle(pool);
+    await migrate(db, {
+      migrationsFolder: resolve(
+        import.meta.dir,
+        "../../../../packages/database/drizzle",
+      ),
+      migrationsSchema: "public",
+      migrationsTable: "__data01_test_migrations",
+    });
+    await pool.query("truncate table bulk_data_artifacts, bulk_data_jobs");
+    repository = new DrizzleBulkDataRepository(db);
+  });
+
+  afterAll(async () => {
+    await pool?.end();
+  });
+
+  test("并发相同幂等请求只创建一个任务和一份 source artifact", async () => {
+    const input = newJob("same-key");
+    const source = newArtifact("pending", "source", baseTime);
+    const results = await Promise.all([
+      repository.createJob(input, source),
+      repository.createJob(input, source),
+    ]);
+    expect(results.filter(({ created }) => created)).toHaveLength(1);
+    expect(new Set(results.map(({ job }) => job.id)).size).toBe(1);
+    const jobId = results[0]!.job.id;
+    expect(await repository.getArtifact(jobId, "source")).toMatchObject({
+      jobId,
+      content: "{}\n",
+    });
+    const claimed = await repository.claimNext(baseTime);
+    if (!claimed) throw new Error("应领取幂等任务");
+    await repository.finish(
+      claimed.id,
+      "succeeded",
+      { processedRows: 1, succeededRows: 1, failedRows: 0 },
+      baseTime,
+    );
+  });
+
+  test("SKIP LOCKED 并发领取一次并持久化进度与完成状态", async () => {
+    const created = await repository.createJob(newJob("claim-key"));
+    const claims = await Promise.all([
+      repository.claimNext(baseTime),
+      repository.claimNext(baseTime),
+    ]);
+    const claimed = claims.find((value) => value?.id === created.job.id);
+    expect(claims.filter((value) => value?.id === created.job.id)).toHaveLength(
+      1,
+    );
+    if (!claimed) throw new Error("应领取新建任务");
+    await repository.updateProgress(
+      claimed.id,
+      { processedRows: 50, succeededRows: 49, failedRows: 1 },
+      baseTime,
+    );
+    await repository.finish(
+      claimed.id,
+      "completed-with-errors",
+      { processedRows: 100, succeededRows: 99, failedRows: 1 },
+      baseTime,
+    );
+    expect(await repository.getJob(claimed.id)).toMatchObject({
+      status: "completed-with-errors",
+      processedRows: 100,
+      succeededRows: 99,
+      failedRows: 1,
+      attempt: 1,
+    });
+  });
+
+  test("重试到期后可由重建 Repository 领取，artifact 到期清理", async () => {
+    const created = await repository.createJob(newJob("retry-key"));
+    const claimed = await repository.claimNext(baseTime);
+    if (!claimed || claimed.id !== created.job.id)
+      throw new Error("应领取重试任务");
+    await repository.retryOrFail(
+      claimed.id,
+      "模拟暂时错误",
+      baseTime,
+      new Date("2026-07-10T00:00:30.000Z"),
+    );
+    expect(
+      await repository.claimNext(new Date("2026-07-10T00:00:29.999Z")),
+    ).toBeUndefined();
+
+    const restarted = new DrizzleBulkDataRepository(drizzle(pool));
+    expect(
+      await restarted.claimNext(new Date("2026-07-10T00:00:30.000Z")),
+    ).toMatchObject({ id: claimed.id, attempt: 2, status: "running" });
+    await restarted.finish(
+      claimed.id,
+      "succeeded",
+      { processedRows: 1, succeededRows: 1, failedRows: 0 },
+      new Date("2026-07-10T00:00:30.000Z"),
+    );
+    await restarted.putArtifact(
+      newArtifact(claimed.id, "result", new Date("2026-07-17T00:00:00.000Z")),
+    );
+    expect(
+      await restarted.cleanupExpired(new Date("2026-07-16T23:59:59.999Z")),
+    ).toHaveLength(1);
+    expect(await restarted.getArtifact(claimed.id, "result")).toBeDefined();
+    expect(
+      await restarted.cleanupExpired(new Date("2026-07-17T00:00:00.000Z")),
+    ).toHaveLength(1);
+  });
+});
+
+function newJob(idempotencyKey: string) {
+  return {
+    kind: "import" as const,
+    resource: "dictionaries",
+    mode: "upsert" as const,
+    idempotencyKey,
+    requestDigest: "a".repeat(64),
+    requestedBy: "user_pg",
+    totalRows: 100,
+    maxAttempts: 3,
+  };
+}
+
+function newArtifact(
+  jobId: string,
+  kind: "source" | "result",
+  expiresAt: Date,
+) {
+  return {
+    jobId,
+    kind,
+    filename: `${kind}.ndjson`,
+    contentType: "application/x-ndjson; charset=utf-8",
+    content: "{}\n",
+    sha256: "b".repeat(64),
+    byteSize: 3,
+    expiresAt,
+  };
+}