create-windy 0.1.0

package/template/apps/server/src/bulk-data/drizzle-repository.ts

@@ -0,0 +1,298 @@
+import { bulkDataArtifacts, bulkDataJobs } from "@windy/database";
+import { artifactRef } from "@windy/storage";
+import { and, asc, desc, eq, gt, isNotNull, lte, or, sql } from "drizzle-orm";
+import type { ArtifactRef } from "@windy/storage";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import type {
+  BulkArtifactKind,
+  BulkDataArtifact,
+  BulkDataJob,
+  JobProgress,
+  NewBulkJob,
+} from "./types.js";
+import type {
+  BulkDataRepository,
+  CreateJobResult,
+  NewArtifact,
+} from "./repository.js";
+
+export class DrizzleBulkDataRepository implements BulkDataRepository {
+  constructor(private readonly db: NodePgDatabase) {}
+
+  async createJob(input: NewBulkJob, source?: NewArtifact) {
+    return this.db.transaction(async (tx) => {
+      const now = new Date();
+      const [created] = await tx
+        .insert(bulkDataJobs)
+        .values({
+          ...input,
+          id: Bun.randomUUIDv7(),
+          status: "queued",
+          createdAt: now,
+          updatedAt: now,
+        })
+        .onConflictDoNothing({
+          target: [
+            bulkDataJobs.requestedBy,
+            bulkDataJobs.kind,
+            bulkDataJobs.resource,
+            bulkDataJobs.idempotencyKey,
+          ],
+        })
+        .returning();
+      const row =
+        created ||
+        (
+          await tx
+            .select()
+            .from(bulkDataJobs)
+            .where(idempotencyWhere(input))
+            .limit(1)
+        )[0];
+      if (!row) throw new Error("幂等任务创建后无法读取");
+      if (created && source) {
+        await putArtifact(tx, { ...source, jobId: row.id });
+      }
+      return {
+        job: mapJob(row),
+        created: Boolean(created),
+        digestConflict: row.requestDigest !== input.requestDigest,
+      } satisfies CreateJobResult;
+    });
+  }
+
+  async getJob(id: string) {
+    const [row] = await this.db
+      .select()
+      .from(bulkDataJobs)
+      .where(eq(bulkDataJobs.id, id))
+      .limit(1);
+    return row ? mapJob(row) : undefined;
+  }
+
+  async listJobs(requestedBy: string, limit = 50) {
+    const rows = await this.db
+      .select()
+      .from(bulkDataJobs)
+      .where(eq(bulkDataJobs.requestedBy, requestedBy))
+      .orderBy(desc(bulkDataJobs.createdAt))
+      .limit(limit);
+    return rows.map(mapJob);
+  }
+
+  async listUnlinkedRunning(limit = 100) {
+    const rows = await this.db
+      .select()
+      .from(bulkDataJobs)
+      .where(
+        and(
+          eq(bulkDataJobs.status, "running"),
+          sql`${bulkDataJobs.durableJobId} is null`,
+        ),
+      )
+      .orderBy(asc(bulkDataJobs.startedAt))
+      .limit(limit);
+    return rows.map(mapJob);
+  }
+
+  async claimNext(now: Date) {
+    return this.db.transaction(async (tx) => {
+      const rows = await tx
+        .select({ id: bulkDataJobs.id })
+        .from(bulkDataJobs)
+        .where(
+          or(
+            eq(bulkDataJobs.status, "queued"),
+            and(
+              eq(bulkDataJobs.status, "retry-pending"),
+              lte(bulkDataJobs.nextAttemptAt, now),
+            ),
+          ),
+        )
+        .orderBy(asc(bulkDataJobs.createdAt))
+        .for("update", { skipLocked: true })
+        .limit(1);
+      const id = rows[0]?.id;
+      if (!id) return undefined;
+      const [claimed] = await tx
+        .update(bulkDataJobs)
+        .set({
+          status: "running",
+          attempt: sql`${bulkDataJobs.attempt} + 1`,
+          startedAt: now,
+          nextAttemptAt: null,
+          error: null,
+          durableJobId: null,
+          updatedAt: now,
+        })
+        .where(eq(bulkDataJobs.id, id))
+        .returning();
+      return claimed ? mapJob(claimed) : undefined;
+    });
+  }
+
+  async linkDurableJob(jobId: string, attempt: number, durableJobId: string) {
+    const [row] = await this.db
+      .update(bulkDataJobs)
+      .set({ durableJobId })
+      .where(
+        and(
+          eq(bulkDataJobs.id, jobId),
+          eq(bulkDataJobs.status, "running"),
+          eq(bulkDataJobs.attempt, attempt),
+          sql`(${bulkDataJobs.durableJobId} is null or ${bulkDataJobs.durableJobId} = ${durableJobId})`,
+        ),
+      )
+      .returning({ id: bulkDataJobs.id });
+    return Boolean(row);
+  }
+
+  async updateProgress(id: string, progress: JobProgress, now: Date) {
+    await this.db
+      .update(bulkDataJobs)
+      .set({ ...progress, updatedAt: now })
+      .where(and(eq(bulkDataJobs.id, id), eq(bulkDataJobs.status, "running")));
+  }
+
+  async finish(
+    id: string,
+    status: "succeeded" | "completed-with-errors",
+    progress: JobProgress,
+    now: Date,
+  ) {
+    await this.db
+      .update(bulkDataJobs)
+      .set({ ...progress, status, finishedAt: now, updatedAt: now })
+      .where(and(eq(bulkDataJobs.id, id), eq(bulkDataJobs.status, "running")));
+  }
+
+  async retryOrFail(id: string, error: string, now: Date, retryAt?: Date) {
+    const job = await this.getJob(id);
+    if (!job) throw new Error(`批处理任务不存在：${id}`);
+    const retry = Boolean(retryAt && job.attempt < job.maxAttempts);
+    await this.db
+      .update(bulkDataJobs)
+      .set({
+        status: retry ? "retry-pending" : "failed",
+        error,
+        nextAttemptAt: retry ? retryAt : null,
+        finishedAt: retry ? null : now,
+        updatedAt: now,
+      })
+      .where(and(eq(bulkDataJobs.id, id), eq(bulkDataJobs.status, "running")));
+  }
+
+  putArtifact(input: NewArtifact) {
+    return putArtifact(this.db, input);
+  }
+
+  async getArtifact(jobId: string, kind: BulkArtifactKind) {
+    const [row] = await this.db
+      .select()
+      .from(bulkDataArtifacts)
+      .where(
+        and(
+          eq(bulkDataArtifacts.jobId, jobId),
+          eq(bulkDataArtifacts.kind, kind),
+        ),
+      )
+      .limit(1);
+    return row ? mapArtifact(row) : undefined;
+  }
+
+  async listActiveArtifactRefs(now: Date): Promise<ArtifactRef[]> {
+    const rows = await this.db
+      .select({ artifactRef: bulkDataArtifacts.artifactRef })
+      .from(bulkDataArtifacts)
+      .where(
+        and(
+          isNotNull(bulkDataArtifacts.artifactRef),
+          gt(bulkDataArtifacts.expiresAt, now),
+        ),
+      );
+    return rows.map(({ artifactRef: value }) => artifactRef(value!));
+  }
+
+  async cleanupExpired(now: Date) {
+    const deleted = await this.db
+      .delete(bulkDataArtifacts)
+      .where(
+        and(
+          lte(bulkDataArtifacts.expiresAt, now),
+          sql`exists (
+            select 1 from ${bulkDataJobs}
+            where ${bulkDataJobs.id} = ${bulkDataArtifacts.jobId}
+              and ${bulkDataJobs.status} in ('succeeded', 'completed-with-errors', 'failed')
+          )`,
+        ),
+      )
+      .returning();
+    return deleted.map(mapArtifact);
+  }
+}
+
+type DatabaseLike = Pick<NodePgDatabase, "insert">;
+
+async function putArtifact(db: DatabaseLike, input: NewArtifact) {
+  const [row] = await db
+    .insert(bulkDataArtifacts)
+    .values({
+      ...input,
+      id: Bun.randomUUIDv7(),
+      createdAt: new Date(),
+    })
+    .onConflictDoUpdate({
+      target: [bulkDataArtifacts.jobId, bulkDataArtifacts.kind],
+      set: {
+        filename: input.filename,
+        contentType: input.contentType,
+        content: input.content,
+        artifactRef: input.artifactRef,
+        sha256: input.sha256,
+        byteSize: input.byteSize,
+        createdAt: new Date(),
+        expiresAt: input.expiresAt,
+      },
+    })
+    .returning();
+  if (!row) throw new Error("批处理 artifact 写入失败");
+  return mapArtifact(row);
+}
+
+function idempotencyWhere(input: NewBulkJob) {
+  return and(
+    eq(bulkDataJobs.requestedBy, input.requestedBy),
+    eq(bulkDataJobs.kind, input.kind),
+    eq(bulkDataJobs.resource, input.resource),
+    eq(bulkDataJobs.idempotencyKey, input.idempotencyKey),
+  );
+}
+
+function mapJob(row: typeof bulkDataJobs.$inferSelect): BulkDataJob {
+  return {
+    ...row,
+    kind: row.kind as BulkDataJob["kind"],
+    mode: row.mode as BulkDataJob["mode"],
+    status: row.status as BulkDataJob["status"],
+    nextAttemptAt: row.nextAttemptAt?.toISOString(),
+    error: row.error || undefined,
+    durableJobId: row.durableJobId || undefined,
+    createdAt: row.createdAt.toISOString(),
+    startedAt: row.startedAt?.toISOString(),
+    finishedAt: row.finishedAt?.toISOString(),
+    updatedAt: row.updatedAt.toISOString(),
+  };
+}
+
+function mapArtifact(
+  row: typeof bulkDataArtifacts.$inferSelect,
+): BulkDataArtifact {
+  return {
+    ...row,
+    kind: row.kind as BulkDataArtifact["kind"],
+    createdAt: row.createdAt.toISOString(),
+    expiresAt: row.expiresAt.toISOString(),
+    content: row.content ?? undefined,
+    artifactRef: row.artifactRef ? artifactRef(row.artifactRef) : undefined,
+  };
+}

package/template/apps/server/src/bulk-data/helpers.ts

@@ -0,0 +1,36 @@
+import type { RequestActor } from "../guards.js";
+import { platformDataActor } from "../data-access/platform-actor.js";
+import type { NewArtifact } from "./repository.js";
+import type { BulkArtifactKind } from "./types.js";
+
+export function createTextArtifact(
+  jobId: string,
+  kind: BulkArtifactKind,
+  filename: string,
+  content: string,
+  expiresAt: Date,
+  contentType = "application/x-ndjson; charset=utf-8",
+): NewArtifact {
+  return {
+    jobId,
+    kind,
+    filename,
+    contentType,
+    content,
+    sha256: sha256(content),
+    byteSize: Buffer.byteLength(content),
+    expiresAt,
+  };
+}
+
+export function sha256(value: string) {
+  return new Bun.CryptoHasher("sha256").update(value).digest("hex");
+}
+
+export function safeBulkError(error: unknown) {
+  return (error instanceof Error ? error.message : String(error)).slice(0, 500);
+}
+
+export function bulkJobActor(requestedBy: string): RequestActor {
+  return platformDataActor(requestedBy, "批处理任务申请人", "user");
+}

package/template/apps/server/src/bulk-data/input-validation.ts

@@ -0,0 +1,99 @@
+import { safeBulkError, sha256 } from "./helpers.js";
+import {
+  BulkRowImportError,
+  BulkRowValidationError,
+} from "./resource-handler.js";
+import type { BulkRowFailure } from "./types.js";
+
+export const BULK_DATA_DEFAULT_RETENTION_DAYS = 7;
+export const BULK_DATA_MAX_INPUT_BYTES = 10 * 1024 * 1024;
+export const BULK_DATA_MAX_ROWS = 50_000;
+export const BULK_DATA_MAX_LINE_BYTES = 256 * 1024;
+
+export interface BulkDataLimits {
+  retentionDays: number;
+  maxInputBytes: number;
+  maxRows: number;
+  maxLineBytes: number;
+  maxAttempts: number;
+  retryDelaySeconds: number;
+}
+
+export const defaultBulkDataLimits: BulkDataLimits = {
+  retentionDays: BULK_DATA_DEFAULT_RETENTION_DAYS,
+  maxInputBytes: BULK_DATA_MAX_INPUT_BYTES,
+  maxRows: BULK_DATA_MAX_ROWS,
+  maxLineBytes: BULK_DATA_MAX_LINE_BYTES,
+  maxAttempts: 3,
+  retryDelaySeconds: 30,
+};
+
+export class BulkDataInputError extends Error {
+  constructor(
+    readonly code:
+      | "INVALID_IDEMPOTENCY_KEY"
+      | "IDEMPOTENCY_CONFLICT"
+      | "INPUT_TOO_LARGE"
+      | "TOO_MANY_ROWS"
+      | "LINE_TOO_LARGE"
+      | "EMPTY_INPUT",
+    message: string,
+  ) {
+    super(message);
+  }
+}
+
+export function validateIdempotencyKey(value: string) {
+  if (!/^[a-zA-Z0-9_.:-]{8,160}$/.test(value)) {
+    throw new BulkDataInputError(
+      "INVALID_IDEMPOTENCY_KEY",
+      "幂等键必须为 8-160 位字母、数字、点、冒号、横线或下划线",
+    );
+  }
+}
+
+export function validateContent(content: string, limits: BulkDataLimits) {
+  if (!content.trim())
+    throw new BulkDataInputError("EMPTY_INPUT", "导入内容不能为空");
+  if (Buffer.byteLength(content) > limits.maxInputBytes) {
+    throw new BulkDataInputError("INPUT_TOO_LARGE", "导入文件超过大小上限");
+  }
+  const lines = nonEmptyLines(content);
+  if (lines.length > limits.maxRows) {
+    throw new BulkDataInputError("TOO_MANY_ROWS", "导入行数超过上限");
+  }
+  if (lines.some((line) => Buffer.byteLength(line) > limits.maxLineBytes)) {
+    throw new BulkDataInputError("LINE_TOO_LARGE", "单行超过大小上限");
+  }
+  return lines;
+}
+
+export function nonEmptyLines(content: string) {
+  return content.split(/\r?\n/).filter((line) => line.trim());
+}
+
+export function rowFailure(
+  line: number,
+  input: string,
+  error: unknown,
+): BulkRowFailure {
+  return {
+    line,
+    code:
+      error instanceof SyntaxError
+        ? "INVALID_JSON"
+        : error instanceof BulkRowValidationError
+          ? "INVALID_ROW"
+          : "IMPORT_FAILED",
+    message: safeBulkError(error),
+    inputDigest: sha256(input),
+  };
+}
+
+export function isExpectedRowError(error: unknown) {
+  return (
+    error instanceof SyntaxError ||
+    error instanceof BulkRowValidationError ||
+    error instanceof BulkRowImportError
+  );
+}

package/template/apps/server/src/bulk-data/repository.ts

@@ -0,0 +1,246 @@
+import { randomUUIDv7 } from "bun";
+import type { ArtifactRef } from "@windy/storage";
+import type {
+  BulkArtifactKind,
+  BulkDataArtifact,
+  BulkDataJob,
+  BulkJobStatus,
+  JobProgress,
+  NewBulkJob,
+} from "./types.js";
+
+export interface CreateJobResult {
+  job: BulkDataJob;
+  created: boolean;
+  digestConflict: boolean;
+}
+
+export interface NewArtifact {
+  jobId: string;
+  kind: BulkArtifactKind;
+  filename: string;
+  contentType: string;
+  content?: string;
+  artifactRef?: ArtifactRef;
+  sha256: string;
+  byteSize: number;
+  expiresAt: Date;
+}
+
+export interface BulkDataRepository {
+  createJob(input: NewBulkJob, source?: NewArtifact): Promise<CreateJobResult>;
+  getJob(id: string): Promise<BulkDataJob | undefined>;
+  listJobs(requestedBy: string, limit?: number): Promise<BulkDataJob[]>;
+  listUnlinkedRunning(limit?: number): Promise<BulkDataJob[]>;
+  claimNext(now: Date): Promise<BulkDataJob | undefined>;
+  linkDurableJob(
+    jobId: string,
+    attempt: number,
+    durableJobId: string,
+  ): Promise<boolean>;
+  updateProgress(id: string, progress: JobProgress, now: Date): Promise<void>;
+  finish(
+    id: string,
+    status: Extract<BulkJobStatus, "succeeded" | "completed-with-errors">,
+    progress: JobProgress,
+    now: Date,
+  ): Promise<void>;
+  retryOrFail(
+    id: string,
+    error: string,
+    now: Date,
+    retryAt?: Date,
+  ): Promise<void>;
+  putArtifact(input: NewArtifact): Promise<BulkDataArtifact>;
+  getArtifact(
+    jobId: string,
+    kind: BulkArtifactKind,
+  ): Promise<BulkDataArtifact | undefined>;
+  listActiveArtifactRefs(now: Date): Promise<ArtifactRef[]>;
+  cleanupExpired(now: Date): Promise<BulkDataArtifact[]>;
+}
+
+export class InMemoryBulkDataRepository implements BulkDataRepository {
+  private readonly jobs = new Map<string, BulkDataJob>();
+  private readonly artifacts = new Map<string, BulkDataArtifact>();
+
+  async createJob(input: NewBulkJob, source?: NewArtifact) {
+    const existing = [...this.jobs.values()].find(
+      (job) =>
+        job.requestedBy === input.requestedBy &&
+        job.kind === input.kind &&
+        job.resource === input.resource &&
+        job.idempotencyKey === input.idempotencyKey,
+    );
+    if (existing) {
+      return {
+        job: { ...existing },
+        created: false,
+        digestConflict: existing.requestDigest !== input.requestDigest,
+      };
+    }
+    const now = new Date().toISOString();
+    const job: BulkDataJob = {
+      ...input,
+      id: randomUUIDv7(),
+      status: "queued",
+      processedRows: 0,
+      succeededRows: 0,
+      failedRows: 0,
+      attempt: 0,
+      createdAt: now,
+      updatedAt: now,
+    };
+    this.jobs.set(job.id, job);
+    if (source) await this.putArtifact({ ...source, jobId: job.id });
+    return { job: { ...job }, created: true, digestConflict: false };
+  }
+
+  async getJob(id: string) {
+    const job = this.jobs.get(id);
+    return job ? { ...job } : undefined;
+  }
+
+  async listJobs(requestedBy: string, limit = 50) {
+    return [...this.jobs.values()]
+      .filter((job) => job.requestedBy === requestedBy)
+      .sort((a, b) => b.createdAt.localeCompare(a.createdAt))
+      .slice(0, limit)
+      .map((job) => ({ ...job }));
+  }
+
+  async listUnlinkedRunning(limit = 100) {
+    return [...this.jobs.values()]
+      .filter((job) => job.status === "running" && !job.durableJobId)
+      .sort((a, b) => a.startedAt!.localeCompare(b.startedAt!))
+      .slice(0, limit)
+      .map((job) => ({ ...job }));
+  }
+
+  async claimNext(now: Date) {
+    const job = [...this.jobs.values()]
+      .filter(
+        (item) =>
+          item.status === "queued" ||
+          (item.status === "retry-pending" &&
+            Boolean(item.nextAttemptAt) &&
+            item.nextAttemptAt! <= now.toISOString()),
+      )
+      .sort((a, b) => a.createdAt.localeCompare(b.createdAt))[0];
+    if (!job) return undefined;
+    Object.assign(job, {
+      status: "running",
+      attempt: job.attempt + 1,
+      startedAt: now.toISOString(),
+      nextAttemptAt: undefined,
+      error: undefined,
+      durableJobId: undefined,
+      updatedAt: now.toISOString(),
+    });
+    return { ...job };
+  }
+
+  async linkDurableJob(jobId: string, attempt: number, durableJobId: string) {
+    const job = this.jobs.get(jobId);
+    if (
+      !job ||
+      job.status !== "running" ||
+      job.attempt !== attempt ||
+      (job.durableJobId && job.durableJobId !== durableJobId)
+    ) {
+      return false;
+    }
+    job.durableJobId = durableJobId;
+    return true;
+  }
+
+  async updateProgress(id: string, progress: JobProgress, now: Date) {
+    Object.assign(this.requireJob(id), progress, {
+      updatedAt: now.toISOString(),
+    });
+  }
+
+  async finish(
+    id: string,
+    status: "succeeded" | "completed-with-errors",
+    progress: JobProgress,
+    now: Date,
+  ) {
+    const job = this.requireJob(id);
+    if (job.status !== "running") return;
+    Object.assign(job, progress, {
+      status,
+      finishedAt: now.toISOString(),
+      updatedAt: now.toISOString(),
+    });
+  }
+
+  async retryOrFail(id: string, error: string, now: Date, retryAt?: Date) {
+    const job = this.requireJob(id);
+    if (job.status !== "running") return;
+    Object.assign(job, {
+      status:
+        retryAt && job.attempt < job.maxAttempts ? "retry-pending" : "failed",
+      error,
+      nextAttemptAt:
+        retryAt && job.attempt < job.maxAttempts
+          ? retryAt.toISOString()
+          : undefined,
+      finishedAt:
+        retryAt && job.attempt < job.maxAttempts
+          ? undefined
+          : now.toISOString(),
+      updatedAt: now.toISOString(),
+    });
+  }
+
+  async putArtifact(input: NewArtifact) {
+    const key = `${input.jobId}:${input.kind}`;
+    const now = new Date().toISOString();
+    const artifact: BulkDataArtifact = {
+      ...input,
+      id: this.artifacts.get(key)?.id || randomUUIDv7(),
+      createdAt: now,
+      expiresAt: input.expiresAt.toISOString(),
+    };
+    this.artifacts.set(key, artifact);
+    return { ...artifact };
+  }
+
+  async getArtifact(jobId: string, kind: BulkArtifactKind) {
+    const artifact = this.artifacts.get(`${jobId}:${kind}`);
+    return artifact ? { ...artifact } : undefined;
+  }
+
+  async listActiveArtifactRefs(now: Date) {
+    const timestamp = now.toISOString();
+    return [...this.artifacts.values()]
+      .filter(
+        (artifact) =>
+          Boolean(artifact.artifactRef) && artifact.expiresAt > timestamp,
+      )
+      .map((artifact) => artifact.artifactRef!);
+  }
+
+  async cleanupExpired(now: Date) {
+    const deleted: BulkDataArtifact[] = [];
+    for (const [key, artifact] of this.artifacts) {
+      const job = this.jobs.get(artifact.jobId);
+      if (
+        artifact.expiresAt <= now.toISOString() &&
+        job &&
+        ["succeeded", "completed-with-errors", "failed"].includes(job.status)
+      ) {
+        this.artifacts.delete(key);
+        deleted.push({ ...artifact });
+      }
+    }
+    return deleted;
+  }
+
+  private requireJob(id: string) {
+    const job = this.jobs.get(id);
+    if (!job) throw new Error(`批处理任务不存在：${id}`);
+    return job;
+  }
+}

package/template/apps/server/src/bulk-data/resource-handler.ts

@@ -0,0 +1,27 @@
+import type { RequestActor } from "../guards.js";
+
+export interface BulkResourceHandler {
+  /** V1 批处理只允许显式平台级资源，禁止把请求级 ScopeContext 丢进异步队列。 */
+  readonly scopeMode: "platform";
+  resource: string;
+  importRow(
+    value: unknown,
+    mode: "create" | "upsert",
+    actor: RequestActor,
+  ): Promise<void>;
+  exportRows(actor: RequestActor): Promise<unknown[]>;
+}
+
+export class BulkRowValidationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "BulkRowValidationError";
+  }
+}
+
+export class BulkRowImportError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "BulkRowImportError";
+  }
+}