create-windy 0.1.0

package/template/apps/server/src/data-governance/export/job-runtime.test.ts

@@ -0,0 +1,339 @@
+import { describe, expect, test } from "bun:test";
+import {
+  MemoryBlobStore,
+  type BlobStore,
+  type PutBlobInput,
+} from "@windy/storage";
+import { buildScopeContext } from "../../data-access/context.js";
+import { DurableHandlerRegistry } from "../../jobs/handler-registry.js";
+import { InMemoryDurableExecutionRepository } from "../../jobs/in-memory-execution.js";
+import { DurableJobService } from "../../jobs/service.js";
+import { DurableJobWorker } from "../../jobs/worker.js";
+import { createServerRuntime, type AuditEventWriter } from "../../runtime.js";
+import { GovernedExportJobRuntime } from "./job-runtime.js";
+import type { GovernedExportAuthorizationPort } from "./authorization-port.js";
+import { InMemoryGovernedExportRepository } from "./repository.js";
+import { captureGovernedExportScope } from "./scope-snapshot.js";
+import type { GovernedExportSource } from "./source.js";
+
+const base = new Date("2026-07-16T03:00:00.000Z");
+const canary = "sensitive-canary-13800138000";
+
+describe("governed export job runtime", () => {
+  test("payload 只保存 requestId，并分页流式生成带内部水印的 fenced artifact", async () => {
+    const fixture = await createFixture();
+    const job = await fixture.exports.submit(fixture.request);
+    expect(job.payload).toEqual({ requestId: fixture.request.id });
+
+    expect(await fixture.exports.runNext("export-worker")).toEqual({
+      status: "succeeded",
+      jobId: job.id,
+    });
+    const snapshot = await fixture.jobs.get(job.id);
+    const artifact = snapshot?.artifacts[0];
+    if (!artifact) throw new Error("治理导出 artifact 未发布");
+    const blob = await fixture.blobs.open(artifact.artifactRef);
+    if (!blob) throw new Error("治理导出 Blob 不存在");
+    const content = await new Response(blob.body).text();
+
+    expect(content).toContain(canary);
+    for (const field of [
+      fixture.request.requesterId,
+      fixture.request.scopeHash,
+      fixture.request.id,
+      job.id,
+      base.toISOString(),
+    ]) {
+      expect(content).toContain(field);
+    }
+    expect(snapshot?.progress).toMatchObject([
+      { completedUnits: 1 },
+      { completedUnits: 2 },
+    ]);
+    expect(JSON.stringify(snapshot?.progress)).not.toContain(canary);
+    expect(JSON.stringify(fixture.runtime.auditSink.list())).not.toContain(
+      canary,
+    );
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(({ action }) => action === "data.governed-export.generate"),
+    ).toEqual([
+      expect.objectContaining({
+        outcome: "attempted",
+        metadata: expect.objectContaining({ phase: "intent" }),
+      }),
+    ]);
+  });
+
+  test("关键审计失败时清理 Blob 且 Failure 只保留稳定错误", async () => {
+    const fixture = await createFixture({
+      async write() {
+        throw new Error(`database failed ${canary}`);
+      },
+    });
+    const job = await fixture.exports.submit(fixture.request);
+
+    expect(await fixture.exports.runNext("audit-failed-worker")).toMatchObject({
+      status: "failed",
+      jobId: job.id,
+    });
+    expect(fixture.blobs.size).toBe(0);
+    const snapshot = await fixture.jobs.get(job.id);
+    expect(snapshot?.artifacts).toEqual([]);
+    expect(snapshot?.failures[0]).toMatchObject({
+      code: "EXPORT_AUDIT_UNAVAILABLE",
+      message: "治理导出生成失败",
+    });
+    expect(JSON.stringify(snapshot)).not.toContain(canary);
+  });
+
+  test("生成意图审计恢复后重试只发布一份可下载 artifact", async () => {
+    let now = base;
+    let failOnce = true;
+    const fixture = await createFixture(
+      {
+        async write(event) {
+          if (failOnce && event.action === "data.governed-export.generate") {
+            failOnce = false;
+            throw new Error("audit temporarily unavailable");
+          }
+        },
+      },
+      new MemoryBlobStore(),
+      () => now,
+    );
+    const job = await fixture.exports.submit(fixture.request);
+
+    expect(await fixture.exports.runNext("retry-worker-1")).toMatchObject({
+      status: "failed",
+      jobId: job.id,
+    });
+    expect(fixture.blobs.size).toBe(0);
+    expect((await fixture.jobs.get(job.id))?.artifacts).toEqual([]);
+
+    now = new Date(base.getTime() + 1_000);
+    expect(await fixture.exports.runNext("retry-worker-2")).toMatchObject({
+      status: "succeeded",
+      jobId: job.id,
+    });
+    const artifacts = (await fixture.jobs.get(job.id))?.artifacts || [];
+    expect(artifacts).toHaveLength(1);
+    const downloadable = await fixture.blobs.open(artifacts[0]!.artifactRef);
+    if (!downloadable) throw new Error("重试后的治理导出不可下载");
+    expect(await new Response(downloadable.body).text()).toContain(canary);
+  });
+
+  test("执行时权限或范围失效会在查询与 Blob 写入前终止申请", async () => {
+    const fixture = await createFixture(
+      undefined,
+      new MemoryBlobStore(),
+      () => base,
+      {
+        async evaluate() {
+          return { allowed: false, reason: "scope-changed" };
+        },
+      },
+    );
+    const job = await fixture.exports.submit(fixture.request);
+
+    expect(await fixture.exports.runNext("revoked-worker")).toMatchObject({
+      status: "failed",
+      jobId: job.id,
+    });
+    expect(fixture.pageReads()).toBe(0);
+    expect(fixture.blobs.size).toBe(0);
+    expect(await fixture.requests.get(fixture.request.id)).toMatchObject({
+      status: "failed",
+    });
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(({ action }) => action === "data.governed-export.generate"),
+    ).toEqual([
+      expect.objectContaining({
+        outcome: "denied",
+        metadata: expect.objectContaining({
+          phase: "authorization",
+          reason: "scope-changed",
+        }),
+      }),
+    ]);
+  });
+
+  test("执行授权拒绝审计不可用时仍以申请终态 fail closed", async () => {
+    const fixture = await createFixture(
+      {
+        async write(event) {
+          if (event.outcome === "denied") throw new Error(`audit ${canary}`);
+        },
+      },
+      new MemoryBlobStore(),
+      () => base,
+      {
+        async evaluate() {
+          return { allowed: false, reason: "permission-revoked" };
+        },
+      },
+    );
+    const job = await fixture.exports.submit(fixture.request);
+
+    expect(await fixture.exports.runNext("denied-audit-worker")).toMatchObject({
+      status: "failed",
+      jobId: job.id,
+    });
+    expect(fixture.pageReads()).toBe(0);
+    expect(fixture.blobs.size).toBe(0);
+    expect(await fixture.requests.get(fixture.request.id)).toMatchObject({
+      status: "failed",
+    });
+    expect((await fixture.jobs.get(job.id))?.failures[0]?.code).toBe(
+      "EXPORT_AUTHORIZATION_AUDIT_UNAVAILABLE",
+    );
+    expect(JSON.stringify(await fixture.jobs.get(job.id))).not.toContain(
+      canary,
+    );
+  });
+
+  test("attempt 在 Blob 写入后过期时拒绝 stale publish 并清理对象", async () => {
+    let now = base;
+    const delegate = new MemoryBlobStore();
+    const blobs: BlobStore = {
+      async put(input: PutBlobInput) {
+        const stored = await delegate.put(input);
+        now = new Date(base.getTime() + 31_000);
+        return stored;
+      },
+      inspect: (ref) => delegate.inspect(ref),
+      open: (ref) => delegate.open(ref),
+      delete: (ref) => delegate.delete(ref),
+    };
+    const fixture = await createFixture(undefined, blobs, () => now);
+    const job = await fixture.exports.submit(fixture.request);
+
+    expect(await fixture.exports.runNext("stale-worker")).toEqual({
+      status: "lease-lost",
+      jobId: job.id,
+    });
+    expect(delegate.size).toBe(0);
+    expect((await fixture.jobs.get(job.id))?.artifacts).toEqual([]);
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(({ action }) => action === "data.governed-export.generate"),
+    ).toEqual([
+      expect.objectContaining({
+        outcome: "attempted",
+        metadata: expect.objectContaining({ phase: "intent" }),
+      }),
+    ]);
+  });
+
+  test("Blob 写入后收到取消请求时不发布 artifact 并清理对象", async () => {
+    const delegate = new MemoryBlobStore();
+    let requestCancel = async () => false;
+    const blobs: BlobStore = {
+      async put(input: PutBlobInput) {
+        const stored = await delegate.put(input);
+        await requestCancel();
+        return stored;
+      },
+      inspect: (ref) => delegate.inspect(ref),
+      open: (ref) => delegate.open(ref),
+      delete: (ref) => delegate.delete(ref),
+    };
+    const fixture = await createFixture(undefined, blobs);
+    const job = await fixture.exports.submit(fixture.request);
+    requestCancel = () => fixture.exports.cancel(job.id);
+
+    expect(await fixture.exports.runNext("cancel-worker")).toEqual({
+      status: "cancelled",
+      jobId: job.id,
+    });
+    expect(delegate.size).toBe(0);
+    expect((await fixture.jobs.get(job.id))?.artifacts).toEqual([]);
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(({ action }) => action === "data.governed-export.generate"),
+    ).toEqual([
+      expect.objectContaining({
+        outcome: "attempted",
+        metadata: expect.objectContaining({ phase: "intent" }),
+      }),
+    ]);
+  });
+});
+
+async function createFixture(
+  auditWriter?: AuditEventWriter,
+  blobStore: BlobStore = new MemoryBlobStore(),
+  now: () => Date = () => base,
+  authorization?: GovernedExportAuthorizationPort,
+) {
+  const runtime = createServerRuntime({}, { auditWriter });
+  const requests = new InMemoryGovernedExportRepository();
+  const durable = new InMemoryDurableExecutionRepository();
+  const jobs = new DurableJobService(durable, now);
+  const registry = new DurableHandlerRegistry();
+  const worker = new DurableJobWorker(durable, registry, { now });
+  let pageReads = 0;
+  const source: GovernedExportSource = {
+    resourceType: "system.user",
+    async *pages() {
+      pageReads += 1;
+      yield [{ id: "user-1", phone: canary }];
+      yield [{ id: "user-2", email: canary }];
+    },
+  };
+  const scope = captureGovernedExportScope(
+    buildScopeContext("auditor-1", "dept-1", [
+      {
+        id: "role-auditor",
+        code: "auditor",
+        dataScope: "department",
+        departmentIds: [],
+      },
+    ]),
+  );
+  const created = await requests.create(
+    {
+      id: "export-request-1",
+      resourceType: "system.user",
+      policyKey: "system.user.fields",
+      policyVersion: "1",
+      fieldKeys: ["email", "phone"],
+      requesterId: "auditor-1",
+      requesterName: "安全审计员",
+      idempotencyKey: "export-1",
+      requestFingerprint: "b".repeat(64),
+      reason: "合规复核",
+      scopeSnapshot: scope.snapshot,
+      scopeHash: scope.hash,
+      querySnapshot: { includeDeleted: false },
+      approvalRequired: false,
+      expiresAt: "2026-07-17T03:00:00.000Z",
+    },
+    now(),
+  );
+  return {
+    request: created.request,
+    exports: new GovernedExportJobRuntime(
+      requests,
+      jobs,
+      worker,
+      registry,
+      blobStore,
+      runtime.auditSink,
+      [source],
+      undefined,
+      authorization,
+      now,
+    ),
+    blobs: blobStore as MemoryBlobStore,
+    jobs,
+    pageReads: () => pageReads,
+    requests,
+    runtime,
+  };
+}

package/template/apps/server/src/data-governance/export/job-runtime.ts

@@ -0,0 +1,242 @@
+import type { BlobStore } from "@windy/storage";
+import type { createServerRuntime } from "../../runtime.js";
+import { DurableHandlerError } from "../../jobs/handler-registry.js";
+import type { DurableHandlerRegistry } from "../../jobs/handler-registry.js";
+import type { DurableJobService } from "../../jobs/service.js";
+import type { DurableJobWorker } from "../../jobs/worker.js";
+import type { GovernedExportRepository } from "./repository.js";
+import type { GovernedExportSource } from "./source.js";
+import { createWatermarkedNdjsonStream } from "./ndjson-stream.js";
+import type { GovernedExportRequest } from "./types.js";
+import {
+  allowGovernedExportExecution,
+  type GovernedExportAuthorizationDecision,
+  type GovernedExportAuthorizationPort,
+} from "./authorization-port.js";
+
+const handlerKey = "system.governed-export.generate";
+const queue = "system.governed-export";
+
+export class GovernedExportJobRuntime {
+  constructor(
+    private readonly requests: GovernedExportRepository,
+    private readonly jobs: DurableJobService,
+    private readonly worker: DurableJobWorker,
+    registry: DurableHandlerRegistry,
+    private readonly blobStore: BlobStore,
+    private readonly audit: ReturnType<typeof createServerRuntime>["auditSink"],
+    private readonly sources: readonly GovernedExportSource[],
+    private readonly approvalSatisfied: (
+      request: GovernedExportRequest,
+    ) => Promise<boolean> = async (request) => !request.approvalRequired,
+    private readonly authorization: GovernedExportAuthorizationPort = allowGovernedExportExecution,
+    private readonly now: () => Date = () => new Date(),
+  ) {
+    registry.register({
+      key: handlerKey,
+      payload: { parse: parsePayload },
+      execute: (payload, context) => this.generate(payload.requestId, context),
+    });
+  }
+
+  async submit(request: GovernedExportRequest) {
+    const result = await this.jobs.submit({
+      handlerKey,
+      queue,
+      requestedBy: request.requesterId,
+      idempotencyKey: request.id,
+      payload: { requestId: request.id },
+      maxAttempts: 3,
+      timeoutSeconds: 900,
+    });
+    await this.requests.update(request.id, {
+      durableJobId: result.job.id,
+      status: "queued",
+      updatedAt: this.now().toISOString(),
+    });
+    return result.job;
+  }
+
+  runNext(workerId = `server-${process.pid}-governed-export`) {
+    return this.worker.runNext(queue, workerId);
+  }
+
+  cancel(jobId: string) {
+    return this.worker.cancel(jobId);
+  }
+
+  private async generate(
+    requestId: string,
+    context: Parameters<
+      NonNullable<ReturnType<DurableHandlerRegistry["get"]>>["execute"]
+    >[1],
+  ) {
+    const request = await this.requests.get(requestId);
+    if (!request) throw safeFailure("EXPORT_REQUEST_NOT_FOUND", false);
+    if (["cancelled", "failed", "rejected"].includes(request.status)) {
+      throw safeFailure("EXPORT_REQUEST_TERMINAL", false);
+    }
+    if (request.expiresAt <= this.now().toISOString()) {
+      throw safeFailure("EXPORT_REQUEST_EXPIRED", false);
+    }
+    if (!(await this.approvalSatisfied(request))) {
+      throw safeFailure("EXPORT_APPROVAL_REQUIRED", false);
+    }
+    const authorization = await this.authorization.evaluate(request);
+    if (!authorization.allowed) {
+      await this.denyAuthorization(request, authorization);
+    }
+    const source = this.sources.find(
+      ({ resourceType }) => resourceType === request.resourceType,
+    );
+    if (!source) throw safeFailure("EXPORT_SOURCE_NOT_REGISTERED", false);
+    await this.requests.update(request.id, {
+      status: "running",
+      updatedAt: this.now().toISOString(),
+    });
+    await this.writeArtifact(request, source, context);
+  }
+
+  private async denyAuthorization(
+    request: GovernedExportRequest,
+    decision: Extract<GovernedExportAuthorizationDecision, { allowed: false }>,
+  ): Promise<never> {
+    await this.requests.update(request.id, {
+      status: "failed",
+      updatedAt: this.now().toISOString(),
+    });
+    try {
+      await this.audit.recordCriticalEvent(
+        authorizationDeniedEvent(request, decision, this.now().toISOString()),
+      );
+    } catch {
+      throw safeFailure("EXPORT_AUTHORIZATION_AUDIT_UNAVAILABLE", false);
+    }
+    throw safeFailure("EXPORT_AUTHORIZATION_REVOKED", false);
+  }
+
+  private async writeArtifact(
+    request: GovernedExportRequest,
+    source: GovernedExportSource,
+    context: Parameters<
+      NonNullable<ReturnType<DurableHandlerRegistry["get"]>>["execute"]
+    >[1],
+  ) {
+    const generatedAt = this.now().toISOString();
+    try {
+      await this.audit.recordCriticalEvent(
+        generationIntentEvent(request, context.jobId, generatedAt),
+      );
+    } catch {
+      throw safeFailure("EXPORT_AUDIT_UNAVAILABLE", true);
+    }
+    let stored;
+    try {
+      stored = await this.blobStore.put({
+        body: createWatermarkedNdjsonStream({
+          request,
+          source,
+          signal: context.signal,
+          watermark: {
+            actorId: request.requesterId,
+            generatedAt,
+            scopeHash: request.scopeHash,
+            jobId: context.jobId,
+            requestId: request.id,
+          },
+          pageCompleted: async (completed) => {
+            await context.heartbeat();
+            await context.progress({ completedUnits: completed });
+          },
+        }),
+      });
+    } catch {
+      throw safeFailure("EXPORT_STREAM_FAILED", true);
+    }
+    try {
+      await context.publishArtifact({
+        kind: "governed-export",
+        artifactRef: stored.ref,
+        filename: "governed-users.ndjson",
+        contentType: "application/x-ndjson",
+        byteSize: stored.byteSize,
+        sha256: stored.contentHash.value,
+        ownerId: request.requesterId,
+        expiresAt: request.expiresAt,
+      });
+    } catch (error) {
+      await this.blobStore.delete(stored.ref);
+      throw error;
+    }
+  }
+}
+
+function parsePayload(value: unknown): { requestId: string } {
+  const payload = value as Record<string, unknown>;
+  if (
+    !payload ||
+    Object.keys(payload).length !== 1 ||
+    typeof payload.requestId !== "string" ||
+    !payload.requestId.trim()
+  ) {
+    throw safeFailure("EXPORT_PAYLOAD_INVALID", false);
+  }
+  return { requestId: payload.requestId };
+}
+
+function safeFailure(code: string, retryable: boolean) {
+  return new DurableHandlerError(code, "治理导出生成失败", retryable);
+}
+
+function generationIntentEvent(
+  request: GovernedExportRequest,
+  jobId: string,
+  generatedAt: string,
+) {
+  return {
+    id: Bun.randomUUIDv7(),
+    action: "data.governed-export.generate" as const,
+    outcome: "attempted" as const,
+    actor: {
+      id: request.requesterId,
+      type: "user" as const,
+      name: request.requesterName,
+    },
+    target: { type: request.resourceType, id: request.id },
+    requestId: request.id,
+    metadata: {
+      policyKey: request.policyKey,
+      policyVersion: request.policyVersion,
+      scopeHash: request.scopeHash,
+      jobId,
+      phase: "intent",
+    },
+    occurredAt: generatedAt,
+  };
+}
+
+function authorizationDeniedEvent(
+  request: GovernedExportRequest,
+  decision: Extract<GovernedExportAuthorizationDecision, { allowed: false }>,
+  occurredAt: string,
+) {
+  return {
+    id: Bun.randomUUIDv7(),
+    action: "data.governed-export.generate" as const,
+    outcome: "denied" as const,
+    actor: {
+      id: request.requesterId,
+      type: "user" as const,
+      name: request.requesterName,
+    },
+    target: { type: request.resourceType, id: request.id },
+    requestId: request.id,
+    metadata: {
+      phase: "authorization",
+      reason: decision.reason,
+      requestedScopeHash: request.scopeHash,
+      currentScopeHash: decision.currentScopeHash,
+    },
+    occurredAt,
+  };
+}

package/template/apps/server/src/data-governance/export/ndjson-stream.ts

@@ -0,0 +1,62 @@
+import type { GovernedExportSource } from "./source.js";
+import type { GovernedExportRequest } from "./types.js";
+
+export interface ExportWatermark {
+  actorId: string;
+  generatedAt: string;
+  scopeHash: string;
+  jobId: string;
+  requestId: string;
+}
+
+export function createWatermarkedNdjsonStream(input: {
+  request: GovernedExportRequest;
+  source: GovernedExportSource;
+  watermark: ExportWatermark;
+  signal: AbortSignal;
+  pageCompleted(completed: number): Promise<void>;
+}) {
+  const iterator = lines(input)[Symbol.asyncIterator]();
+  const encoder = new TextEncoder();
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const result = await iterator.next();
+        if (result.done) controller.close();
+        else
+          controller.enqueue(
+            encoder.encode(`${JSON.stringify(result.value)}\n`),
+          );
+      } catch (error) {
+        controller.error(error);
+      }
+    },
+    async cancel() {
+      await iterator.return?.();
+    },
+  });
+}
+
+async function* lines(input: {
+  request: GovernedExportRequest;
+  source: GovernedExportSource;
+  watermark: ExportWatermark;
+  signal: AbortSignal;
+  pageCompleted(completed: number): Promise<void>;
+}) {
+  yield {
+    type: "watermark",
+    policyKey: input.request.policyKey,
+    policyVersion: input.request.policyVersion,
+    watermark: input.watermark,
+  };
+  let completed = 0;
+  for await (const page of input.source.pages(input.request)) {
+    if (input.signal.aborted) throw new Error("EXPORT_ABORTED");
+    for (const record of page) {
+      yield { type: "record", watermark: input.watermark, data: record };
+      completed += 1;
+    }
+    await input.pageCompleted(completed);
+  }
+}