create-windy 0.1.0

package/template/apps/server/src/data-governance/export/postgres-api.integration.test.ts

@@ -0,0 +1,298 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { utimes } from "node:fs/promises";
+import { join } from "node:path";
+import { LocalFileBlobMaintenance } from "@windy/storage";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { Pool } from "pg";
+import { DrizzleAuditLogWriter } from "../../audit/drizzle-log-store.js";
+import { collectActiveBlobRefs } from "../../storage/blob-reference-registry.js";
+import {
+  assertNoCanaries,
+  createExport,
+  deniedBody,
+  json,
+  readDatabaseSurfaces,
+  required,
+} from "./postgres-api-assertions.js";
+import {
+  apiRequest,
+  approveExport,
+  createGovernanceStack,
+  createPostgresApiFixture,
+  outsideCanaries,
+  ownerCanaries,
+} from "./postgres-api-fixture.js";
+
+const databaseUrl = process.env.GOVERNANCE_TEST_DATABASE_URL;
+const describePostgres = databaseUrl ? describe : describe.skip;
+
+describePostgres("敏感数据治理真实 PostgreSQL API", () => {
+  let pool: Pool;
+  let fixture: Awaited<ReturnType<typeof createPostgresApiFixture>>;
+
+  beforeAll(async () => {
+    pool = new Pool({ connectionString: databaseUrl });
+    fixture = await createPostgresApiFixture(
+      pool,
+      new DrizzleAuditLogWriter(drizzle(pool)),
+    );
+  });
+
+  afterAll(async () => {
+    await fixture?.close();
+    await pool?.end();
+  });
+
+  test("真实 HTTP、审批、重启和恢复链全程不可绕过 Scope 或泄漏 Canary", async () => {
+    const errors: string[] = [];
+    const masked = await apiRequest(
+      fixture.app,
+      "owner",
+      "GET",
+      "/system/users/user_scope_owner",
+    );
+    const maskedBody = await masked.text();
+    expect(masked.status).toBe(200);
+    expect(maskedBody).toContain("138****8071");
+    assertNoCanaries(maskedBody);
+
+    const searched = await apiRequest(
+      fixture.app,
+      "owner",
+      "GET",
+      `/system/users?q=${encodeURIComponent(ownerCanaries[0] ?? "")}`,
+    );
+    const searchedBody = await searched.text();
+    expect(searched.status).toBe(200);
+    assertNoCanaries(searchedBody);
+
+    errors.push(
+      await deniedBody(
+        apiRequest(fixture.app, "owner", "POST", "/system/users/exports", {
+          idempotencyKey: "scope-spoof",
+          reason: "验证客户端范围伪造被拒绝",
+          query: {},
+          scope: "all",
+        }),
+        400,
+      ),
+      await deniedBody(
+        apiRequest(fixture.app, "denied", "POST", "/system/users/exports", {
+          idempotencyKey: "missing-export-permission",
+          reason: "验证缺少权限时拒绝申请",
+          query: {},
+        }),
+        403,
+      ),
+      await deniedBody(
+        apiRequest(
+          fixture.app,
+          "owner",
+          "POST",
+          "/system/users/plaintext/batch",
+          { ids: ["user_scope_owner", "user_scope_outside"] },
+        ),
+        403,
+      ),
+    );
+
+    const created = await createExport(fixture.app, "owner", "restart-export");
+    const replayed = await createExport(fixture.app, "owner", "restart-export");
+    expect(replayed.id).toBe(created.id);
+    errors.push(
+      await deniedBody(
+        apiRequest(fixture.app, "owner", "POST", "/system/users/exports", {
+          idempotencyKey: "restart-export",
+          reason: "同一幂等键改为不同导出用途",
+          query: { includeDeleted: false },
+        }),
+        409,
+      ),
+    );
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(({ action }) => action === "data.governed-export.request")
+        .map(({ target }) => target?.id),
+    ).toEqual([created.id, created.id]);
+    expect(
+      (
+        await pool.query<{ count: string }>(
+          `select count(*)::text as count from governed_export_requests
+           where requester_id = $1 and idempotency_key = $2`,
+          [fixture.actors.owner.id, "restart-export"],
+        )
+      ).rows[0]?.count,
+    ).toBe("1");
+    await approveExport(
+      fixture.workflow,
+      required(created.approvalId),
+      fixture.actors.reviewer,
+    );
+    const queued = await json(
+      apiRequest(
+        fixture.app,
+        "owner",
+        "GET",
+        `/system/users/exports/${created.id}`,
+      ),
+    );
+    expect(queued.status).toBe("queued");
+
+    const active = createGovernanceStack({
+      db: fixture.db,
+      runtime: fixture.runtime,
+      blobStore: fixture.blobStore,
+      clock: fixture.clock,
+      logs: fixture.logs,
+    });
+    expect(
+      await active.exports.runNext("restarted-governance-worker"),
+    ).toMatchObject({
+      status: "succeeded",
+    });
+    expect(
+      await json(
+        apiRequest(
+          active.app,
+          "owner",
+          "GET",
+          `/system/users/exports/${created.id}`,
+        ),
+      ),
+    ).toMatchObject({ status: "succeeded" });
+
+    for (const token of ["outside", "ownerChanged"]) {
+      errors.push(
+        await deniedBody(
+          apiRequest(
+            active.app,
+            token,
+            "GET",
+            `/system/users/exports/${created.id}/download`,
+          ),
+          403,
+        ),
+      );
+    }
+    const artifacts: string[] = [];
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      const response = await apiRequest(
+        active.app,
+        "owner",
+        "GET",
+        `/system/users/exports/${created.id}/download`,
+      );
+      expect(response.status).toBe(200);
+      artifacts.push(await response.text());
+    }
+    expect(artifacts[0]).toContain(ownerCanaries[0]);
+    expect(artifacts[0]).toContain(ownerCanaries[1]);
+    expect(artifacts[0]).toContain(fixture.actors.owner.id);
+    expect(artifacts[0]).toContain(created.id);
+    for (const canary of outsideCanaries)
+      expect(artifacts[0]).not.toContain(canary);
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .filter(({ action }) => action === "data.governed-export.download"),
+    ).toHaveLength(4);
+
+    const cancelled = await createExport(active.app, "owner", "cancel-export");
+    await approveExport(
+      active.workflow,
+      required(cancelled.approvalId),
+      fixture.actors.reviewer,
+    );
+    await apiRequest(
+      active.app,
+      "owner",
+      "GET",
+      `/system/users/exports/${cancelled.id}`,
+    );
+    const firstCancel = await json(
+      apiRequest(
+        active.app,
+        "owner",
+        "POST",
+        `/system/users/exports/${cancelled.id}/cancel`,
+        { reason: "申请人撤销本次导出" },
+      ),
+    );
+    const repeatCancel = await json(
+      apiRequest(
+        active.app,
+        "owner",
+        "POST",
+        `/system/users/exports/${cancelled.id}/cancel`,
+        { reason: "重复撤销不产生新副作用" },
+      ),
+    );
+    expect([firstCancel.status, repeatCancel.status]).toEqual([
+      "cancelled",
+      "cancelled",
+    ]);
+
+    const persistedRequest = await active.requests.get(created.id);
+    const durableJobId = required(persistedRequest?.durableJobId);
+    const artifact = (await active.facts.get(durableJobId))?.artifacts[0];
+    if (!artifact) throw new Error("治理导出产物未持久化");
+    const blobDirectory = join(
+      fixture.storageRoot,
+      "blobs",
+      artifact.artifactRef.slice(5, 7),
+    );
+    const agedAt = new Date(fixture.clock.current.getTime() - 61 * 60_000);
+    await Promise.all(
+      ["blob", "json"].map((extension) =>
+        utimes(
+          join(blobDirectory, `${artifact.artifactRef}.${extension}`),
+          agedAt,
+          agedAt,
+        ),
+      ),
+    );
+    const maintenance = new LocalFileBlobMaintenance(fixture.storageRoot);
+    const activeRefs = await collectActiveBlobRefs(
+      [active.facts],
+      fixture.clock.current,
+    );
+    expect(
+      await maintenance.cleanupOrphans({
+        referencedRefs: activeRefs,
+        olderThan: new Date(fixture.clock.current.getTime() - 60 * 60_000),
+        limit: 10,
+      }),
+    ).toBe(0);
+    expect(await fixture.blobStore.inspect(artifact.artifactRef)).toBeDefined();
+
+    fixture.clock.current = new Date(
+      new Date(required(artifact.expiresAt)).getTime() + 1,
+    );
+    const expiredRefs = await collectActiveBlobRefs(
+      [active.facts],
+      fixture.clock.current,
+    );
+    expect(expiredRefs.has(artifact.artifactRef)).toBe(false);
+    expect(
+      await maintenance.cleanupOrphans({
+        referencedRefs: expiredRefs,
+        olderThan: fixture.clock.current,
+        limit: 10,
+      }),
+    ).toBe(1);
+    expect(
+      await fixture.blobStore.inspect(artifact.artifactRef),
+    ).toBeUndefined();
+
+    const databaseSurfaces = await readDatabaseSurfaces(pool);
+    for (const surface of [
+      JSON.stringify(errors),
+      JSON.stringify(fixture.logs),
+      JSON.stringify(fixture.runtime.auditSink.list()),
+      ...databaseSurfaces,
+    ]) {
+      assertNoCanaries(surface);
+    }
+  });
+});

package/template/apps/server/src/data-governance/export/postgres-recovery.integration.test.ts

@@ -0,0 +1,344 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { join } from "node:path";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { Pool } from "pg";
+import { FileAuditRecoveryQueue } from "../../audit/recovery-queue.js";
+import { ReliableAuditWriter } from "../../audit/reliable-writer.js";
+import { DrizzleAuditLogWriter } from "../../audit/drizzle-log-store.js";
+import { createServerRuntime, type AuditEventWriter } from "../../runtime.js";
+import {
+  assertNoCanaries,
+  createExport,
+  deniedBody,
+  json,
+  readDatabaseSurfaces,
+  required,
+} from "./postgres-api-assertions.js";
+import {
+  apiRequest,
+  approveExport,
+  createGovernanceStack,
+  createPostgresApiFixture,
+  ownerCanaries,
+} from "./postgres-api-fixture.js";
+import { captureGovernedExportScope } from "./scope-snapshot.js";
+import type { GovernedExportSource } from "./source.js";
+
+const databaseUrl = process.env.GOVERNANCE_TEST_DATABASE_URL;
+const describePostgres = databaseUrl ? describe : describe.skip;
+
+describePostgres("敏感数据治理恢复与泄漏防线", () => {
+  let pool: Pool;
+  let fixture: Awaited<ReturnType<typeof createPostgresApiFixture>>;
+
+  beforeAll(async () => {
+    pool = new Pool({ connectionString: databaseUrl });
+    fixture = await createPostgresApiFixture(
+      pool,
+      new DrizzleAuditLogWriter(drizzle(pool)),
+    );
+  });
+
+  afterAll(async () => {
+    await fixture?.close();
+    await pool?.end();
+  });
+
+  test("失败重试、审计恢复和双故障均保持 fail closed", async () => {
+    const created = await createExport(fixture.app, "owner", "fault-export");
+    await approveExport(
+      fixture.workflow,
+      required(created.approvalId),
+      fixture.actors.reviewer,
+    );
+    await apiRequest(
+      fixture.app,
+      "owner",
+      "GET",
+      `/system/users/exports/${created.id}`,
+    );
+    expect(await fixture.exports.runNext("fault-export-worker")).toMatchObject({
+      status: "succeeded",
+    });
+    expect(
+      await json(
+        apiRequest(
+          fixture.app,
+          "owner",
+          "GET",
+          `/system/users/exports/${created.id}`,
+        ),
+      ),
+    ).toMatchObject({ status: "succeeded" });
+
+    await exerciseFailureRetry(fixture);
+    await exerciseAuthorizationRevocation(fixture);
+    const queuedAudit = await exerciseRecoveryQueue(fixture);
+    const beforeOpen = fixture.openCount();
+    const errors = await exerciseDoubleFault(fixture, created.id);
+    expect(fixture.openCount()).toBe(beforeOpen);
+
+    for (const surface of [
+      JSON.stringify(errors),
+      queuedAudit,
+      JSON.stringify(fixture.logs),
+      JSON.stringify(fixture.runtime.auditSink.list()),
+      ...(await readDatabaseSurfaces(pool)),
+    ]) {
+      assertNoCanaries(surface);
+    }
+  });
+});
+
+async function exerciseAuthorizationRevocation(
+  fixture: Awaited<ReturnType<typeof createPostgresApiFixture>>,
+) {
+  let pageReads = 0;
+  const stack = createGovernanceStack({
+    db: fixture.db,
+    runtime: fixture.runtime,
+    blobStore: fixture.blobStore,
+    clock: fixture.clock,
+    sources: [
+      {
+        resourceType: "system.user",
+        async *pages() {
+          pageReads += 1;
+          yield [{ id: "forbidden", phone: ownerCanaries[1] }];
+        },
+      },
+    ],
+  });
+  const created = await createExport(
+    stack.app,
+    "owner",
+    "authorization-revoked",
+  );
+  await approveExport(
+    stack.workflow,
+    required(created.approvalId),
+    fixture.actors.reviewer,
+  );
+  await apiRequest(
+    stack.app,
+    "owner",
+    "GET",
+    `/system/users/exports/${created.id}`,
+  );
+  const role = await stack.system.roles.get("role_user_scope_owner");
+  if (!role) throw new Error("治理导出测试角色不存在");
+  await stack.system.roles.update(
+    role.id,
+    {
+      permissionKeys: role.permissionKeys.filter(
+        (key) => key !== "system.user.export",
+      ),
+    },
+    fixture.actors.bootstrap,
+  );
+  try {
+    expect(await stack.exports.runNext("revoked-pg-worker")).toMatchObject({
+      status: "failed",
+    });
+    expect(pageReads).toBe(0);
+    expect(await stack.requests.get(created.id)).toMatchObject({
+      status: "failed",
+    });
+    const request = await stack.requests.get(created.id);
+    const job = await stack.facts.get(required(request?.durableJobId));
+    expect(job?.artifacts).toEqual([]);
+    expect(
+      fixture.runtime.auditSink
+        .list()
+        .find(
+          ({ action, target, outcome }) =>
+            action === "data.governed-export.generate" &&
+            target?.id === created.id &&
+            outcome === "denied",
+        ),
+    ).toMatchObject({
+      metadata: {
+        phase: "authorization",
+        reason: "permission-revoked",
+      },
+    });
+  } finally {
+    await stack.system.roles.update(
+      role.id,
+      { permissionKeys: role.permissionKeys },
+      fixture.actors.bootstrap,
+    );
+  }
+  const restarted = createGovernanceStack({
+    db: fixture.db,
+    runtime: fixture.runtime,
+    blobStore: fixture.blobStore,
+    clock: fixture.clock,
+  });
+  expect(await restarted.exports.runNext("restored-pg-worker")).toEqual({
+    status: "idle",
+  });
+  expect(await restarted.requests.get(created.id)).toMatchObject({
+    status: "failed",
+  });
+}
+
+async function exerciseFailureRetry(
+  fixture: Awaited<ReturnType<typeof createPostgresApiFixture>>,
+) {
+  const scope = captureGovernedExportScope(fixture.actors.owner.scopeContext);
+  const request = await fixture.requests.create(
+    {
+      id: Bun.randomUUIDv7(),
+      resourceType: "canary.failure",
+      policyKey: "system.user.fields",
+      policyVersion: "1",
+      fieldKeys: ["phone"],
+      requesterId: fixture.actors.owner.id,
+      requesterName: "治理测试员",
+      idempotencyKey: "failure-retry",
+      requestFingerprint: "b".repeat(64),
+      reason: "验证失败信息脱敏与重试",
+      scopeSnapshot: scope.snapshot,
+      scopeHash: scope.hash,
+      querySnapshot: { includeDeleted: false },
+      approvalRequired: false,
+      expiresAt: new Date(
+        fixture.clock.current.getTime() + 86_400_000,
+      ).toISOString(),
+    },
+    fixture.clock.current,
+  );
+  const failing: GovernedExportSource = {
+    resourceType: "canary.failure",
+    async *pages() {
+      yield [];
+      throw new Error(`source failed ${ownerCanaries[1]}`);
+    },
+  };
+  let jobId = "";
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const restarted = createGovernanceStack({
+      db: fixture.db,
+      runtime: fixture.runtime,
+      blobStore: fixture.blobStore,
+      clock: fixture.clock,
+      sources: [failing],
+    });
+    if (attempt === 0)
+      jobId = (await restarted.exports.submit(request.request)).id;
+    expect(
+      await restarted.exports.runNext(`failure-worker-${attempt}`),
+    ).toMatchObject({ status: "failed" });
+    fixture.clock.current = new Date(fixture.clock.current.getTime() + 1_001);
+  }
+  const snapshot = await fixture.jobs.get(required(jobId));
+  expect(snapshot?.job.status).toBe("failed");
+  expect(snapshot?.failures).toHaveLength(3);
+  assertNoCanaries(JSON.stringify(snapshot));
+}
+
+async function exerciseRecoveryQueue(
+  fixture: Awaited<ReturnType<typeof createPostgresApiFixture>>,
+) {
+  const primary = new ToggleWriter(new DrizzleAuditLogWriter(fixture.db));
+  primary.failing = true;
+  const queue = new FileAuditRecoveryQueue(
+    join(fixture.storageRoot, "audit-recovery.json"),
+  );
+  const reliable = reliableWriter(primary, queue);
+  const runtime = createServerRuntime(
+    {},
+    { actorResolver: fixture.actorResolver, auditWriter: reliable },
+  );
+  const stack = createGovernanceStack({
+    db: fixture.db,
+    runtime,
+    blobStore: fixture.blobStore,
+    clock: fixture.clock,
+  });
+  const response = await apiRequest(
+    stack.app,
+    "owner",
+    "GET",
+    "/system/users/user_scope_owner/plaintext",
+  );
+  expect(response.status).toBe(200);
+  expect(await response.text()).toContain(ownerCanaries[0]);
+  const queued = JSON.stringify(await queue.read());
+  assertNoCanaries(queued);
+  primary.failing = false;
+  await reliable.recover();
+  expect(await queue.read()).toEqual([]);
+  return queued;
+}
+
+async function exerciseDoubleFault(
+  fixture: Awaited<ReturnType<typeof createPostgresApiFixture>>,
+  exportId: string,
+) {
+  const reliable = reliableWriter(
+    {
+      async write() {
+        throw new Error(`primary ${ownerCanaries[0]}`);
+      },
+    },
+    {
+      async read() {
+        throw new Error("queue unavailable");
+      },
+      async replace() {
+        throw new Error("queue unavailable");
+      },
+    },
+  );
+  const runtime = createServerRuntime(
+    {},
+    { actorResolver: fixture.actorResolver, auditWriter: reliable },
+  );
+  const stack = createGovernanceStack({
+    db: fixture.db,
+    runtime,
+    blobStore: fixture.blobStore,
+    clock: fixture.clock,
+  });
+  return Promise.all([
+    deniedBody(
+      apiRequest(
+        stack.app,
+        "owner",
+        "GET",
+        "/system/users/user_scope_owner/plaintext",
+      ),
+      500,
+    ),
+    deniedBody(
+      apiRequest(
+        stack.app,
+        "owner",
+        "GET",
+        `/system/users/exports/${exportId}/download`,
+      ),
+      500,
+    ),
+  ]);
+}
+
+function reliableWriter(
+  primary: AuditEventWriter,
+  queue: ConstructorParameters<typeof ReliableAuditWriter>[1],
+) {
+  return new ReliableAuditWriter(primary, queue, {
+    maxAttempts: 1,
+    delay: async () => undefined,
+  });
+}
+
+class ToggleWriter implements AuditEventWriter {
+  failing = false;
+  constructor(private readonly delegate: AuditEventWriter) {}
+  write(...args: Parameters<AuditEventWriter["write"]>) {
+    if (this.failing) throw new Error("primary unavailable");
+    return this.delegate.write(...args);
+  }
+}

package/template/apps/server/src/data-governance/export/repository.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, test } from "bun:test";
+import { InMemoryGovernedExportRepository } from "./repository.js";
+
+const now = new Date("2026-07-16T01:00:00.000Z");
+
+describe("governed export repository", () => {
+  test("申请按发起人、资源和幂等键复用并持久化服务器范围快照", async () => {
+    const repository = new InMemoryGovernedExportRepository();
+    const input = {
+      id: "export-1",
+      resourceType: "system.user",
+      policyKey: "system.user.fields",
+      policyVersion: "1",
+      fieldKeys: ["email", "phone"],
+      requesterId: "user-1",
+      requesterName: "审计员",
+      idempotencyKey: "request-1",
+      requestFingerprint: "b".repeat(64),
+      reason: "合规复核",
+      scopeSnapshot: {
+        tenantId: "single-tenant" as const,
+        actorId: "user-1",
+        grants: [{ scope: "self" }],
+        roleIds: ["role-auditor"],
+      },
+      scopeHash: "a".repeat(64),
+      querySnapshot: { includeDeleted: false },
+      approvalRequired: true,
+      expiresAt: "2026-07-17T01:00:00.000Z",
+    } satisfies Parameters<typeof repository.create>[0];
+
+    const created = await repository.create(input, now);
+    const repeated = await repository.create({ ...input, id: "export-2" }, now);
+
+    expect(created).toMatchObject({ created: true });
+    expect(created.request.status).toBe("pending-approval");
+    expect(repeated).toMatchObject({ created: false });
+    expect(repeated.request.id).toBe("export-1");
+    expect(repeated.request.scopeSnapshot.grants).toEqual([{ scope: "self" }]);
+
+    const conflict = await repository.create(
+      {
+        ...input,
+        id: "export-3",
+        requestFingerprint: "c".repeat(64),
+      },
+      now,
+    );
+    expect(conflict).toMatchObject({
+      created: false,
+      fingerprintConflict: true,
+      request: { id: "export-1" },
+    });
+  });
+});