create-windy 0.3.2 → 0.3.5

package/template/apps/server/src/data-governance/export/scope-snapshot.ts

@@ -1,27 +1,15 @@
-import { createHash } from "node:crypto";
 import type { GovernedExportScopeSnapshot } from "@southwind-ai/database";
-import { snapshotServerScopeContext } from "../../data-access/context.js";
+import {
+  captureAuthorizationScope,
+  hashAuthorizationScope,
+} from "../../data-access/scope-snapshot.js";
 
 export function captureGovernedExportScope(value: unknown) {
-  const snapshot = snapshotServerScopeContext(value);
-  return { snapshot, hash: hashGovernedExportScope(snapshot) };
+  return captureAuthorizationScope(value);
 }
 
 export function hashGovernedExportScope(
   snapshot: GovernedExportScopeSnapshot,
 ): string {
-  return createHash("sha256").update(stableJson(snapshot)).digest("hex");
-}
-
-function stableJson(value: unknown): string {
-  if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
-  if (value && typeof value === "object") {
-    const entries = Object.entries(value as Record<string, unknown>)
-      .filter(([, item]) => item !== undefined)
-      .sort(([left], [right]) => left.localeCompare(right));
-    return `{${entries
-      .map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`)
-      .join(",")}}`;
-  }
-  return JSON.stringify(value);
+  return hashAuthorizationScope(snapshot);
 }

package/template/apps/server/src/data-governance/export/service-support.ts

@@ -0,0 +1,188 @@
+import type { GovernedExportQuerySnapshot } from "@southwind-ai/database";
+import type { AuditEvent } from "@southwind-ai/shared";
+import { auditActor, type RequestGuardContext } from "../../guards.js";
+import type { DurableArtifactService } from "../../jobs/artifact-service.js";
+import {
+  legacyGovernedExportRendererSnapshot,
+  type GovernedExportRequest,
+  type GovernedExportStatus,
+} from "./types.js";
+
+export interface CreateGovernedExportInput {
+  idempotencyKey: string;
+  reason: string;
+  query: GovernedExportQuerySnapshot;
+  formatKey?: string;
+}
+
+export interface GovernedExportPolicy {
+  approvalRequired: boolean;
+  retentionSeconds: number;
+}
+
+export interface GovernedExportTarget {
+  readonly resourceType: string;
+  readonly policyKey: string;
+  readonly policyVersion: string;
+  readonly fieldKeys: readonly string[];
+  readonly approvalRequired?: boolean;
+  readonly retentionSeconds?: number;
+  readonly renderers?: readonly GovernedExportRendererTarget[];
+}
+
+export interface GovernedExportRendererTarget {
+  readonly formatKey: string;
+  readonly templateKey: string;
+  readonly templateVersion: string;
+  readonly inputSchemaKey: string;
+  readonly contentType: string;
+  readonly filenameExtension: string;
+  readonly maxRecords: number;
+  readonly maxOutputBytes: number;
+  readonly timeoutSeconds: number;
+}
+
+export const legacyNdjsonRenderer: GovernedExportRendererTarget = {
+  ...legacyGovernedExportRendererSnapshot,
+};
+
+export const governedUserTarget: GovernedExportTarget = {
+  resourceType: "system.user",
+  policyKey: "system.user.fields",
+  policyVersion: "1",
+  fieldKeys: ["email", "phone"],
+};
+
+export class GovernedExportServiceError extends Error {
+  constructor(
+    readonly code: string,
+    readonly status: number,
+  ) {
+    super(code);
+    this.name = "GovernedExportServiceError";
+  }
+}
+
+export function validateCreate(input: CreateGovernedExportInput) {
+  if (!input.idempotencyKey.trim() || input.idempotencyKey.length > 120) {
+    throw new GovernedExportServiceError("EXPORT_IDEMPOTENCY_KEY_INVALID", 400);
+  }
+  if (input.reason.trim().length < 5 || input.reason.length > 200) {
+    throw new GovernedExportServiceError("EXPORT_REASON_INVALID", 400);
+  }
+}
+
+export function selectRenderer(
+  input: CreateGovernedExportInput,
+  target: GovernedExportTarget,
+): GovernedExportRendererTarget {
+  const renderers = target.renderers;
+  if (!renderers?.length) {
+    if (input.formatKey !== undefined && input.formatKey !== "ndjson") {
+      throw new GovernedExportServiceError("EXPORT_FORMAT_NOT_REGISTERED", 400);
+    }
+    return legacyNdjsonRenderer;
+  }
+  if (!input.formatKey?.trim()) {
+    throw new GovernedExportServiceError("EXPORT_FORMAT_REQUIRED", 400);
+  }
+  const renderer = renderers.find(
+    ({ formatKey }) => formatKey === input.formatKey,
+  );
+  if (!renderer) {
+    throw new GovernedExportServiceError("EXPORT_FORMAT_NOT_REGISTERED", 400);
+  }
+  return renderer;
+}
+
+export function validateTarget(target: GovernedExportTarget) {
+  if (
+    !target.resourceType.trim() ||
+    !target.policyKey.trim() ||
+    !target.policyVersion.trim() ||
+    target.fieldKeys.length === 0 ||
+    new Set(target.fieldKeys).size !== target.fieldKeys.length ||
+    target.fieldKeys.some((key) => !key.trim()) ||
+    (target.retentionSeconds !== undefined &&
+      (!Number.isInteger(target.retentionSeconds) ||
+        target.retentionSeconds < 60))
+  ) {
+    throw new GovernedExportServiceError("EXPORT_TARGET_INVALID", 500);
+  }
+}
+
+export function durableStatus(
+  status: string,
+): GovernedExportStatus | undefined {
+  return ["queued", "running", "succeeded", "failed", "cancelled"].includes(
+    status,
+  )
+    ? (status as GovernedExportStatus)
+    : undefined;
+}
+
+export function assertArtifactMetadata(
+  artifact: Awaited<ReturnType<DurableArtifactService["inspect"]>>,
+  request: GovernedExportRequest,
+) {
+  if (
+    artifact.jobId !== request.durableJobId ||
+    artifact.ownerId !== request.requesterId ||
+    artifact.contentType !== request.renderer.contentType ||
+    !artifact.filename.endsWith(`.${request.renderer.filenameExtension}`) ||
+    artifact.metadata?.formatKey !== request.renderer.formatKey ||
+    artifact.metadata?.templateKey !== request.renderer.templateKey ||
+    artifact.metadata?.templateVersion !== request.renderer.templateVersion ||
+    artifact.metadata?.inputSchemaKey !== request.renderer.inputSchemaKey ||
+    artifact.byteSize < 0 ||
+    artifact.byteSize > request.renderer.maxOutputBytes ||
+    !/^[a-f0-9]{64}$/.test(artifact.sha256)
+  ) {
+    throw new GovernedExportServiceError("EXPORT_ARTIFACT_INVALID", 409);
+  }
+}
+
+export function exportEvent(
+  context: RequestGuardContext,
+  action: AuditEvent["action"],
+  id: string,
+  metadata: Record<string, unknown>,
+  resourceType: string,
+): AuditEvent {
+  return {
+    id: Bun.randomUUIDv7(),
+    action,
+    outcome: "success",
+    actor: auditActor(context),
+    target: { type: `${resourceType}.governed-export`, id },
+    requestId: context.requestId,
+    metadata,
+    occurredAt: new Date().toISOString(),
+  };
+}
+
+export function downloadEvent(
+  context: RequestGuardContext,
+  request: GovernedExportRequest,
+  artifact: Awaited<ReturnType<DurableArtifactService["inspect"]>>,
+  phase: "intent" | "authorized",
+) {
+  return exportEvent(
+    context,
+    "data.governed-export.download",
+    request.id,
+    {
+      phase,
+      scopeHash: request.scopeHash,
+      jobId: request.durableJobId,
+      sha256: artifact.sha256,
+      byteSize: artifact.byteSize,
+      filename: artifact.filename,
+      contentType: artifact.contentType,
+      formatKey: request.renderer.formatKey,
+      templateKey: request.renderer.templateKey,
+      templateVersion: request.renderer.templateVersion,
+    },
+    request.resourceType,
+  );
+}

package/template/apps/server/src/data-governance/export/service.ts

@@ -1,6 +1,4 @@
-import type { GovernedExportQuerySnapshot } from "@southwind-ai/database";
-import type { AuditEvent } from "@southwind-ai/shared";
-import { auditActor, type RequestGuardContext } from "../../guards.js";
+import type { RequestGuardContext } from "../../guards.js";
 import {
   DurableArtifactError,
   type DurableArtifactService,
@@ -14,17 +12,27 @@ import type { GovernedExportRepository } from "./repository.js";
 import { captureGovernedExportScope } from "./scope-snapshot.js";
 import type { GovernedExportRequest, GovernedExportStatus } from "./types.js";
 import { fingerprintGovernedExportRequest } from "./request-fingerprint.js";
+import {
+  assertArtifactMetadata,
+  downloadEvent,
+  durableStatus,
+  exportEvent,
+  GovernedExportServiceError,
+  governedUserTarget,
+  selectRenderer,
+  validateCreate,
+  validateTarget,
+  type CreateGovernedExportInput,
+  type GovernedExportPolicy,
+  type GovernedExportTarget,
+} from "./service-support.js";
 
-export interface CreateGovernedExportInput {
-  idempotencyKey: string;
-  reason: string;
-  query: GovernedExportQuerySnapshot;
-}
-
-export interface GovernedExportPolicy {
-  approvalRequired: boolean;
-  retentionSeconds: number;
-}
+export {
+  GovernedExportServiceError,
+  type CreateGovernedExportInput,
+  type GovernedExportPolicy,
+  type GovernedExportTarget,
+} from "./service-support.js";
 
 type AuditSink = ReturnType<typeof createServerRuntime>["auditSink"];
 
@@ -43,19 +51,26 @@ export class GovernedExportService {
   async request(
     input: CreateGovernedExportInput,
     context: RequestGuardContext,
+    target: GovernedExportTarget = governedUserTarget,
   ) {
     validateCreate(input);
+    validateTarget(target);
+    const renderer = selectRenderer(input, target);
     const scope = captureGovernedExportScope(context.actor.scopeContext);
     const id = Bun.randomUUIDv7();
+    const retentionSeconds = Math.min(
+      target.retentionSeconds ?? this.policy.retentionSeconds,
+      this.policy.retentionSeconds,
+    );
     const expiresAt = new Date(
-      this.now().getTime() + this.policy.retentionSeconds * 1_000,
+      this.now().getTime() + retentionSeconds * 1_000,
     ).toISOString();
     const draft = {
       id,
-      resourceType: "system.user",
-      policyKey: "system.user.fields",
-      policyVersion: "1",
-      fieldKeys: ["email", "phone"],
+      resourceType: target.resourceType,
+      policyKey: target.policyKey,
+      policyVersion: target.policyVersion,
+      fieldKeys: [...target.fieldKeys].sort(),
       requesterId: context.actor.id,
       requesterName: context.actor.displayName || context.actor.name,
       idempotencyKey: input.idempotencyKey,
@@ -63,7 +78,8 @@ export class GovernedExportService {
       scopeSnapshot: scope.snapshot,
       scopeHash: scope.hash,
       querySnapshot: structuredClone(input.query),
-      approvalRequired: this.policy.approvalRequired,
+      renderer: structuredClone(renderer),
+      approvalRequired: target.approvalRequired ?? this.policy.approvalRequired,
       expiresAt,
     };
     const created = await this.requests.create(
@@ -77,29 +93,53 @@ export class GovernedExportService {
       throw new GovernedExportServiceError("EXPORT_IDEMPOTENCY_CONFLICT", 409);
     }
     await this.audit.recordCriticalEvent(
-      event(context, "data.governed-export.request", created.request.id, {
-        policyKey: created.request.policyKey,
-        policyVersion: created.request.policyVersion,
-        scopeHash: created.request.scopeHash,
-        requestFingerprint: created.request.requestFingerprint,
-        replayed: !created.created,
-      }),
+      exportEvent(
+        context,
+        "data.governed-export.request",
+        created.request.id,
+        {
+          policyKey: created.request.policyKey,
+          policyVersion: created.request.policyVersion,
+          scopeHash: created.request.scopeHash,
+          requestFingerprint: created.request.requestFingerprint,
+          formatKey: created.request.renderer.formatKey,
+          templateKey: created.request.renderer.templateKey,
+          templateVersion: created.request.renderer.templateVersion,
+          replayed: !created.created,
+        },
+        created.request.resourceType,
+      ),
     );
     return this.reconcile(created.request, context);
   }
 
-  async status(id: string, context: RequestGuardContext) {
-    return this.reconcile(await this.owned(id, context), context);
+  async status(
+    id: string,
+    context: RequestGuardContext,
+    target?: GovernedExportTarget,
+  ) {
+    return this.reconcile(await this.owned(id, context, target), context);
   }
 
-  async cancel(id: string, context: RequestGuardContext, reason: string) {
-    const request = await this.owned(id, context);
+  async cancel(
+    id: string,
+    context: RequestGuardContext,
+    reason: string,
+    target?: GovernedExportTarget,
+  ) {
+    const request = await this.owned(id, context, target);
     if (request.status === "cancelled") return request;
     await this.audit.recordCriticalEvent(
-      event(context, "data.governed-export.cancel", request.id, {
-        scopeHash: request.scopeHash,
-        jobId: request.durableJobId,
-      }),
+      exportEvent(
+        context,
+        "data.governed-export.cancel",
+        request.id,
+        {
+          scopeHash: request.scopeHash,
+          jobId: request.durableJobId,
+        },
+        request.resourceType,
+      ),
     );
     await this.approvals.cancel(
       request,
@@ -121,9 +161,10 @@ export class GovernedExportService {
   async download(
     id: string,
     context: RequestGuardContext,
+    target?: GovernedExportTarget,
   ): Promise<OpenedDurableArtifact> {
     const request = await this.reconcile(
-      await this.owned(id, context),
+      await this.owned(id, context, target),
       context,
     );
     if (request.status !== "succeeded" || !request.durableJobId) {
@@ -138,12 +179,12 @@ export class GovernedExportService {
     );
     assertArtifactMetadata(artifact, request);
     await this.audit.recordCriticalEvent(
-      downloadEvent(context, request, artifact.sha256, "intent"),
+      downloadEvent(context, request, artifact, "intent"),
     );
     const opened = await this.artifacts.openInspected(artifact);
     try {
       await this.audit.recordCriticalEvent(
-        downloadEvent(context, request, artifact.sha256, "authorized"),
+        downloadEvent(context, request, artifact, "authorized"),
       );
     } catch (error) {
       await opened.body.cancel().catch(() => undefined);
@@ -152,10 +193,20 @@ export class GovernedExportService {
     return opened;
   }
 
-  private async owned(id: string, context: RequestGuardContext) {
+  private async owned(
+    id: string,
+    context: RequestGuardContext,
+    target?: GovernedExportTarget,
+  ) {
     const request = await this.requests.get(id);
     if (!request) throw new GovernedExportServiceError("EXPORT_NOT_FOUND", 404);
-    if (request.requesterId !== context.actor.id) {
+    if (
+      request.requesterId !== context.actor.id ||
+      (target &&
+        (request.resourceType !== target.resourceType ||
+          request.policyKey !== target.policyKey ||
+          request.policyVersion !== target.policyVersion))
+    ) {
       throw new GovernedExportServiceError("EXPORT_FORBIDDEN", 403);
     }
     const current = captureGovernedExportScope(context.actor.scopeContext);
@@ -257,77 +308,3 @@ export class GovernedExportService {
     }
   }
 }
-
-export class GovernedExportServiceError extends Error {
-  constructor(
-    readonly code: string,
-    readonly status: number,
-  ) {
-    super(code);
-    this.name = "GovernedExportServiceError";
-  }
-}
-
-function validateCreate(input: CreateGovernedExportInput) {
-  if (!input.idempotencyKey.trim() || input.idempotencyKey.length > 120) {
-    throw new GovernedExportServiceError("EXPORT_IDEMPOTENCY_KEY_INVALID", 400);
-  }
-  if (input.reason.trim().length < 5 || input.reason.length > 200) {
-    throw new GovernedExportServiceError("EXPORT_REASON_INVALID", 400);
-  }
-}
-
-function durableStatus(status: string): GovernedExportStatus | undefined {
-  return ["queued", "running", "succeeded", "failed", "cancelled"].includes(
-    status,
-  )
-    ? (status as GovernedExportStatus)
-    : undefined;
-}
-
-function assertArtifactMetadata(
-  artifact: Awaited<ReturnType<DurableArtifactService["inspect"]>>,
-  request: GovernedExportRequest,
-) {
-  if (
-    artifact.jobId !== request.durableJobId ||
-    artifact.ownerId !== request.requesterId ||
-    artifact.contentType !== "application/x-ndjson" ||
-    artifact.byteSize < 0 ||
-    !/^[a-f0-9]{64}$/.test(artifact.sha256)
-  ) {
-    throw new GovernedExportServiceError("EXPORT_ARTIFACT_INVALID", 409);
-  }
-}
-
-function event(
-  context: RequestGuardContext,
-  action: AuditEvent["action"],
-  id: string,
-  metadata: Record<string, unknown>,
-): AuditEvent {
-  return {
-    id: Bun.randomUUIDv7(),
-    action,
-    outcome: "success",
-    actor: auditActor(context),
-    target: { type: "system.user.governed-export", id },
-    requestId: context.requestId,
-    metadata,
-    occurredAt: new Date().toISOString(),
-  };
-}
-
-function downloadEvent(
-  context: RequestGuardContext,
-  request: GovernedExportRequest,
-  sha256: string,
-  phase: "intent" | "authorized",
-) {
-  return event(context, "data.governed-export.download", request.id, {
-    phase,
-    scopeHash: request.scopeHash,
-    jobId: request.durableJobId,
-    sha256,
-  });
-}

package/template/apps/server/src/data-governance/export/source.ts

@@ -1,6 +1,9 @@
 import type { GovernedExportQuerySnapshot } from "@southwind-ai/database";
+import type { DataGovernancePolicyDefinition } from "@southwind-ai/shared";
+import type { ModuleGovernedResourceRegistration } from "@southwind-ai/server-sdk";
 import { restoreServerScopeContext } from "../../data-access/context.js";
 import type { RequestActor } from "../../guards.js";
+import { toModuleActor } from "../../module-host/request-context.js";
 import type { ManagedUser } from "../../system/entities.js";
 import type { ResourceRouteConfig } from "../../system/route-types.js";
 import type { GovernedExportRequest } from "./types.js";
@@ -9,6 +12,7 @@ export interface GovernedExportSource {
   readonly resourceType: string;
   pages(
     request: GovernedExportRequest,
+    signal?: AbortSignal,
   ): AsyncIterable<readonly Record<string, unknown>[]>;
 }
 
@@ -44,6 +48,103 @@ export class GovernedUserExportSource implements GovernedExportSource {
   }
 }
 
+export class ModuleGovernedExportSource implements GovernedExportSource {
+  readonly resourceType: string;
+
+  constructor(
+    private readonly registration: ModuleGovernedResourceRegistration,
+    policy: DataGovernancePolicyDefinition,
+    private readonly resolveActor: (
+      actorId: string,
+    ) => Promise<RequestActor | undefined>,
+  ) {
+    this.resourceType = policy.resourceType;
+  }
+
+  async *pages(request: GovernedExportRequest, signal?: AbortSignal) {
+    const actor = await this.resolveActor(request.requesterId);
+    if (!actor) throw new Error("EXPORT_ACTOR_UNAVAILABLE");
+    const source = this.registration.export;
+    if (!source) throw new Error("EXPORT_SOURCE_NOT_REGISTERED");
+    let cursor: string | undefined;
+    let pageCount = 0;
+    const seen = new Set<string>();
+    do {
+      if (signal?.aborted) throw new Error("EXPORT_ABORTED");
+      pageCount += 1;
+      if (pageCount > 10_000) throw new Error("EXPORT_PAGE_BUDGET_EXCEEDED");
+      const page = await source.readPage(
+        {
+          query: request.querySnapshot,
+          cursor,
+          limit: 100,
+        },
+        {
+          actor: toModuleActor(actor),
+          requestId: request.id,
+          signal: signal ?? new AbortController().signal,
+        },
+      );
+      validatePage(page.records, page.nextCursor, cursor, seen);
+      yield page.records;
+      cursor = page.nextCursor;
+    } while (cursor);
+  }
+}
+
+function validatePage(
+  records: readonly Readonly<Record<string, unknown>>[],
+  nextCursor: string | undefined,
+  currentCursor: string | undefined,
+  seen: Set<string>,
+) {
+  if (records.length > 100) throw new Error("EXPORT_PAGE_LIMIT_EXCEEDED");
+  for (const record of records) {
+    if (
+      !record ||
+      Array.isArray(record) ||
+      typeof record !== "object" ||
+      !isJsonValue(record)
+    ) {
+      throw new Error("EXPORT_RECORD_INVALID");
+    }
+  }
+  if (nextCursor !== undefined) {
+    if (
+      !nextCursor ||
+      nextCursor.length > 500 ||
+      nextCursor === currentCursor ||
+      seen.has(nextCursor)
+    ) {
+      throw new Error("EXPORT_CURSOR_INVALID");
+    }
+    seen.add(nextCursor);
+  }
+}
+
+function isJsonValue(value: unknown, depth = 0): boolean {
+  if (depth > 10) return false;
+  if (
+    value === null ||
+    typeof value === "string" ||
+    typeof value === "boolean"
+  ) {
+    return true;
+  }
+  if (typeof value === "number") return Number.isFinite(value);
+  if (Array.isArray(value)) {
+    return value.every((item) => isJsonValue(item, depth + 1));
+  }
+  if (
+    typeof value !== "object" ||
+    (Object.getPrototypeOf(value) !== Object.prototype &&
+      Object.getPrototypeOf(value) !== null)
+  ) {
+    return false;
+  }
+  return Object.values(value).every((item) => isJsonValue(item, depth + 1));
+}
+
 function exportActor(request: GovernedExportRequest): RequestActor {
   return {
     id: request.requesterId,

package/template/apps/server/src/data-governance/export/types.ts

@@ -26,6 +26,7 @@ export interface GovernedExportRequest {
   scopeSnapshot: GovernedExportScopeSnapshot;
   scopeHash: string;
   querySnapshot: GovernedExportQuerySnapshot;
+  renderer: GovernedExportRendererSnapshot;
   status: GovernedExportStatus;
   approvalRequired: boolean;
   approvalId?: string;
@@ -49,7 +50,33 @@ export interface CreateGovernedExportRequest {
   scopeSnapshot: GovernedExportScopeSnapshot;
   scopeHash: string;
   querySnapshot: GovernedExportQuerySnapshot;
+  renderer: GovernedExportRendererSnapshot;
   approvalRequired: boolean;
   approvalId?: string;
   expiresAt: string;
 }
+
+export interface GovernedExportRendererSnapshot {
+  formatKey: string;
+  templateKey: string;
+  templateVersion: string;
+  inputSchemaKey: string;
+  contentType: string;
+  filenameExtension: string;
+  maxRecords: number;
+  maxOutputBytes: number;
+  timeoutSeconds: number;
+}
+
+export const legacyGovernedExportRendererSnapshot: GovernedExportRendererSnapshot =
+  {
+    formatKey: "ndjson",
+    templateKey: "system.governed-export.ndjson",
+    templateVersion: "1",
+    inputSchemaKey: "system.governed-export.record.v1",
+    contentType: "application/x-ndjson",
+    filenameExtension: "ndjson",
+    maxRecords: 50_000,
+    maxOutputBytes: 10 * 1024 * 1024,
+    timeoutSeconds: 900,
+  };