@anthusai/papyrus 1.0.0-next.53 → 1.0.0-next.54

package/amplify/functions/content-actions/handler.py

@@ -7,6 +7,7 @@ from typing import Any
 from papyrus_content.edge_invalidation import distribution_ids_from_environment, invalidate_revalidated_paths
 from papyrus_content.graphql_authoring import create_authoring_client
 from papyrus_content.markus_renderer.derive import derive_body
+from papyrus_content.projection.deploy_render import deploy_render, drift_metric_line
 from papyrus_content.projection.engine import project, project_all, project_item_pages
 from papyrus_content.projection.runtime import (
     DEAD_LETTER_QUEUE_ENV,
@@ -146,6 +147,18 @@ def schedule_unpurged_invalidation(keys: list[str]) -> None:
     schedule_projection_task({"task": "invalidate", "keys": keys}, "edge invalidation failed")
 
 
+def run_deploy_render_task(client) -> dict:
+    """Render every published page after a backend deploy, then report drift (a dry run: nothing is deleted)."""
+    context = projection_context(client)
+    report = deploy_render(context)
+    print(drift_metric_line(report))
+    if report.render.unpurged:
+        schedule_unpurged_invalidation(report.render.unpurged)
+    if not report.ok:
+        raise ProjectionTaskFailed("; ".join([*report.render.errors, *report.drift.errors]))
+    return report.to_dict()
+
+
 def run_projection_task(task: dict) -> dict:
     """An asynchronous projection task. Raising makes Lambda retry it, then send the event to the dead letter queue."""
     client, _claims = create_authoring_client()
@@ -157,8 +170,8 @@ def run_projection_task(task: dict) -> dict:
             schedule_projection_task({"task": "item-pages", "exceptItemId": task["itemId"]}, "follow-up")
     elif task.get("task") == "all":
         report = project_all(projection_context(client))
-    elif task.get("task") == "all":
-        report = project_all(projection_context(client))
+    elif task.get("task") == "deploy-render":
+        return run_deploy_render_task(client)
     elif task.get("task") == "invalidate":
         invalidate = projection_context(client).invalidate
         if invalidate is None:

package/amplify/publication/reproject-on-deploy.js

@@ -1,31 +1,73 @@
+import { CustomResource, Duration } from "aws-cdk-lib";
+import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
 import { PolicyStatement } from "aws-cdk-lib/aws-iam";
-import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from "aws-cdk-lib/custom-resources";
-export const REPROJECT_ON_DEPLOY_TIMEOUT_SECONDS = 300;
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import { Provider } from "aws-cdk-lib/custom-resources";
+export const REPROJECT_ON_DEPLOY_TIMEOUT_SECONDS = 900;
+export const DEPLOY_RENDER_METRIC_NAMESPACE = "Papyrus/Projection";
+export const DEPLOY_RENDER_DRIFT_METRIC_NAME = "DeployRenderDrift";
+export const DEPLOY_RENDER_TASK = "deploy-render";
+const FAILURE_MESSAGE_LIMIT = 600;
+const RESULT_SUMMARY_LIMIT = 900;
 export function reprojectDeployToken(environment, now = Date.now) {
     const buildJobId = (environment.AWS_JOB_ID ?? "").trim();
     return buildJobId !== "" ? `job-${buildJobId}` : `time-${now()}`;
 }
 /**
- * Starts the content-actions `all` projection task (asynchronously) every time the stack deploys.
- * The token is in the request, so CloudFormation sees a changed property on each deploy.
+ * Runs inside the deploy: invokes the content-actions Lambda synchronously, so a render that fails fails the
+ * stack update (and with it the Amplify build) with the failure in the build log. The Lambda's result, which
+ * includes the drift check, is written to its log and returned as the resource's data.
+ */
+export const DEPLOY_RENDER_HANDLER_SOURCE = `
+const { LambdaClient, InvokeCommand } = require("@aws-sdk/client-lambda");
+
+exports.handler = async (event) => {
+  if (event.RequestType === "Delete") return { PhysicalResourceId: event.PhysicalResourceId };
+  const properties = event.ResourceProperties;
+  const client = new LambdaClient({ maxAttempts: 1 });
+  const response = await client.send(new InvokeCommand({
+    FunctionName: properties.FunctionName,
+    InvocationType: "RequestResponse",
+    Payload: JSON.stringify({ projectionTask: { task: properties.Task, deployToken: properties.DeployToken } }),
+  }));
+  const text = Buffer.from(response.Payload || []).toString("utf8");
+  console.log("deploy render result: " + text);
+  if (response.FunctionError) {
+    throw new Error("The post-deploy render failed (" + response.FunctionError + "): " + text.slice(0, ${FAILURE_MESSAGE_LIMIT}));
+  }
+  return { PhysicalResourceId: "reproject-" + properties.DeployToken, Data: { Result: text.slice(0, ${RESULT_SUMMARY_LIMIT}) } };
+};
+`;
+/**
+ * Renders every published page after each backend deploy and checks the site store for drift (a dry run:
+ * nothing is deleted). The token is a property, so CloudFormation runs the resource again on every deploy.
  */
 export function addReprojectOnDeploy(stack, contentActions, deployToken) {
-    const invocation = {
-        service: "Lambda",
-        action: "invoke",
-        parameters: {
-            FunctionName: contentActions.functionName,
-            InvocationType: "Event",
-            Payload: JSON.stringify({ projectionTask: { task: "all", deployToken } }),
-        },
-        physicalResourceId: PhysicalResourceId.of(`reproject-${deployToken}`),
-    };
-    return new AwsCustomResource(stack, "ReprojectOnDeploy", {
-        onCreate: invocation,
-        onUpdate: invocation,
-        installLatestAwsSdk: false,
-        policy: AwsCustomResourcePolicy.fromStatements([
-            new PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [contentActions.functionArn] }),
-        ]),
+    const handler = new lambda.Function(stack, "DeployRenderHandler", {
+        runtime: lambda.Runtime.NODEJS_20_X,
+        handler: "index.handler",
+        code: lambda.Code.fromInline(DEPLOY_RENDER_HANDLER_SOURCE),
+        timeout: Duration.seconds(REPROJECT_ON_DEPLOY_TIMEOUT_SECONDS),
+        description: "Starts the post-deploy render of the site and fails the deploy if it cannot finish.",
+    });
+    handler.addToRolePolicy(new PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [contentActions.functionArn] }));
+    const provider = new Provider(stack, "DeployRenderProvider", { onEventHandler: handler });
+    new cloudwatch.Alarm(stack, "DeployRenderDriftAlarm", {
+        metric: new cloudwatch.Metric({
+            namespace: DEPLOY_RENDER_METRIC_NAMESPACE,
+            metricName: DEPLOY_RENDER_DRIFT_METRIC_NAME,
+            dimensionsMap: { FunctionName: contentActions.functionName },
+            statistic: "Maximum",
+            period: Duration.days(1),
+        }),
+        evaluationPeriods: 1,
+        threshold: 1,
+        comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+        alarmDescription: "The last post-deploy render found drift between the published content and the site store (stale pages, orphan pages or redirects). Nothing was deleted; run papyrus ops content project --reconcile to repair.",
+    });
+    return new CustomResource(stack, "ReprojectOnDeploy", {
+        serviceToken: provider.serviceToken,
+        properties: { FunctionName: contentActions.functionName, Task: DEPLOY_RENDER_TASK, DeployToken: deployToken },
     });
 }

package/amplify/publication/reproject-on-deploy.ts

@@ -1,9 +1,15 @@
-import { Stack } from "aws-cdk-lib";
+import { CustomResource, Duration, Stack } from "aws-cdk-lib";
+import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
 import { PolicyStatement } from "aws-cdk-lib/aws-iam";
-import type { IFunction } from "aws-cdk-lib/aws-lambda";
-import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from "aws-cdk-lib/custom-resources";
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import { Provider } from "aws-cdk-lib/custom-resources";
 
-export const REPROJECT_ON_DEPLOY_TIMEOUT_SECONDS = 300;
+export const REPROJECT_ON_DEPLOY_TIMEOUT_SECONDS = 900;
+export const DEPLOY_RENDER_METRIC_NAMESPACE = "Papyrus/Projection";
+export const DEPLOY_RENDER_DRIFT_METRIC_NAME = "DeployRenderDrift";
+export const DEPLOY_RENDER_TASK = "deploy-render";
+const FAILURE_MESSAGE_LIMIT = 600;
+const RESULT_SUMMARY_LIMIT = 900;
 
 export function reprojectDeployToken(environment: Record<string, string | undefined>, now: () => number = Date.now): string {
   const buildJobId = (environment.AWS_JOB_ID ?? "").trim();
@@ -11,26 +17,62 @@ export function reprojectDeployToken(environment: Record<string, string | undefi
 }
 
 /**
- * Starts the content-actions `all` projection task (asynchronously) every time the stack deploys.
- * The token is in the request, so CloudFormation sees a changed property on each deploy.
+ * Runs inside the deploy: invokes the content-actions Lambda synchronously, so a render that fails fails the
+ * stack update (and with it the Amplify build) with the failure in the build log. The Lambda's result, which
+ * includes the drift check, is written to its log and returned as the resource's data.
  */
-export function addReprojectOnDeploy(stack: Stack, contentActions: IFunction, deployToken: string): AwsCustomResource {
-  const invocation = {
-    service: "Lambda",
-    action: "invoke",
-    parameters: {
-      FunctionName: contentActions.functionName,
-      InvocationType: "Event",
-      Payload: JSON.stringify({ projectionTask: { task: "all", deployToken } }),
-    },
-    physicalResourceId: PhysicalResourceId.of(`reproject-${deployToken}`),
-  };
-  return new AwsCustomResource(stack, "ReprojectOnDeploy", {
-    onCreate: invocation,
-    onUpdate: invocation,
-    installLatestAwsSdk: false,
-    policy: AwsCustomResourcePolicy.fromStatements([
-      new PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [contentActions.functionArn] }),
-    ]),
+export const DEPLOY_RENDER_HANDLER_SOURCE = `
+const { LambdaClient, InvokeCommand } = require("@aws-sdk/client-lambda");
+
+exports.handler = async (event) => {
+  if (event.RequestType === "Delete") return { PhysicalResourceId: event.PhysicalResourceId };
+  const properties = event.ResourceProperties;
+  const client = new LambdaClient({ maxAttempts: 1 });
+  const response = await client.send(new InvokeCommand({
+    FunctionName: properties.FunctionName,
+    InvocationType: "RequestResponse",
+    Payload: JSON.stringify({ projectionTask: { task: properties.Task, deployToken: properties.DeployToken } }),
+  }));
+  const text = Buffer.from(response.Payload || []).toString("utf8");
+  console.log("deploy render result: " + text);
+  if (response.FunctionError) {
+    throw new Error("The post-deploy render failed (" + response.FunctionError + "): " + text.slice(0, ${FAILURE_MESSAGE_LIMIT}));
+  }
+  return { PhysicalResourceId: "reproject-" + properties.DeployToken, Data: { Result: text.slice(0, ${RESULT_SUMMARY_LIMIT}) } };
+};
+`;
+
+/**
+ * Renders every published page after each backend deploy and checks the site store for drift (a dry run:
+ * nothing is deleted). The token is a property, so CloudFormation runs the resource again on every deploy.
+ */
+export function addReprojectOnDeploy(stack: Stack, contentActions: lambda.IFunction, deployToken: string): CustomResource {
+  const handler = new lambda.Function(stack, "DeployRenderHandler", {
+    runtime: lambda.Runtime.NODEJS_20_X,
+    handler: "index.handler",
+    code: lambda.Code.fromInline(DEPLOY_RENDER_HANDLER_SOURCE),
+    timeout: Duration.seconds(REPROJECT_ON_DEPLOY_TIMEOUT_SECONDS),
+    description: "Starts the post-deploy render of the site and fails the deploy if it cannot finish.",
+  });
+  handler.addToRolePolicy(new PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [contentActions.functionArn] }));
+  const provider = new Provider(stack, "DeployRenderProvider", { onEventHandler: handler });
+  new cloudwatch.Alarm(stack, "DeployRenderDriftAlarm", {
+    metric: new cloudwatch.Metric({
+      namespace: DEPLOY_RENDER_METRIC_NAMESPACE,
+      metricName: DEPLOY_RENDER_DRIFT_METRIC_NAME,
+      dimensionsMap: { FunctionName: contentActions.functionName },
+      statistic: "Maximum",
+      period: Duration.days(1),
+    }),
+    evaluationPeriods: 1,
+    threshold: 1,
+    comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+    treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+    alarmDescription:
+      "The last post-deploy render found drift between the published content and the site store (stale pages, orphan pages or redirects). Nothing was deleted; run papyrus ops content project --reconcile to repair.",
+  });
+  return new CustomResource(stack, "ReprojectOnDeploy", {
+    serviceToken: provider.serviceToken,
+    properties: { FunctionName: contentActions.functionName, Task: DEPLOY_RENDER_TASK, DeployToken: deployToken },
   });
 }

package/lib/graphql-content-repository.ts

@@ -21,6 +21,7 @@ import { SITE_BRAND } from "./site-brand";
 import { shouldFetchVideoScripts } from "./video-mode";
 import { collectVideoScriptTargets, parseVideoScriptRef, videomlItemSlug, type VideoScriptRef } from "./video-script";
 import { currentContentSource, currentRequestClient, runWithContentSource } from "./content-source-context";
+import { withRateLimitRetry } from "./rate-limit-retry";
 import { getContentSource } from "./site-env";
 
 const DEFAULT_EDITION_SLUG = "current";
@@ -157,17 +158,19 @@ async function withReaderGraphQLContext<T>(operation: () => Promise<T>): Promise
     return runWithContentSource(source, operation, client);
   }
   return runWithContentSource(source, () =>
-    runWithAmplifyServerContext({
-      nextServerContext: null,
-      operation: async () => {
-        readerContextClient = generateClient<Schema>({ authMode: authMode() });
-        try {
-          return await operation();
-        } finally {
-          readerContextClient = null;
-        }
-      },
-    }),
+    withRateLimitRetry(() =>
+      runWithAmplifyServerContext({
+        nextServerContext: null,
+        operation: async () => {
+          readerContextClient = generateClient<Schema>({ authMode: authMode() });
+          try {
+            return await operation();
+          } finally {
+            readerContextClient = null;
+          }
+        },
+      }),
+    ),
   );
 }
 

package/lib/rate-limit-retry.ts

@@ -0,0 +1,27 @@
+export const RATE_LIMIT_RETRY_ATTEMPTS = 6;
+export const RATE_LIMIT_RETRY_BASE_MILLISECONDS = 300;
+
+export function isRateLimitError(error: unknown): boolean {
+  if (!error || typeof error !== "object") return false;
+  const { name, message } = error as { name?: unknown; message?: unknown };
+  const text = `${typeof name === "string" ? name : ""} ${typeof message === "string" ? message : ""}`;
+  return /TooManyRequests|ThrottlingException|Rate exceeded|Unauthorized[\s\S]*set the \\?"authMode\\?"/i.test(text);
+}
+
+export async function withRateLimitRetry<T>(
+  operation: () => Promise<T>,
+  options: { attempts?: number; baseMilliseconds?: number; sleep?: (milliseconds: number) => Promise<void>; random?: () => number } = {},
+): Promise<T> {
+  const attempts = options.attempts ?? RATE_LIMIT_RETRY_ATTEMPTS;
+  const baseMilliseconds = options.baseMilliseconds ?? RATE_LIMIT_RETRY_BASE_MILLISECONDS;
+  const sleep = options.sleep ?? ((milliseconds: number) => new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
+  const random = options.random ?? Math.random;
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await operation();
+    } catch (error) {
+      if (attempt >= attempts || !isRateLimitError(error)) throw error;
+      await sleep(baseMilliseconds * 2 ** (attempt - 1) * (0.5 + random()));
+    }
+  }
+}

package/package.json

@@ -197,6 +197,7 @@
     "./lib/pretext-layout": "./lib/pretext-layout.ts",
     "./lib/public-placeholder-config": "./lib/public-placeholder-config.ts",
     "./lib/publication-items": "./lib/publication-items.ts",
+    "./lib/rate-limit-retry": "./lib/rate-limit-retry.ts",
     "./lib/reader-base-path": "./lib/reader-base-path.ts",
     "./lib/reader-cache-lifetime": "./lib/reader-cache-lifetime.ts",
     "./lib/reader-route-config": "./lib/reader-route-config.ts",
@@ -309,7 +310,7 @@
       "optional": true
     }
   },
-  "version": "1.0.0-next.53",
+  "version": "1.0.0-next.54",
   "dependencies": {
     "@aws-amplify/adapter-nextjs": "^1.7.3",
     "@aws-sdk/client-cognito-identity-provider": "3.1045.0",

package/routes.manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.0-next.53",
+  "version": "1.0.0-next.54",
   "routes": [
     {
       "file": "app/%5F_preview/[[...path]]/route.ts",