@anthusai/papyrus 5.43.0 → 5.45.0

package/infra/chat.test.ts

@@ -0,0 +1,151 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { App } from "aws-cdk-lib";
+import { AmplifyAppShellStack } from "./amplify-app-shell";
+import { PapyrusChatStack } from "./chat-stack";
+import { parseSiteConfig, resolveChatLogGroupPrefix, resolveChatStackName, resolveStackName } from "./site-config";
+
+// Executes infra/amplify-app-shell/features/per-site-private-chat.feature (run through scripts/test-infra-chat.ts).
+const examples = path.resolve(__dirname, "../examples");
+const example = (name: string, change: (config: any) => void = () => {}): any => {
+  const config = JSON.parse(fs.readFileSync(path.join(examples, `${name}.site.json`), "utf8"));
+  change(config);
+  return config;
+};
+const siteNames = fs.readdirSync(examples).filter((name) => name.endsWith(".site.json")).map((name) => name.replace(/\.site\.json$/, ""));
+const lambdaDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "papyrus-chat-test-"));
+for (const file of ["run-worker.js", "probe-worker.js"]) fs.writeFileSync(path.join(lambdaDirectory, file), "exports.handler = async () => ({});\n");
+
+const resourcesOf = (template: any, type: string): any[] => Object.values<any>(template.Resources).filter((resource) => resource.Type === type);
+function shellTemplate(raw: unknown): any {
+  const config = parseSiteConfig(raw);
+  const stack = new AmplifyAppShellStack(new App(), "Shell", config, { env: { account: "712236451410", region: "us-east-1" }, stackName: resolveStackName(config) });
+  return JSON.parse(JSON.stringify((App.of(stack) as App).synth().getStackByName(stack.stackName).template));
+}
+function chatTemplate(raw: unknown): any {
+  const config = parseSiteConfig(raw);
+  const app = new App();
+  const stack = new PapyrusChatStack(app, { config, env: { account: "712236451410", region: "us-east-1" }, lambdaAssetDirectory: lambdaDirectory });
+  return JSON.parse(JSON.stringify(app.synth().getStackByName(stack.stackName).template));
+}
+const branchEnvironment = (template: any, branchName: string): Record<string, unknown> => {
+  const branch = resourcesOf(template, "AWS::Amplify::Branch").find((resource) => resource.Properties.BranchName === branchName);
+  return Object.fromEntries(branch.Properties.EnvironmentVariables.map((variable: any) => [variable.Name, variable.Value]));
+};
+const CHAT_NAMES = ["PAPYRUS_CHAT_MESSAGING_TABLE", "PAPYRUS_CHAT_CONVERSATIONS_TABLE", "PAPYRUS_CHAT_SESSION_BUCKET", "PAPYRUS_CHAT_TURN_RUNS_QUEUE_URL", "PAPYRUS_CHAT_TURN_PROBES_QUEUE_URL", "PAPYRUS_CHAT_PRIVATE_BOT_ID", "PAPYRUS_CHAT_PRIVATE_BOT_NAME"];
+
+// Scenario: a site with chat enabled gets the chat environment and a data stack
+{
+  const raw = example("anth-us");
+  assert.equal(raw.chat.modelKeyParameter, "/papyrus/shipped/openai-api-key");
+  const config = parseSiteConfig(raw);
+  assert.equal(resolveChatStackName(config), "papyrus-chat-anth-us");
+  const production = branchEnvironment(shellTemplate(raw), "main");
+  assert.equal(production.PAPYRUS_PRIVATE_CHAT_ENABLED, "true");
+  assert.deepEqual(production.PAPYRUS_CHAT_MESSAGING_TABLE, { "Fn::ImportValue": "papyrus-chat-anth-us:MessagingTableName" });
+  assert.deepEqual(production.PAPYRUS_CHAT_SESSION_BUCKET, { "Fn::ImportValue": "papyrus-chat-anth-us:SessionBucketName" });
+  assert.deepEqual(production.PAPYRUS_CHAT_TURN_RUNS_QUEUE_URL, { "Fn::ImportValue": "papyrus-chat-anth-us:TurnRunsQueueUrl" });
+  assert.equal(production.PAPYRUS_CHAT_PRIVATE_BOT_ID, "papyrus-private-assistant");
+
+  const chat = chatTemplate(raw);
+  assert.equal(resourcesOf(chat, "AWS::DynamoDB::Table").length, 2);
+  const [bucket] = resourcesOf(chat, "AWS::S3::Bucket");
+  assert.equal(bucket.Properties.VersioningConfiguration.Status, "Enabled");
+  assert.equal(bucket.Properties.PublicAccessBlockConfiguration.BlockPublicPolicy, true);
+  assert.equal(resourcesOf(chat, "AWS::SQS::Queue").length, 4);
+  assert.equal(resourcesOf(chat, "AWS::Lambda::Function").length, 2);
+  const mappings = resourcesOf(chat, "AWS::Lambda::EventSourceMapping");
+  assert.equal(mappings.length, 2);
+  assert.ok(mappings.every((mapping) => mapping.Properties.Enabled === true));
+  for (const key of ["MessagingTableName", "ConversationIndexTableName", "SessionBucketName", "TurnRunsQueueUrl", "TurnProbesQueueUrl"]) {
+    assert.equal(chat.Outputs[key].Export.Name, `papyrus-chat-anth-us:${key}`);
+  }
+  assert.deepEqual(Object.keys(chat.Parameters).filter((name) => name !== "BootstrapVersion").sort(), ["GatewayRoleArn", "PrivateAssistantBotId", "PrivateAssistantBotName"]);
+
+  // Only the run worker may read the model key parameter; the gateway policy never names it.
+  const parameterGrants = resourcesOf(chat, "AWS::IAM::Policy").filter((policy) => JSON.stringify(policy.Properties.PolicyDocument).includes("openai-api-key"));
+  assert.equal(parameterGrants.length, 1);
+  assert.ok(parameterGrants[0].Properties.Roles.some((role: any) => JSON.stringify(role).includes("RunWorkerRole")));
+  assert.equal(JSON.stringify(chat).split("/papyrus/shipped/openai-api-key").length - 1, 3, "the parameter appears in the run worker grant, the run worker environment and one output only");
+  const gateway = resourcesOf(chat, "AWS::IAM::Policy").find((policy) => policy.Properties.Roles.some((role: any) => JSON.stringify(role).includes("GatewayRoleArn")));
+  assert.ok(gateway, "the gateway policy attaches to the role named by GatewayRoleArn");
+  const gatewayActions = gateway.Properties.PolicyDocument.Statement.flatMap((statement: any) => [statement.Action].flat());
+  assert.ok(!gatewayActions.some((action: string) => action.startsWith("ssm:") || action.endsWith("*")), "the gateway has no SSM access and no wildcard action");
+}
+
+// Scenario: the staging branch never gets private chat
+{
+  const staging = branchEnvironment(shellTemplate(example("p-apyr-us")), "staging");
+  assert.equal(staging.PAPYRUS_PRIVATE_CHAT_ENABLED, "false");
+  assert.deepEqual(CHAT_NAMES.filter((name) => name in staging), []);
+}
+
+// Scenario: a site without chat gets neither
+{
+  const withoutBlock = example("anth-us", (config) => delete config.chat);
+  const disabled = example("anth-us", (config) => (config.chat.enabled = false));
+  for (const raw of [withoutBlock, disabled]) {
+    const production = branchEnvironment(shellTemplate(raw), "main");
+    assert.equal(production.PAPYRUS_PRIVATE_CHAT_ENABLED, "false");
+    assert.deepEqual(CHAT_NAMES.filter((name) => name in production), []);
+    assert.equal(resolveChatStackName(parseSiteConfig(raw)), undefined);
+    assert.throws(() => chatTemplate(raw), /does not enable chat/);
+  }
+}
+
+// Scenario: the chat block cannot be contradicted by hand-set variables
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => (config.cms.environment.PAPYRUS_PRIVATE_CHAT_ENABLED = "false"))), /cms\.environment\.PAPYRUS_PRIVATE_CHAT_ENABLED/);
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => (config.cms.environment.PAPYRUS_CHAT_SESSION_BUCKET = "x"))), /cms\.environment\.PAPYRUS_CHAT_SESSION_BUCKET/);
+// Without chat, the existing hand-managed flag keeps working.
+assert.equal(branchEnvironment(shellTemplate(example("anth-us", (config) => { delete config.chat; config.cms.environment.PAPYRUS_PRIVATE_CHAT_ENABLED = "true"; })), "main").PAPYRUS_PRIVATE_CHAT_ENABLED, "true");
+
+// Scenario: chat needs a CMS and a model key parameter
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => delete config.chat.modelKeyParameter)), /chat\.modelKeyParameter/);
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => (config.chat.modelKeyParameter = "not-a-path"))), /chat\.modelKeyParameter/);
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => (config.chat.enabled = "yes"))), /chat\.enabled/);
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => (config.chat.extra = 1))), /chat\.extra/);
+assert.throws(() => parseSiteConfig(example("anth-us", (config) => (config.chat.workerCodeS3Key = "abc"))), /chat\.workerCodeS3Key/);
+assert.throws(() => parseSiteConfig({ siteId: "reader-only", repository: "https://github.com/example/x", brand: "x", frontend: "markus-static", papyrusVersion: "5.42.1", hostedZoneId: "Z10285921B1G7MVRV06W9", reader: { domainName: "x.example.test", buildCommand: "npm run build", baseDirectory: "out" }, chat: { enabled: true, modelKeyParameter: "/papyrus/x/openai-api-key" } }), /chat\.enabled.*requires cms/);
+assert.throws(() => new PapyrusChatStack(new App(), { config: parseSiteConfig(example("p-apyr-us", (config) => delete config.chat.workerCodeS3Key)), env: { account: "712236451410", region: "us-east-1" } }), /bundle the chat workers first/);
+
+// Scenario: two sites in one account do not share chat storage or log groups
+{
+  const enabled = siteNames.map((name) => parseSiteConfig(example(name))).filter((config) => config.chat?.enabled);
+  assert.deepEqual(enabled.map((config) => config.siteId).sort(), ["anth-us", "dataparade", "distinctive", "p-apyr-us", "pilobol-us", "shipped", "threat-intelligence"], "every Papyrus site example enables chat");
+  const names = enabled.map((config) => resolveChatStackName(config));
+  const prefixes = enabled.map((config) => resolveChatLogGroupPrefix(config));
+  assert.equal(new Set(names).size, names.length, "stack names are unique");
+  assert.equal(new Set(prefixes).size, prefixes.length, "worker log group prefixes are unique");
+  for (const config of enabled) {
+    assert.ok(/^\/(papyrus\/[a-z0-9-]+\/openai-api-key|amplify\/shared\/[a-z0-9]+\/OPENAI_API_KEY)$/.test(config.chat!.modelKeyParameter), `${config.siteId} reuses an existing model key parameter`);
+  }
+}
+
+// Scenario: the activated p.apyr.us stack is adopted without replacing anything
+{
+  const activated = JSON.parse(fs.readFileSync(path.resolve(__dirname, "../../shared-chat-runtime/cdk/PapyrusChatDataCandidate.template.json"), "utf8"));
+  const generated = chatTemplate(example("p-apyr-us"));
+  assert.equal(resolveChatStackName(parseSiteConfig(example("p-apyr-us"))), "PapyrusChatDataCandidate");
+  const normalize = (resource: any) => {
+    const copy = JSON.parse(JSON.stringify(resource));
+    delete copy.Metadata;
+    const statements = copy.Properties?.PolicyDocument?.Statement;
+    if (statements) copy.Properties.PolicyDocument.Statement = [...statements].sort((a: any, b: any) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
+    return copy;
+  };
+  assert.deepEqual(Object.keys(generated.Resources).sort(), Object.keys(activated.Resources).sort(), "no resource is added, removed or renamed (a changed logical id would replace it)");
+  for (const [logicalId, resource] of Object.entries<any>(activated.Resources)) {
+    const expected = normalize(resource);
+    // The activated template is the reviewed candidate; the queue mappings were enabled live afterwards.
+    if (resource.Type === "AWS::Lambda::EventSourceMapping") expected.Properties.Enabled = true;
+    assert.deepEqual(normalize(generated.Resources[logicalId]), expected, `${logicalId} is unchanged`);
+  }
+  for (const key of Object.keys(activated.Outputs)) {
+    assert.deepEqual({ ...generated.Outputs[key], Export: undefined }, { ...activated.Outputs[key], Export: undefined, ...(key === "FeatureDefault" ? { Value: generated.Outputs[key].Value } : {}) }, `output ${key} keeps its value`);
+  }
+  assert.deepEqual(Object.keys(generated.Outputs).sort(), Object.keys(activated.Outputs).sort());
+}
+
+console.log("per-site private chat infra tests passed");

package/infra/index.js

@@ -10,6 +10,7 @@ try {
   );
 }
 export const { AmplifyAppShellStack } = await import("./amplify-app-shell.js");
+export const { PapyrusChatStack } = await import("./chat-stack.js");
 export const { SecureStringParameter } = await import("./secure-string-parameter.js");
 export const { GithubOidcProviderStack } = await import("./github-oidc-provider.js");
 export { parseSiteConfig } from "./site-config.js";

package/infra/site-config.js

@@ -28,6 +28,9 @@ const HOST_NAME_PATTERN = /^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}$/;
 const SITE_ORIGIN_PATTERN = /^https:\/\/([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}(:\d{1,5})?$/;
 const DOMAIN_PREFIX_PATTERN = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
 const APP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9 _.-]*$/;
+const CHAT_MODEL_KEY_PARAMETER_PATTERN = /^\/[A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)+$/;
+const CHAT_LOG_GROUP_PREFIX_PATTERN = /^\/aws\/lambda\/[A-Za-z0-9_.-]{1,100}$/;
+const CHAT_WORKER_CODE_KEY_PATTERN = /^[0-9a-f]{64}\.zip$/;
 const STACK_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9-]{0,127}$/;
 const BRANCH_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9/_.-]*$/;
 const PAPYRUS_VERSION_PATTERN = /^\d+\.\d+\.\d+(-[0-9A-Za-z-]+\.\d+)?$/;
@@ -211,6 +214,58 @@ export function resolveStagingDomainName(config) {
 export function resolveStackName(config) {
     return config.stackName ?? `amplify-app-shell-${config.siteId}`;
 }
+/** The chat data stack of a site; its construct id and CloudFormation name. Nothing when chat is not enabled. */
+export function resolveChatStackName(config) {
+    if (!config.chat?.enabled)
+        return undefined;
+    return config.chat.stackName ?? `papyrus-chat-${config.siteId}`;
+}
+export function resolveChatLogGroupPrefix(config) {
+    return config.chat?.logGroupPrefix ?? `/aws/lambda/papyrus-chat-${config.siteId}`;
+}
+/** CloudFormation export names the chat stack publishes and the app shell imports (no stack ever references the other directly). */
+export const CHAT_EXPORTED_RESOURCES = ["MessagingTableName", "ConversationIndexTableName", "SessionBucketName", "TurnRunsQueueUrl", "TurnProbesQueueUrl"];
+export const CHAT_ENVIRONMENT_NAMES = {
+    MessagingTableName: "PAPYRUS_CHAT_MESSAGING_TABLE",
+    ConversationIndexTableName: "PAPYRUS_CHAT_CONVERSATIONS_TABLE",
+    SessionBucketName: "PAPYRUS_CHAT_SESSION_BUCKET",
+    TurnRunsQueueUrl: "PAPYRUS_CHAT_TURN_RUNS_QUEUE_URL",
+    TurnProbesQueueUrl: "PAPYRUS_CHAT_TURN_PROBES_QUEUE_URL",
+};
+export const CHAT_PRIVATE_ASSISTANT_ID = "papyrus-private-assistant";
+export const CHAT_PRIVATE_ASSISTANT_NAME = "Papyrus Assistant";
+export function chatExportName(stackName, resource) {
+    return `${stackName}:${resource}`;
+}
+function parseChat(value, cms) {
+    if (value === undefined)
+        return undefined;
+    const record = requireObject(value, "chat");
+    rejectUnknownKeys(record, ["enabled", "modelKeyParameter", "stackName", "logGroupPrefix", "workerCodeS3Key"], "chat");
+    if (typeof record.enabled !== "boolean")
+        fail("chat.enabled", "must be a boolean");
+    const chat = {
+        enabled: record.enabled,
+        modelKeyParameter: requireString(record.modelKeyParameter, "chat.modelKeyParameter", CHAT_MODEL_KEY_PARAMETER_PATTERN, "SSM parameter name such as /papyrus/<site>/openai-api-key"),
+        stackName: optionalString(record.stackName, "chat.stackName", STACK_NAME_PATTERN, "CloudFormation stack name"),
+        logGroupPrefix: optionalString(record.logGroupPrefix, "chat.logGroupPrefix", CHAT_LOG_GROUP_PREFIX_PATTERN, "/aws/lambda/<name>"),
+        workerCodeS3Key: optionalString(record.workerCodeS3Key, "chat.workerCodeS3Key", CHAT_WORKER_CODE_KEY_PATTERN, "<64 hex>.zip CDK asset key"),
+    };
+    for (const key of ["stackName", "logGroupPrefix", "workerCodeS3Key"]) {
+        if (chat[key] === undefined)
+            delete chat[key];
+    }
+    if (chat.enabled && cms === undefined)
+        fail("chat.enabled", "requires cms (the chat gateway runs in the CMS compute role)");
+    if (chat.enabled && cms) {
+        for (const key of Object.keys(cms.environment)) {
+            if (key === "PAPYRUS_PRIVATE_CHAT_ENABLED" || key.startsWith("PAPYRUS_CHAT_")) {
+                fail(`cms.environment.${key}`, "is managed by the chat block when chat is enabled: remove it and use chat.enabled");
+            }
+        }
+    }
+    return chat;
+}
 export function resolveStoragePreviewPrefix(config) {
     return config.storagePreviewPrefix ?? "preview/";
 }
@@ -355,7 +410,7 @@ function parseAws(value) {
 }
 export function parseSiteConfig(raw) {
     const record = requireObject(raw, "site");
-    rejectUnknownKeys(record, ["siteId", "repository", "repositoryAccessTokenSecretArn", "brand", "frontend", "hostedZoneId", "stackName", "aws", "cms", "email", "reader", "papyrusVersion", "storagePreviewPrefix", "knowledgeRun", "autoDeploy", "github"], "site");
+    rejectUnknownKeys(record, ["siteId", "repository", "repositoryAccessTokenSecretArn", "brand", "frontend", "hostedZoneId", "stackName", "aws", "cms", "email", "reader", "papyrusVersion", "storagePreviewPrefix", "chat", "knowledgeRun", "autoDeploy", "github"], "site");
     const siteId = requireString(record.siteId, "siteId", SITE_ID_PATTERN, "kebab-case");
     const repository = requireString(record.repository, "repository", REPOSITORY_PATTERN, "https://github.com/<org>/<repo>");
     const repositoryAccessTokenSecretArn = record.repositoryAccessTokenSecretArn === undefined
@@ -448,6 +503,7 @@ export function parseSiteConfig(raw) {
             fail("github", "ownerId and repoId must be set together");
         github = { owner, repo, branches, ciCanDeployInfra: ciCanDeployInfra, ...(ownerId !== undefined ? { ownerId, repoId } : {}) };
     }
+    const chat = parseChat(record.chat, cms);
     const hasDomain = cms?.domainName !== undefined || reader?.domainName !== undefined;
     if (hasDomain && hostedZoneId === undefined) {
         fail("hostedZoneId", "is required when cms.domainName or reader.domainName is set");
@@ -469,6 +525,7 @@ export function parseSiteConfig(raw) {
         reader,
         papyrusVersion,
         storagePreviewPrefix,
+        ...(chat ? { chat } : {}),
         autoDeploy: autoDeploy,
         github,
         ...(record.knowledgeRun !== undefined ? { knowledgeRun: record.knowledgeRun } : {}),

package/infra/site-config.ts

@@ -55,6 +55,21 @@ export type CmsConfig = {
   deployStackInBuild?: boolean;
 };
 
+export type ChatConfig = {
+  enabled: boolean;
+  /** SSM SecureString parameter holding the site's OpenAI key; only the chat run worker may read it. */
+  modelKeyParameter: string;
+  /** Chat data stack name (also its CDK construct id). Default `papyrus-chat-<siteId>`. */
+  stackName?: string;
+  /** CloudWatch log group prefix of the two workers (`<prefix>-run-worker`, `<prefix>-probe-worker`). Default `/aws/lambda/papyrus-chat-<siteId>`. */
+  logGroupPrefix?: string;
+  /**
+   * Pins the worker code to an already uploaded CDK asset object (key in the CDK assets bucket) instead of
+   * bundling and uploading new bytes. Only for adopting a stack whose workers are already verified live.
+   */
+  workerCodeS3Key?: string;
+};
+
 export type AmplifyAppShellSiteConfig = {
   siteId: string;
   repository: string;
@@ -82,6 +97,11 @@ export type AmplifyAppShellSiteConfig = {
   };
   papyrusVersion: string;
   storagePreviewPrefix?: string;
+  /**
+   * Private newsroom chat for this publication (docs/private-newsroom-chat-per-site.md). Absent or
+   * `enabled: false` means the site gets neither the chat data stack nor the chat environment on its CMS.
+   */
+  chat?: ChatConfig;
   /**
    * The scheduled cloud knowledge run for this publication. The app-shell stack ignores it; it is validated by
    * infra/scheduled-jobs/lib/knowledge-run-config.ts, which owns the schema.
@@ -153,6 +173,9 @@ const HOST_NAME_PATTERN = /^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}$/;
 const SITE_ORIGIN_PATTERN = /^https:\/\/([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}(:\d{1,5})?$/;
 const DOMAIN_PREFIX_PATTERN = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
 const APP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9 _.-]*$/;
+const CHAT_MODEL_KEY_PARAMETER_PATTERN = /^\/[A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)+$/;
+const CHAT_LOG_GROUP_PREFIX_PATTERN = /^\/aws\/lambda\/[A-Za-z0-9_.-]{1,100}$/;
+const CHAT_WORKER_CODE_KEY_PATTERN = /^[0-9a-f]{64}\.zip$/;
 const STACK_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9-]{0,127}$/;
 const BRANCH_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9/_.-]*$/;
 const PAPYRUS_VERSION_PATTERN = /^\d+\.\d+\.\d+(-[0-9A-Za-z-]+\.\d+)?$/;
@@ -339,6 +362,57 @@ export function resolveStackName(config: Pick<AmplifyAppShellSiteConfig, "siteId
   return config.stackName ?? `amplify-app-shell-${config.siteId}`;
 }
 
+/** The chat data stack of a site; its construct id and CloudFormation name. Nothing when chat is not enabled. */
+export function resolveChatStackName(config: Pick<AmplifyAppShellSiteConfig, "siteId" | "chat">): string | undefined {
+  if (!config.chat?.enabled) return undefined;
+  return config.chat.stackName ?? `papyrus-chat-${config.siteId}`;
+}
+
+export function resolveChatLogGroupPrefix(config: Pick<AmplifyAppShellSiteConfig, "siteId" | "chat">): string {
+  return config.chat?.logGroupPrefix ?? `/aws/lambda/papyrus-chat-${config.siteId}`;
+}
+
+/** CloudFormation export names the chat stack publishes and the app shell imports (no stack ever references the other directly). */
+export const CHAT_EXPORTED_RESOURCES = ["MessagingTableName", "ConversationIndexTableName", "SessionBucketName", "TurnRunsQueueUrl", "TurnProbesQueueUrl"] as const;
+export const CHAT_ENVIRONMENT_NAMES: Record<(typeof CHAT_EXPORTED_RESOURCES)[number], string> = {
+  MessagingTableName: "PAPYRUS_CHAT_MESSAGING_TABLE",
+  ConversationIndexTableName: "PAPYRUS_CHAT_CONVERSATIONS_TABLE",
+  SessionBucketName: "PAPYRUS_CHAT_SESSION_BUCKET",
+  TurnRunsQueueUrl: "PAPYRUS_CHAT_TURN_RUNS_QUEUE_URL",
+  TurnProbesQueueUrl: "PAPYRUS_CHAT_TURN_PROBES_QUEUE_URL",
+};
+export const CHAT_PRIVATE_ASSISTANT_ID = "papyrus-private-assistant";
+export const CHAT_PRIVATE_ASSISTANT_NAME = "Papyrus Assistant";
+export function chatExportName(stackName: string, resource: (typeof CHAT_EXPORTED_RESOURCES)[number]): string {
+  return `${stackName}:${resource}`;
+}
+
+function parseChat(value: unknown, cms: CmsConfig | undefined): ChatConfig | undefined {
+  if (value === undefined) return undefined;
+  const record = requireObject(value, "chat");
+  rejectUnknownKeys(record, ["enabled", "modelKeyParameter", "stackName", "logGroupPrefix", "workerCodeS3Key"], "chat");
+  if (typeof record.enabled !== "boolean") fail("chat.enabled", "must be a boolean");
+  const chat: ChatConfig = {
+    enabled: record.enabled,
+    modelKeyParameter: requireString(record.modelKeyParameter, "chat.modelKeyParameter", CHAT_MODEL_KEY_PARAMETER_PATTERN, "SSM parameter name such as /papyrus/<site>/openai-api-key"),
+    stackName: optionalString(record.stackName, "chat.stackName", STACK_NAME_PATTERN, "CloudFormation stack name"),
+    logGroupPrefix: optionalString(record.logGroupPrefix, "chat.logGroupPrefix", CHAT_LOG_GROUP_PREFIX_PATTERN, "/aws/lambda/<name>"),
+    workerCodeS3Key: optionalString(record.workerCodeS3Key, "chat.workerCodeS3Key", CHAT_WORKER_CODE_KEY_PATTERN, "<64 hex>.zip CDK asset key"),
+  };
+  for (const key of ["stackName", "logGroupPrefix", "workerCodeS3Key"] as const) {
+    if (chat[key] === undefined) delete chat[key];
+  }
+  if (chat.enabled && cms === undefined) fail("chat.enabled", "requires cms (the chat gateway runs in the CMS compute role)");
+  if (chat.enabled && cms) {
+    for (const key of Object.keys(cms.environment)) {
+      if (key === "PAPYRUS_PRIVATE_CHAT_ENABLED" || key.startsWith("PAPYRUS_CHAT_")) {
+        fail(`cms.environment.${key}`, "is managed by the chat block when chat is enabled: remove it and use chat.enabled");
+      }
+    }
+  }
+  return chat;
+}
+
 export function resolveStoragePreviewPrefix(config: Pick<AmplifyAppShellSiteConfig, "storagePreviewPrefix">): string {
   return config.storagePreviewPrefix ?? "preview/";
 }
@@ -498,7 +572,7 @@ export function parseSiteConfig(raw: unknown): AmplifyAppShellSiteConfig {
   const record = requireObject(raw, "site");
   rejectUnknownKeys(
     record,
-    ["siteId", "repository", "repositoryAccessTokenSecretArn", "brand", "frontend", "hostedZoneId", "stackName", "aws", "cms", "email", "reader", "papyrusVersion", "storagePreviewPrefix", "knowledgeRun", "autoDeploy", "github"],
+    ["siteId", "repository", "repositoryAccessTokenSecretArn", "brand", "frontend", "hostedZoneId", "stackName", "aws", "cms", "email", "reader", "papyrusVersion", "storagePreviewPrefix", "chat", "knowledgeRun", "autoDeploy", "github"],
     "site",
   );
 
@@ -590,6 +664,8 @@ export function parseSiteConfig(raw: unknown): AmplifyAppShellSiteConfig {
     github = { owner, repo, branches, ciCanDeployInfra: ciCanDeployInfra as boolean, ...(ownerId !== undefined ? { ownerId, repoId } : {}) };
   }
 
+  const chat = parseChat(record.chat, cms);
+
   const hasDomain = cms?.domainName !== undefined || reader?.domainName !== undefined;
   if (hasDomain && hostedZoneId === undefined) {
     fail("hostedZoneId", "is required when cms.domainName or reader.domainName is set");
@@ -612,6 +688,7 @@ export function parseSiteConfig(raw: unknown): AmplifyAppShellSiteConfig {
     reader,
     papyrusVersion,
     storagePreviewPrefix,
+    ...(chat ? { chat } : {}),
     autoDeploy: autoDeploy as boolean,
     github,
     ...(record.knowledgeRun !== undefined ? { knowledgeRun: record.knowledgeRun } : {}),

package/lib/category-repository.ts

@@ -656,6 +656,8 @@ export type NewsroomSummaryFacets = {
 
 export type CategorySteeringDashboard = {
   isDemo?: boolean;
+  /** Local preview only: the demo newsroom is showing a real export (scripts/export-reference-feed-fixture.py), so it must not say "synthetic". */
+  realDataPreview?: boolean;
   isPublicSkeleton?: boolean;
   summaryStatus?: "loading" | "missing" | "ready";
   summary?: NewsroomSummaryRecord | null;
@@ -820,7 +822,8 @@ function sortCategorys(categorys: CategorySteeringCategory[]): CategorySteeringC
 }
 
 function defaultNewsroomSections(importedAt: string, sectionsConfigPath?: string): NewsroomSectionRecord[] {
-  if (!sectionsConfigPath) return [];
+  // A data-driven site names a seed file it does not ship: its Sections live in the backend, so there is nothing to seed.
+  if (!sectionsConfigPath || !fs.existsSync(sectionsConfigPath)) return [];
   return loadNewsroomSectionSeedRows(sectionsConfigPath).map((row, index) => ({
     ...row,
     enabledStatus: "enabled",

package/lib/local-preview-feed-fixture.ts

@@ -0,0 +1,43 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { CategorySteeringDashboard, ReferenceAttachmentRecord, ReferenceRecord, SemanticRelationRecord } from "./category-repository";
+
+/**
+ * Local preview only (server side): when PAPYRUS_FEED_FIXTURE names a JSON file written by
+ * scripts/export-reference-feed-fixture.py, the demo newsroom shows those References, their previews and the Cyclotron's
+ * decisions instead of the invented sample. Without the variable, or without the file, nothing changes.
+ */
+export type ReferenceFeedFixture = {
+  references: ReferenceRecord[];
+  referenceAttachments: ReferenceAttachmentRecord[];
+  semanticRelations: SemanticRelationRecord[];
+};
+
+export function loadReferenceFeedFixture(env: NodeJS.ProcessEnv = process.env): ReferenceFeedFixture | null {
+  const configured = env.PAPYRUS_FEED_FIXTURE?.trim();
+  if (!configured || env.NEXT_PUBLIC_PAPYRUS_LOCAL_PREVIEW !== "1") return null;
+  try {
+    const parsed = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), configured), "utf8"));
+    if (parsed?.schema !== "papyrus-reference-feed-fixture/1" || !Array.isArray(parsed.references)) return null;
+    return {
+      references: parsed.references,
+      referenceAttachments: Array.isArray(parsed.referenceAttachments) ? parsed.referenceAttachments : [],
+      semanticRelations: Array.isArray(parsed.semanticRelations) ? parsed.semanticRelations : [],
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function withReferenceFeedFixture(dashboard: CategorySteeringDashboard): CategorySteeringDashboard {
+  const fixture = loadReferenceFeedFixture();
+  if (!fixture) return dashboard;
+  return {
+    ...dashboard,
+    references: fixture.references,
+    referenceAttachments: fixture.referenceAttachments,
+    semanticRelations: fixture.semanticRelations,
+    messages: [],
+    realDataPreview: true,
+  };
+}

package/lib/local-preview-synthetic-rejects.ts

@@ -0,0 +1,41 @@
+import type { RelevanceDecisionView } from "./relevance-decisions";
+
+/** Strong Reject confidences, strongest first, for the preview's made-up predictions. */
+const SYNTHETIC_REJECT_CONFIDENCE = [0.95, 0.91, 0.87, 0.83, 0.79, 0.75, 0.71, 0.67];
+
+/**
+ * Local preview only. A young corpus can have no Reject predictions at all, so the Reject end of the feed cannot be reviewed on real
+ * data. `?feedSynthRejects=N` turns the N weakest Accept predictions of pending items into made-up Rejects, and the page says so
+ * in a banner. Never reachable outside local preview.
+ */
+export function chooseSyntheticRejects(
+  decisions: ReadonlyMap<string, RelevanceDecisionView> | undefined,
+  pendingLineageIds: ReadonlySet<string>,
+  count: number,
+): string[] {
+  return [...(decisions ?? new Map()).entries()]
+    .filter(([lineageId, decision]) => pendingLineageIds.has(lineageId) && decision.label === (decision.positiveLabel ?? "include"))
+    .sort((left, right) => (left[1].confidence ?? 0) - (right[1].confidence ?? 0))
+    .slice(0, Math.max(0, Math.min(count, SYNTHETIC_REJECT_CONFIDENCE.length)))
+    .map(([lineageId]) => lineageId);
+}
+
+/** Applies the chosen ids, weakest first getting the lowest of the made-up confidences. The choice is made once, so rating a card never moves it. */
+export function withSyntheticRejects(
+  decisions: ReadonlyMap<string, RelevanceDecisionView> | undefined,
+  chosen: readonly string[],
+): { decisions: Map<string, RelevanceDecisionView>; changed: number } {
+  const result = new Map(decisions ?? []);
+  let changed = 0;
+  chosen.forEach((lineageId, index) => {
+    const decision = result.get(lineageId);
+    if (!decision) return;
+    result.set(lineageId, {
+      ...decision,
+      label: decision.classes.find((label) => label !== (decision.positiveLabel ?? "include")) ?? "exclude",
+      confidence: SYNTHETIC_REJECT_CONFIDENCE[index],
+    });
+    changed += 1;
+  });
+  return { decisions: result, changed };
+}

package/lib/monitoring-client.ts

@@ -3,6 +3,9 @@
 import { generateClient } from "aws-amplify/data";
 import { emailReferenceBatches } from "./monitoring-overview";
 import type {
+  EmailDecisionResult,
+  MonitoringListName,
+  MonitoringPage,
   EmailReferenceImportItem,
   EmailReferenceImportOutcome,
   MonitoringEmailArticleResolution,
@@ -10,10 +13,10 @@ import type {
   MonitoringResult,
 } from "./monitoring-overview";
 
-type QueryField = "monitoringOverview" | "monitoringEmailSubmission" | "monitoringResolveEmailArticleLink" | "intakeEmailSubmissionReferences" | "monitoringQueueEmailReference";
+type QueryField = "monitoringOverview" | "monitoringPage" | "monitoringEmailSubmission" | "monitoringDecideEmailSubmission" | "monitoringResolveEmailArticleLink" | "intakeEmailSubmissionReferences" | "monitoringQueueEmailReference";
 type GraphQLResult = { data?: Partial<Record<QueryField, unknown>>; errors?: { message?: string; errorType?: string }[] };
 
-async function query<T>(field: QueryField, queryText: string, variables: Record<string, string | number>): Promise<T | { error: "forbidden" } | { state: "unavailable" }> {
+async function query<T>(field: QueryField, queryText: string, variables: Record<string, string | number | null>): Promise<T | { error: "forbidden" } | { state: "unavailable" }> {
   const client = generateClient({ authMode: "userPool" });
   let result: GraphQLResult;
   try {
@@ -34,12 +37,36 @@ export async function loadMonitoringOverview(scope: "publication" | "all"): Prom
     `query MonitoringOverview($scope: String) { monitoringOverview(scope: $scope) }`, { scope });
 }
 
+/**
+ * One more page of one list after `cursor` (null starts the list from its newest row). `heldOnly` lists only held
+ * email submissions, so a held message is findable however old it is.
+ */
+export async function loadMonitoringPage(
+  list: MonitoringListName, cursor: string | null, scope: "publication" | "all", heldOnly = false,
+): Promise<MonitoringPage> {
+  return query<MonitoringPage>("monitoringPage",
+    `query MonitoringPage($list: String!, $cursor: String, $filter: String, $scope: String) { monitoringPage(list: $list, cursor: $cursor, filter: $filter, scope: $scope) }`,
+    { list, cursor, filter: heldOnly ? "held" : null, scope });
+}
+
 /** Loads one receipt's safe plain-text rendering and its already extracted candidate links. */
 export async function loadMonitoringEmailSubmission(receiptId: string): Promise<MonitoringEmailDetails> {
   return query<MonitoringEmailDetails>("monitoringEmailSubmission",
     `query MonitoringEmailSubmission($receiptId: String!) { monitoringEmailSubmission(receiptId: $receiptId) }`, { receiptId });
 }
 
+/**
+ * Accept or Reject one held email submission. "Accept" releases it into the normal intake and
+ * "Reject" discards it; the editor and the optional reason are recorded server-side.
+ */
+export async function decideMonitoringEmailSubmission(
+  receiptId: string, decision: "accept" | "reject", reason: string,
+): Promise<EmailDecisionResult> {
+  return query<EmailDecisionResult>("monitoringDecideEmailSubmission",
+    `mutation MonitoringDecideEmailSubmission($receiptId: String!, $decision: String!, $reason: String) { monitoringDecideEmailSubmission(receiptId: $receiptId, decision: $decision, reason: $reason) }`,
+    { receiptId, decision: decision === "accept" ? "release" : "discard", reason });
+}
+
 /** Resolve one explicitly selected Beehiiv article redirect; this may register a click with the sender. */
 export async function resolveMonitoringEmailArticleLink(receiptId: string, candidateIndex: number): Promise<MonitoringEmailArticleResolution> {
   return query<MonitoringEmailArticleResolution>("monitoringResolveEmailArticleLink",

package/lib/monitoring-overview.ts

@@ -37,9 +37,45 @@ export type MonitoringEmailSummary = {
   receiptId: string; observedAt: string | null; status: string; subject: string | null;
   envelopeSender: string | null; headerFrom: string | null; warnings: string[];
   verdicts: Record<string, string>; linkCount: number; processedCount: number; unresolvedLinkCount: number;
-  reviewRequired: boolean; errorCode: string | null;
+  reviewRequired: boolean; errorCode: string | null; quarantine: EmailQuarantine | null;
+  /** Present when the same email was received more than once: one item, every copy listed. One decision covers all copies. */
+  group?: EmailCopyGroup;
+};
+export type EmailCopyGroup = {
+  fingerprint: string; count: number;
+  copies: { receiptId: string; observedAt: string | null; status: string; linkCount: number; processedCount: number; quarantine: EmailQuarantine | null }[];
+};
+
+/** Where a held email submission stands. `null` means it never needed review. */
+export type EmailQuarantineState =
+  | "quarantined" | "release_requested" | "released" | "discard_requested" | "discarded"
+  | "expired" | "decision_failed" | "release_failed";
+export type EmailQuarantine = {
+  state: EmailQuarantineState;
+  decision: { action: "accept" | "reject"; by: string; at: string | null; reason: string; viaGroup?: boolean } | null;
+  failure: string | null;
+};
+export type EmailDecisionResult =
+  | { ok: true; already: boolean; quarantine: EmailQuarantine }
+  | { error: "forbidden" | "bad_request" | "already_decided" | "not_quarantined" | "expired"; quarantine?: EmailQuarantine }
+  | { state: "unavailable" | "not_configured" };
+
+export const QUARANTINE_LABELS: Record<EmailQuarantineState, string> = {
+  quarantined: "Held for review",
+  release_requested: "Accepted, being queued",
+  released: "Accepted",
+  discard_requested: "Rejected, being closed",
+  discarded: "Rejected",
+  expired: "Expired",
+  decision_failed: "Decision could not be applied",
+  release_failed: "Accepted, retrying",
 };
 
+/** Links and import are available only once a message is accepted (or never needed review). */
+export function emailSubmissionAccepted(quarantine: EmailQuarantine | null): boolean {
+  return quarantine === null || quarantine.state === "released";
+}
+
 export type MonitoringEmailDetails = {
   submission: MonitoringEmailSummary & { corpusId: string | null }; safeBodyText: string;
   candidateLinks: { url: string; title: string; resolutionStatus?: "unresolved" }[]; sourceMaterialIsUntrusted: true;
@@ -47,6 +83,8 @@ export type MonitoringEmailDetails = {
 
 export type MonitoringEmailArticleResolution = {
   url: string; title: string; resolved: true;
+  /** Present when the original link could not be followed and the destination was found by searching the headline. */
+  foundVia?: "search"; search?: { query: string; matchScore: number; provider: string; matchedTitle?: string };
 } | { error: "forbidden" | "bad_request" | "not_resolvable" | "resolution_failed"; reason?: string } | { state: "unavailable" };
 
 export type EmailReferenceImportItem = { candidateIndex: number; source_uri: string };
@@ -112,6 +150,17 @@ export type MonitoringFeed = {
   nextDueAt: string | null;
 };
 
+/** The lists on the page that are paged. Page sizes are the server's; there is no total. */
+export type MonitoringListName = "runs" | "routerRuns" | "items" | "emailSubmissions";
+export type MonitoringCursors = Partial<Record<MonitoringListName, string | null>>;
+export type MonitoringListRows = {
+  runs: MonitoringRun; routerRuns: MonitoringRouterRun; items: MonitoringItem; emailSubmissions: MonitoringEmailSummary;
+};
+/** One more page of one list. `cursor` is null when the list has ended. */
+export type MonitoringPage<L extends MonitoringListName = MonitoringListName> =
+  | { list: L; rows: MonitoringListRows[L][]; cursor: string | null; hiddenNotRelevant?: number }
+  | { error: "forbidden" | "bad_request" } | { state: "unavailable" | "not_configured" };
+
 export type MonitoringOverview = {
   state: "ready" | "empty";
   asOf: string;
@@ -132,6 +181,8 @@ export type MonitoringOverview = {
   otherMonitors?: string[];
   emailSubmissions?: MonitoringEmailSummary[];
   hiddenNotRelevant: number;
+  /** Where each list continues; null or missing when it has no more rows. */
+  cursors?: MonitoringCursors;
 };
 
 export type MonitoringResult =