@anthusai/papyrus 1.0.0-next.43 → 1.0.0-next.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/amplify/functions/content-actions/handler.py +54 -1
- package/amplify/functions/content-actions/requirements.txt +2 -1
- package/amplify/functions/content-actions/resource.js +6 -1
- package/amplify/functions/content-actions/resource.ts +9 -2
- package/amplify/functions/renditions/handler.py +94 -0
- package/amplify/functions/renditions/requirements.txt +4 -0
- package/amplify/functions/shared/python-bundle.js +42 -2
- package/amplify/functions/shared/python-bundle.ts +47 -2
- package/amplify/publication/redirect-and-index.js +37 -0
- package/amplify/publication/renditions-function.js +74 -0
- package/amplify/publication/renditions-function.ts +112 -0
- package/amplify/publication/renditions-settings.js +43 -0
- package/amplify/publication/renditions-settings.ts +50 -0
- package/amplify/publication/s3-projection-settings.js +63 -0
- package/amplify/publication/s3-projection-settings.ts +98 -0
- package/amplify/publication/s3-projection.js +126 -0
- package/amplify/publication/s3-projection.ts +174 -0
- package/amplify/site-backend-config.js +51 -1
- package/amplify/site-backend-config.ts +76 -0
- package/amplify/site-backend.js +27 -3
- package/amplify/site-backend.ts +27 -4
- package/package.json +1 -1
- package/routes.manifest.json +1 -1
|
@@ -6,6 +6,13 @@ from typing import Any
|
|
|
6
6
|
|
|
7
7
|
from papyrus_content.graphql_authoring import create_authoring_client
|
|
8
8
|
from papyrus_content.markus_renderer.derive import derive_body
|
|
9
|
+
from papyrus_content.projection.engine import project
|
|
10
|
+
from papyrus_content.projection.runtime import (
|
|
11
|
+
DEAD_LETTER_QUEUE_ENV,
|
|
12
|
+
context_from_environment,
|
|
13
|
+
projection_configured,
|
|
14
|
+
public_item_url,
|
|
15
|
+
)
|
|
9
16
|
from papyrus_content.rebuild_trigger import (
|
|
10
17
|
DEFAULT_STAGING_BRANCH,
|
|
11
18
|
reader_target_from_environment,
|
|
@@ -56,6 +63,45 @@ def derive_markus_action(arguments: dict) -> dict:
|
|
|
56
63
|
return response
|
|
57
64
|
|
|
58
65
|
|
|
66
|
+
def send_to_projection_dead_letter_queue(item_id: str, unpublished: bool, error: str) -> None:
|
|
67
|
+
queue_url = (os.environ.get(DEAD_LETTER_QUEUE_ENV) or "").strip()
|
|
68
|
+
if not queue_url:
|
|
69
|
+
return
|
|
70
|
+
try:
|
|
71
|
+
import boto3
|
|
72
|
+
|
|
73
|
+
boto3.client("sqs").send_message(
|
|
74
|
+
QueueUrl=queue_url,
|
|
75
|
+
MessageBody=json.dumps({"itemId": item_id, "unpublished": unpublished, "error": error}),
|
|
76
|
+
)
|
|
77
|
+
except Exception: # noqa: BLE001 - the dead letter is best effort; the response already reports the failure
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def project_item_action(client, item_id: str, *, unpublished: bool) -> dict:
|
|
82
|
+
if not projection_configured():
|
|
83
|
+
return {}
|
|
84
|
+
liveness = {} if unpublished else {"live": False}
|
|
85
|
+
try:
|
|
86
|
+
context = context_from_environment(client)
|
|
87
|
+
report = project(context, item_id, unpublished=unpublished)
|
|
88
|
+
except Exception as error: # noqa: BLE001 - a projection failure never fails the save
|
|
89
|
+
message = f"{type(error).__name__}: {error}"
|
|
90
|
+
send_to_projection_dead_letter_queue(item_id, unpublished, message)
|
|
91
|
+
return {**liveness, "projected": False, "paths": [], "error": message}
|
|
92
|
+
response = {"projected": report.ok, "paths": [*report.written, *report.deleted]}
|
|
93
|
+
if not unpublished:
|
|
94
|
+
response["live"] = report.ok
|
|
95
|
+
url = public_item_url(os.environ, report.item_paths)
|
|
96
|
+
if url:
|
|
97
|
+
response["url"] = url
|
|
98
|
+
if not report.ok:
|
|
99
|
+
message = "; ".join(report.errors)
|
|
100
|
+
send_to_projection_dead_letter_queue(item_id, unpublished, message)
|
|
101
|
+
response["error"] = message
|
|
102
|
+
return response
|
|
103
|
+
|
|
104
|
+
|
|
59
105
|
def save_item_draft_action(arguments: dict, actor: str) -> dict:
|
|
60
106
|
client, _claims = create_authoring_client()
|
|
61
107
|
fields = ItemFields(
|
|
@@ -68,7 +114,7 @@ def save_item_draft_action(arguments: dict, actor: str) -> dict:
|
|
|
68
114
|
id=arguments.get("id"),
|
|
69
115
|
)
|
|
70
116
|
record = save_item(client, fields, actor=actor, expected_content_hash=arguments.get("expectedContentHash"))
|
|
71
|
-
|
|
117
|
+
response = {
|
|
72
118
|
"ok": True,
|
|
73
119
|
"item": {
|
|
74
120
|
"id": record["id"],
|
|
@@ -78,6 +124,11 @@ def save_item_draft_action(arguments: dict, actor: str) -> dict:
|
|
|
78
124
|
"versionNumber": record.get("versionNumber"),
|
|
79
125
|
},
|
|
80
126
|
}
|
|
127
|
+
if record["status"] == "published" and projection_configured():
|
|
128
|
+
published = publish_item(client, record["id"], actor=actor)
|
|
129
|
+
response["item"]["versionNumber"] = published.version_number
|
|
130
|
+
response.update(project_item_action(client, record["id"], unpublished=False))
|
|
131
|
+
return response
|
|
81
132
|
|
|
82
133
|
|
|
83
134
|
def site_triggers_for(client, item_id: str) -> dict:
|
|
@@ -108,6 +159,7 @@ def publish_item_action(arguments: dict, actor: str) -> dict:
|
|
|
108
159
|
}
|
|
109
160
|
if result.changed:
|
|
110
161
|
response.update(site_triggers_for(client, arguments["id"]))
|
|
162
|
+
response.update(project_item_action(client, arguments["id"], unpublished=False))
|
|
111
163
|
return response
|
|
112
164
|
|
|
113
165
|
|
|
@@ -117,6 +169,7 @@ def unpublish_item_action(arguments: dict, actor: str) -> dict:
|
|
|
117
169
|
response = {"ok": True, "changed": result.changed}
|
|
118
170
|
if result.changed:
|
|
119
171
|
response.update(site_triggers_for(client, arguments["id"]))
|
|
172
|
+
response.update(project_item_action(client, arguments["id"], unpublished=True))
|
|
120
173
|
return response
|
|
121
174
|
|
|
122
175
|
|
|
@@ -7,11 +7,15 @@ import { fileURLToPath } from "node:url";
|
|
|
7
7
|
import { pythonLambdaCode } from "../shared/python-bundle.js";
|
|
8
8
|
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
9
9
|
const projectRoot = path.resolve(dirname, "../../..");
|
|
10
|
-
export const contentActionsRequirements = ["PyYAML>=6.0.2,<7", "citeproc-py>=0.8.2,<1", "anthus-markus==0.5.1", "Pillow>=
|
|
10
|
+
export const contentActionsRequirements = ["PyYAML>=6.0.2,<7", "citeproc-py>=0.8.2,<1", "anthus-markus==0.5.1", "Pillow>=11.3,<12", "awscrt>=0.23,<1"];
|
|
11
11
|
function installCheckoutRequirements(outputDir) {
|
|
12
12
|
const requirementsPath = path.join(projectRoot, "amplify/functions/content-actions/requirements.txt");
|
|
13
13
|
execSync(`python3 -m pip install -r "${requirementsPath}" -t "${outputDir}" --no-cache-dir --platform manylinux2014_aarch64 --implementation cp --python-version 3.12 --only-binary=:all: --ignore-requires-python`, { cwd: projectRoot, stdio: "inherit" });
|
|
14
14
|
}
|
|
15
|
+
let siteFilesForNextContentActionsBundle;
|
|
16
|
+
export function setSiteFilesForContentActionsBundle(siteFiles) {
|
|
17
|
+
siteFilesForNextContentActionsBundle = siteFiles;
|
|
18
|
+
}
|
|
15
19
|
export const contentActions = defineFunction((scope) => {
|
|
16
20
|
return new Function(scope, "papyrus-content-actions", {
|
|
17
21
|
runtime: Runtime.PYTHON_3_12,
|
|
@@ -25,6 +29,7 @@ export const contentActions = defineFunction((scope) => {
|
|
|
25
29
|
requirements: contentActionsRequirements,
|
|
26
30
|
noDeps: true,
|
|
27
31
|
checkoutExtras: installCheckoutRequirements,
|
|
32
|
+
...(siteFilesForNextContentActionsBundle !== undefined ? { siteFiles: siteFilesForNextContentActionsBundle } : {}),
|
|
28
33
|
}, [
|
|
29
34
|
"bash",
|
|
30
35
|
"-c",
|
|
@@ -5,12 +5,12 @@ import { Construct } from "constructs";
|
|
|
5
5
|
import { execSync } from "node:child_process";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { pythonLambdaCode } from "../shared/python-bundle";
|
|
8
|
+
import { pythonLambdaCode, type SiteBundleFile } from "../shared/python-bundle";
|
|
9
9
|
|
|
10
10
|
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
11
|
const projectRoot = path.resolve(dirname, "../../..");
|
|
12
12
|
|
|
13
|
-
export const contentActionsRequirements = ["PyYAML>=6.0.2,<7", "citeproc-py>=0.8.2,<1", "anthus-markus==0.5.1", "Pillow>=
|
|
13
|
+
export const contentActionsRequirements = ["PyYAML>=6.0.2,<7", "citeproc-py>=0.8.2,<1", "anthus-markus==0.5.1", "Pillow>=11.3,<12", "awscrt>=0.23,<1"];
|
|
14
14
|
|
|
15
15
|
function installCheckoutRequirements(outputDir: string): void {
|
|
16
16
|
const requirementsPath = path.join(projectRoot, "amplify/functions/content-actions/requirements.txt");
|
|
@@ -20,6 +20,12 @@ function installCheckoutRequirements(outputDir: string): void {
|
|
|
20
20
|
);
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
let siteFilesForNextContentActionsBundle: SiteBundleFile[] | undefined;
|
|
24
|
+
|
|
25
|
+
export function setSiteFilesForContentActionsBundle(siteFiles: SiteBundleFile[] | undefined): void {
|
|
26
|
+
siteFilesForNextContentActionsBundle = siteFiles;
|
|
27
|
+
}
|
|
28
|
+
|
|
23
29
|
export const contentActions = defineFunction(
|
|
24
30
|
(scope: Construct) => {
|
|
25
31
|
return new Function(scope, "papyrus-content-actions", {
|
|
@@ -35,6 +41,7 @@ export const contentActions = defineFunction(
|
|
|
35
41
|
requirements: contentActionsRequirements,
|
|
36
42
|
noDeps: true,
|
|
37
43
|
checkoutExtras: installCheckoutRequirements,
|
|
44
|
+
...(siteFilesForNextContentActionsBundle !== undefined ? { siteFiles: siteFilesForNextContentActionsBundle } : {}),
|
|
38
45
|
},
|
|
39
46
|
[
|
|
40
47
|
"bash",
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from papyrus_content.projection.runtime import SITE_BUCKET_ENV, SITE_STATIC_DIRECTORY_ENV
|
|
10
|
+
from papyrus_content.renditions import (
|
|
11
|
+
RenditionSettings,
|
|
12
|
+
S3RenditionStore,
|
|
13
|
+
make_renditions,
|
|
14
|
+
meta_key,
|
|
15
|
+
settings_from_static_directory,
|
|
16
|
+
source_from_media_key,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
_loaded: dict[str, Any] = {}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def configured_settings() -> RenditionSettings | None:
|
|
23
|
+
if "settings" not in _loaded:
|
|
24
|
+
static_directory = (os.environ.get(SITE_STATIC_DIRECTORY_ENV) or "").strip()
|
|
25
|
+
_loaded["settings"] = settings_from_static_directory(Path(static_directory) if static_directory else None)
|
|
26
|
+
return _loaded["settings"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def site_store() -> S3RenditionStore:
|
|
30
|
+
if "store" not in _loaded:
|
|
31
|
+
_loaded["store"] = S3RenditionStore(os.environ[SITE_BUCKET_ENV])
|
|
32
|
+
return _loaded["store"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def s3_client():
|
|
36
|
+
if "s3" not in _loaded:
|
|
37
|
+
import boto3
|
|
38
|
+
|
|
39
|
+
_loaded["s3"] = boto3.client("s3")
|
|
40
|
+
return _loaded["s3"]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def media_objects(event: dict) -> list[tuple[str, str]]:
|
|
44
|
+
if event.get("source") == "aws.s3" and "detail" in event:
|
|
45
|
+
detail = event["detail"]
|
|
46
|
+
return [(detail["bucket"]["name"], detail["object"]["key"])]
|
|
47
|
+
if "bucket" in event and "key" in event:
|
|
48
|
+
return [(event["bucket"], event["key"])]
|
|
49
|
+
raise ValueError("The event is neither an S3 object event nor a {bucket, key} request.")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def render_media_object(
|
|
53
|
+
settings: RenditionSettings, bucket: str, key: str, *, skip_existing: bool = False, dry_run: bool = False
|
|
54
|
+
) -> dict:
|
|
55
|
+
source = source_from_media_key(key)
|
|
56
|
+
if source is None:
|
|
57
|
+
return {"key": key, "action": "ignored", "reason": "not an image under media/"}
|
|
58
|
+
if skip_existing and site_store().get(meta_key(settings.pipeline, source)) is not None:
|
|
59
|
+
return {"key": key, "source": source, "action": "skipped-existing"}
|
|
60
|
+
if dry_run:
|
|
61
|
+
return {"key": key, "source": source, "action": "would-render"}
|
|
62
|
+
started = time.perf_counter()
|
|
63
|
+
data = s3_client().get_object(Bucket=bucket, Key=key)["Body"].read()
|
|
64
|
+
downloaded = time.perf_counter()
|
|
65
|
+
result = make_renditions(settings, site_store(), source, data)
|
|
66
|
+
finished = time.perf_counter()
|
|
67
|
+
return {
|
|
68
|
+
"key": key,
|
|
69
|
+
"source": source,
|
|
70
|
+
"action": "rendered",
|
|
71
|
+
"kind": result["kind"],
|
|
72
|
+
"written": result["written"],
|
|
73
|
+
"sourceBytes": len(data),
|
|
74
|
+
"downloadMilliseconds": round((downloaded - started) * 1000),
|
|
75
|
+
"renditionMilliseconds": round((finished - downloaded) * 1000),
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def handler(event, context):
|
|
80
|
+
settings = configured_settings()
|
|
81
|
+
if settings is None:
|
|
82
|
+
return {"ok": True, "results": [], "skipped": "this site has no images settings in projection.json"}
|
|
83
|
+
results = [
|
|
84
|
+
render_media_object(
|
|
85
|
+
settings,
|
|
86
|
+
bucket,
|
|
87
|
+
key,
|
|
88
|
+
skip_existing=event.get("skipExisting") is True,
|
|
89
|
+
dry_run=event.get("dryRun") is True,
|
|
90
|
+
)
|
|
91
|
+
for bucket, key in media_objects(event)
|
|
92
|
+
]
|
|
93
|
+
print(json.dumps({"renditions": results}))
|
|
94
|
+
return {"ok": True, "results": results}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AssetHashType, DockerImage } from "aws-cdk-lib";
|
|
2
2
|
import { Code, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
3
4
|
import { execFileSync } from "node:child_process";
|
|
4
5
|
import fs from "node:fs";
|
|
5
6
|
import path from "node:path";
|
|
@@ -41,6 +42,39 @@ export function toPep440(version) {
|
|
|
41
42
|
throw new Error(`Cannot map ${version} to a PEP 440 version`);
|
|
42
43
|
return m[2] === undefined ? m[1] : `${m[1]}.dev${m[2]}`;
|
|
43
44
|
}
|
|
45
|
+
function isBytecodeArtifact(sourcePath) {
|
|
46
|
+
return sourcePath.split(path.sep).includes("__pycache__") || sourcePath.endsWith(".pyc");
|
|
47
|
+
}
|
|
48
|
+
export function copySiteFiles(outputDir, siteFiles) {
|
|
49
|
+
for (const siteFile of siteFiles ?? []) {
|
|
50
|
+
const sourcePath = path.resolve(process.cwd(), siteFile.from);
|
|
51
|
+
if (!fs.existsSync(sourcePath))
|
|
52
|
+
throw new Error(`Site file to bundle does not exist: ${siteFile.from}`);
|
|
53
|
+
const destinationPath = path.join(outputDir, siteFile.to);
|
|
54
|
+
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
|
|
55
|
+
fs.cpSync(sourcePath, destinationPath, { recursive: true, filter: (candidate) => !isBytecodeArtifact(candidate) });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function listFilesRecursively(root) {
|
|
59
|
+
if (fs.statSync(root).isFile())
|
|
60
|
+
return [root];
|
|
61
|
+
return fs.readdirSync(root).sort().flatMap((entry) => listFilesRecursively(path.join(root, entry)))
|
|
62
|
+
.filter((candidate) => !isBytecodeArtifact(candidate));
|
|
63
|
+
}
|
|
64
|
+
export function siteFilesContentHash(siteFiles) {
|
|
65
|
+
const hash = createHash("sha256");
|
|
66
|
+
for (const siteFile of siteFiles ?? []) {
|
|
67
|
+
const sourcePath = path.resolve(process.cwd(), siteFile.from);
|
|
68
|
+
hash.update(`${siteFile.to}\0`);
|
|
69
|
+
if (!fs.existsSync(sourcePath))
|
|
70
|
+
continue;
|
|
71
|
+
for (const filePath of listFilesRecursively(sourcePath)) {
|
|
72
|
+
hash.update(`${path.relative(sourcePath, filePath)}\0`);
|
|
73
|
+
hash.update(fs.readFileSync(filePath));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return hash.digest("hex").slice(0, 16);
|
|
77
|
+
}
|
|
44
78
|
function pipSpec() {
|
|
45
79
|
return (process.env.PAPYRUS_PIP_SPEC ?? "").trim() || `papyrus-newsroom==${toPep440(papyrusVersion())}`;
|
|
46
80
|
}
|
|
@@ -77,6 +111,7 @@ function bundleFromPip(outputDir, options) {
|
|
|
77
111
|
], { stdio: "inherit" });
|
|
78
112
|
}
|
|
79
113
|
fs.copyFileSync(path.join(packageRoot, "amplify/functions", options.functionDir, "handler.py"), path.join(outputDir, "handler.py"));
|
|
114
|
+
copySiteFiles(outputDir, options.siteFiles);
|
|
80
115
|
if (options.corpora) {
|
|
81
116
|
const corporaDir = path.join(outputDir, "corpora");
|
|
82
117
|
fs.mkdirSync(corporaDir, { recursive: true });
|
|
@@ -96,6 +131,7 @@ export function bundleFromCheckout(outputDir, options) {
|
|
|
96
131
|
fs.cpSync(path.join(packageRoot, "src", module), path.join(outputDir, module), { recursive: true });
|
|
97
132
|
}
|
|
98
133
|
options.checkoutExtras?.(outputDir);
|
|
134
|
+
copySiteFiles(outputDir, options.siteFiles);
|
|
99
135
|
}
|
|
100
136
|
/**
|
|
101
137
|
* CDK Code for a Python Lambda. In package mode the asset hash is derived
|
|
@@ -103,14 +139,18 @@ export function bundleFromCheckout(outputDir, options) {
|
|
|
103
139
|
*/
|
|
104
140
|
export function pythonLambdaCode(options, dockerCommand) {
|
|
105
141
|
const packageMode = isPackageMode();
|
|
142
|
+
const hasSiteFiles = (options.siteFiles ?? []).length > 0;
|
|
143
|
+
const siteFilesSuffix = hasSiteFiles ? `:site-${siteFilesContentHash(options.siteFiles)}` : "";
|
|
106
144
|
return Code.fromAsset(packageRoot, {
|
|
107
145
|
...(packageMode
|
|
108
146
|
? {
|
|
109
147
|
assetHashType: AssetHashType.CUSTOM,
|
|
110
|
-
assetHash: `${options.functionDir}:${pipSpec()}:${(options.requirements ?? []).join(",")}`
|
|
148
|
+
assetHash: `${options.functionDir}:${pipSpec()}:${(options.requirements ?? []).join(",")}${siteFilesSuffix}`
|
|
111
149
|
.replace(/[^a-zA-Z0-9._:=,-]/g, "_"),
|
|
112
150
|
}
|
|
113
|
-
:
|
|
151
|
+
: hasSiteFiles
|
|
152
|
+
? { assetHashType: AssetHashType.OUTPUT }
|
|
153
|
+
: {}),
|
|
114
154
|
bundling: {
|
|
115
155
|
image: Runtime.PYTHON_3_12.bundlingImage ?? DockerImage.fromRegistry("public.ecr.aws/sam/build-python3.12"),
|
|
116
156
|
local: {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AssetHashType, DockerImage } from "aws-cdk-lib";
|
|
2
2
|
import { Code, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
3
4
|
import { execFileSync } from "node:child_process";
|
|
4
5
|
import fs from "node:fs";
|
|
5
6
|
import path from "node:path";
|
|
@@ -57,8 +58,46 @@ export type PythonBundleOptions = {
|
|
|
57
58
|
noDeps?: boolean;
|
|
58
59
|
/** Copy `<cwd>/corpora/*.yml` into the asset (publication-owned steering config). */
|
|
59
60
|
corpora?: boolean;
|
|
61
|
+
/** Site repo files copied into the asset; `from` is relative to the current working directory, `to` to the asset root. A directory is copied recursively. */
|
|
62
|
+
siteFiles?: SiteBundleFile[];
|
|
60
63
|
};
|
|
61
64
|
|
|
65
|
+
export type SiteBundleFile = { from: string; to: string };
|
|
66
|
+
|
|
67
|
+
function isBytecodeArtifact(sourcePath: string): boolean {
|
|
68
|
+
return sourcePath.split(path.sep).includes("__pycache__") || sourcePath.endsWith(".pyc");
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function copySiteFiles(outputDir: string, siteFiles: SiteBundleFile[] | undefined): void {
|
|
72
|
+
for (const siteFile of siteFiles ?? []) {
|
|
73
|
+
const sourcePath = path.resolve(process.cwd(), siteFile.from);
|
|
74
|
+
if (!fs.existsSync(sourcePath)) throw new Error(`Site file to bundle does not exist: ${siteFile.from}`);
|
|
75
|
+
const destinationPath = path.join(outputDir, siteFile.to);
|
|
76
|
+
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
|
|
77
|
+
fs.cpSync(sourcePath, destinationPath, { recursive: true, filter: (candidate) => !isBytecodeArtifact(candidate) });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function listFilesRecursively(root: string): string[] {
|
|
82
|
+
if (fs.statSync(root).isFile()) return [root];
|
|
83
|
+
return fs.readdirSync(root).sort().flatMap((entry) => listFilesRecursively(path.join(root, entry)))
|
|
84
|
+
.filter((candidate) => !isBytecodeArtifact(candidate));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function siteFilesContentHash(siteFiles: SiteBundleFile[] | undefined): string {
|
|
88
|
+
const hash = createHash("sha256");
|
|
89
|
+
for (const siteFile of siteFiles ?? []) {
|
|
90
|
+
const sourcePath = path.resolve(process.cwd(), siteFile.from);
|
|
91
|
+
hash.update(`${siteFile.to}\0`);
|
|
92
|
+
if (!fs.existsSync(sourcePath)) continue;
|
|
93
|
+
for (const filePath of listFilesRecursively(sourcePath)) {
|
|
94
|
+
hash.update(`${path.relative(sourcePath, filePath)}\0`);
|
|
95
|
+
hash.update(fs.readFileSync(filePath));
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return hash.digest("hex").slice(0, 16);
|
|
99
|
+
}
|
|
100
|
+
|
|
62
101
|
function pipSpec(): string {
|
|
63
102
|
return (process.env.PAPYRUS_PIP_SPEC ?? "").trim() || `papyrus-newsroom==${toPep440(papyrusVersion())}`;
|
|
64
103
|
}
|
|
@@ -107,6 +146,7 @@ function bundleFromPip(outputDir: string, options: PythonBundleOptions): void {
|
|
|
107
146
|
path.join(packageRoot, "amplify/functions", options.functionDir, "handler.py"),
|
|
108
147
|
path.join(outputDir, "handler.py"),
|
|
109
148
|
);
|
|
149
|
+
copySiteFiles(outputDir, options.siteFiles);
|
|
110
150
|
if (options.corpora) {
|
|
111
151
|
const corporaDir = path.join(outputDir, "corpora");
|
|
112
152
|
fs.mkdirSync(corporaDir, { recursive: true });
|
|
@@ -129,6 +169,7 @@ export function bundleFromCheckout(outputDir: string, options: PythonBundleOptio
|
|
|
129
169
|
fs.cpSync(path.join(packageRoot, "src", module), path.join(outputDir, module), { recursive: true });
|
|
130
170
|
}
|
|
131
171
|
options.checkoutExtras?.(outputDir);
|
|
172
|
+
copySiteFiles(outputDir, options.siteFiles);
|
|
132
173
|
}
|
|
133
174
|
|
|
134
175
|
/**
|
|
@@ -137,14 +178,18 @@ export function bundleFromCheckout(outputDir: string, options: PythonBundleOptio
|
|
|
137
178
|
*/
|
|
138
179
|
export function pythonLambdaCode(options: PythonBundleOptions, dockerCommand: string[]): Code {
|
|
139
180
|
const packageMode = isPackageMode();
|
|
181
|
+
const hasSiteFiles = (options.siteFiles ?? []).length > 0;
|
|
182
|
+
const siteFilesSuffix = hasSiteFiles ? `:site-${siteFilesContentHash(options.siteFiles)}` : "";
|
|
140
183
|
return Code.fromAsset(packageRoot, {
|
|
141
184
|
...(packageMode
|
|
142
185
|
? {
|
|
143
186
|
assetHashType: AssetHashType.CUSTOM,
|
|
144
|
-
assetHash: `${options.functionDir}:${pipSpec()}:${(options.requirements ?? []).join(",")}`
|
|
187
|
+
assetHash: `${options.functionDir}:${pipSpec()}:${(options.requirements ?? []).join(",")}${siteFilesSuffix}`
|
|
145
188
|
.replace(/[^a-zA-Z0-9._:=,-]/g, "_"),
|
|
146
189
|
}
|
|
147
|
-
:
|
|
190
|
+
: hasSiteFiles
|
|
191
|
+
? { assetHashType: AssetHashType.OUTPUT }
|
|
192
|
+
: {}),
|
|
148
193
|
bundling: {
|
|
149
194
|
image: Runtime.PYTHON_3_12.bundlingImage ?? DockerImage.fromRegistry("public.ecr.aws/sam/build-python3.12"),
|
|
150
195
|
local: {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import cf from "cloudfront";
|
|
2
|
+
|
|
3
|
+
const redirectStore = cf.kvs();
|
|
4
|
+
|
|
5
|
+
async function lookUpRedirectTarget(uri) {
|
|
6
|
+
try {
|
|
7
|
+
const target = await redirectStore.get(uri);
|
|
8
|
+
return target ? String(target) : null;
|
|
9
|
+
} catch (error) {
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function rewriteToIndexDocument(uri) {
|
|
15
|
+
if (uri.endsWith("/")) {
|
|
16
|
+
return uri + "index.html";
|
|
17
|
+
}
|
|
18
|
+
const lastSegment = uri.substring(uri.lastIndexOf("/") + 1);
|
|
19
|
+
if (lastSegment.indexOf(".") === -1) {
|
|
20
|
+
return uri + "/index.html";
|
|
21
|
+
}
|
|
22
|
+
return uri;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function handler(event) {
|
|
26
|
+
const request = event.request;
|
|
27
|
+
const redirectTarget = await lookUpRedirectTarget(request.uri);
|
|
28
|
+
if (redirectTarget !== null) {
|
|
29
|
+
return {
|
|
30
|
+
statusCode: 301,
|
|
31
|
+
statusDescription: "Moved Permanently",
|
|
32
|
+
headers: { location: { value: redirectTarget } },
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
request.uri = rewriteToIndexDocument(request.uri);
|
|
36
|
+
return request;
|
|
37
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { Duration, Size } from "aws-cdk-lib";
|
|
2
|
+
import * as events from "aws-cdk-lib/aws-events";
|
|
3
|
+
import * as targets from "aws-cdk-lib/aws-events-targets";
|
|
4
|
+
import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
5
|
+
import { Architecture, Function as LambdaFunction, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
6
|
+
import * as sqs from "aws-cdk-lib/aws-sqs";
|
|
7
|
+
import { execSync } from "node:child_process";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { pythonLambdaCode } from "../functions/shared/python-bundle.js";
|
|
11
|
+
import { RENDITIONS_EPHEMERAL_STORAGE_MB, RENDITIONS_MEDIA_PREFIX, RENDITIONS_MEMORY_MB, RENDITIONS_TIMEOUT_SECONDS, renditionsEnvironment, renditionsEventPattern, } from "./renditions-settings.js";
|
|
12
|
+
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const projectRoot = path.resolve(dirname, "../..");
|
|
14
|
+
export const renditionsRequirements = ["PyYAML>=6.0.2,<7", "citeproc-py>=0.8.2,<1", "anthus-markus==0.5.1", "Pillow>=11.3,<12"];
|
|
15
|
+
function installCheckoutRequirements(outputDir) {
|
|
16
|
+
const requirementsPath = path.join(projectRoot, "amplify/functions/renditions/requirements.txt");
|
|
17
|
+
execSync(`python3 -m pip install -r "${requirementsPath}" -t "${outputDir}" --no-cache-dir --platform manylinux2014_aarch64 --implementation cp --python-version 3.12 --only-binary=:all: --ignore-requires-python`, { cwd: projectRoot, stdio: "inherit" });
|
|
18
|
+
}
|
|
19
|
+
export function addRenditionsFunction(scope, props) {
|
|
20
|
+
const deadLetterQueue = new sqs.Queue(scope, "RenditionsDeadLetterQueue", {
|
|
21
|
+
encryption: sqs.QueueEncryption.SQS_MANAGED,
|
|
22
|
+
retentionPeriod: Duration.days(14),
|
|
23
|
+
enforceSSL: true,
|
|
24
|
+
});
|
|
25
|
+
const renditions = new LambdaFunction(scope, "RenditionsFunction", {
|
|
26
|
+
runtime: Runtime.PYTHON_3_12,
|
|
27
|
+
architecture: Architecture.ARM_64,
|
|
28
|
+
handler: "handler.handler",
|
|
29
|
+
timeout: Duration.seconds(RENDITIONS_TIMEOUT_SECONDS),
|
|
30
|
+
memorySize: RENDITIONS_MEMORY_MB,
|
|
31
|
+
ephemeralStorageSize: Size.mebibytes(RENDITIONS_EPHEMERAL_STORAGE_MB),
|
|
32
|
+
retryAttempts: 2,
|
|
33
|
+
deadLetterQueue,
|
|
34
|
+
environment: renditionsEnvironment(props.siteBucket.bucketName),
|
|
35
|
+
code: props.code ?? pythonLambdaCode({
|
|
36
|
+
functionDir: "renditions",
|
|
37
|
+
modules: ["papyrus_content"],
|
|
38
|
+
requirements: renditionsRequirements,
|
|
39
|
+
noDeps: true,
|
|
40
|
+
checkoutExtras: installCheckoutRequirements,
|
|
41
|
+
siteFiles: props.siteFiles,
|
|
42
|
+
}, [
|
|
43
|
+
"bash",
|
|
44
|
+
"-c",
|
|
45
|
+
[
|
|
46
|
+
"set -euo pipefail",
|
|
47
|
+
"mkdir -p /asset-output/papyrus_content",
|
|
48
|
+
"cp amplify/functions/renditions/handler.py /asset-output/handler.py",
|
|
49
|
+
"cp -R src/papyrus_content/. /asset-output/papyrus_content/",
|
|
50
|
+
"python3 -m pip install -r amplify/functions/renditions/requirements.txt -t /asset-output --no-cache-dir",
|
|
51
|
+
].join(" && "),
|
|
52
|
+
]),
|
|
53
|
+
});
|
|
54
|
+
renditions.addToRolePolicy(new PolicyStatement({
|
|
55
|
+
effect: Effect.ALLOW,
|
|
56
|
+
actions: ["s3:GetObject"],
|
|
57
|
+
resources: [props.mediaBucket.arnForObjects(`${RENDITIONS_MEDIA_PREFIX}*`)],
|
|
58
|
+
}));
|
|
59
|
+
renditions.addToRolePolicy(new PolicyStatement({
|
|
60
|
+
effect: Effect.ALLOW,
|
|
61
|
+
actions: ["s3:PutObject", "s3:GetObject"],
|
|
62
|
+
resources: [props.siteBucket.arnForObjects("*")],
|
|
63
|
+
}));
|
|
64
|
+
renditions.addToRolePolicy(new PolicyStatement({
|
|
65
|
+
effect: Effect.ALLOW,
|
|
66
|
+
actions: ["s3:ListBucket"],
|
|
67
|
+
resources: [props.siteBucket.bucketArn],
|
|
68
|
+
}));
|
|
69
|
+
const rule = new events.Rule(scope, "MediaObjectCreatedRule", {
|
|
70
|
+
eventPattern: renditionsEventPattern(props.mediaBucket.bucketName),
|
|
71
|
+
});
|
|
72
|
+
rule.addTarget(new targets.LambdaFunction(renditions, { retryAttempts: 2 }));
|
|
73
|
+
return { function: renditions, rule, deadLetterQueue };
|
|
74
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { Duration, Size } from "aws-cdk-lib";
|
|
2
|
+
import * as events from "aws-cdk-lib/aws-events";
|
|
3
|
+
import * as targets from "aws-cdk-lib/aws-events-targets";
|
|
4
|
+
import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
5
|
+
import { Architecture, type Code, Function as LambdaFunction, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
6
|
+
import * as s3 from "aws-cdk-lib/aws-s3";
|
|
7
|
+
import * as sqs from "aws-cdk-lib/aws-sqs";
|
|
8
|
+
import type { Construct } from "constructs";
|
|
9
|
+
import { execSync } from "node:child_process";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { pythonLambdaCode, type SiteBundleFile } from "../functions/shared/python-bundle";
|
|
13
|
+
import {
|
|
14
|
+
RENDITIONS_EPHEMERAL_STORAGE_MB,
|
|
15
|
+
RENDITIONS_MEDIA_PREFIX,
|
|
16
|
+
RENDITIONS_MEMORY_MB,
|
|
17
|
+
RENDITIONS_TIMEOUT_SECONDS,
|
|
18
|
+
renditionsEnvironment,
|
|
19
|
+
renditionsEventPattern,
|
|
20
|
+
} from "./renditions-settings";
|
|
21
|
+
|
|
22
|
+
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
const projectRoot = path.resolve(dirname, "../..");
|
|
24
|
+
|
|
25
|
+
export const renditionsRequirements = ["PyYAML>=6.0.2,<7", "citeproc-py>=0.8.2,<1", "anthus-markus==0.5.1", "Pillow>=11.3,<12"];
|
|
26
|
+
|
|
27
|
+
export type RenditionsFunctionProps = {
|
|
28
|
+
siteBucket: s3.IBucket;
|
|
29
|
+
mediaBucket: s3.IBucket;
|
|
30
|
+
siteFiles: SiteBundleFile[];
|
|
31
|
+
code?: Code;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type RenditionsFunction = {
|
|
35
|
+
function: LambdaFunction;
|
|
36
|
+
rule: events.Rule;
|
|
37
|
+
deadLetterQueue: sqs.Queue;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
function installCheckoutRequirements(outputDir: string): void {
|
|
41
|
+
const requirementsPath = path.join(projectRoot, "amplify/functions/renditions/requirements.txt");
|
|
42
|
+
execSync(
|
|
43
|
+
`python3 -m pip install -r "${requirementsPath}" -t "${outputDir}" --no-cache-dir --platform manylinux2014_aarch64 --implementation cp --python-version 3.12 --only-binary=:all: --ignore-requires-python`,
|
|
44
|
+
{ cwd: projectRoot, stdio: "inherit" },
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function addRenditionsFunction(scope: Construct, props: RenditionsFunctionProps): RenditionsFunction {
|
|
49
|
+
const deadLetterQueue = new sqs.Queue(scope, "RenditionsDeadLetterQueue", {
|
|
50
|
+
encryption: sqs.QueueEncryption.SQS_MANAGED,
|
|
51
|
+
retentionPeriod: Duration.days(14),
|
|
52
|
+
enforceSSL: true,
|
|
53
|
+
});
|
|
54
|
+
const renditions = new LambdaFunction(scope, "RenditionsFunction", {
|
|
55
|
+
runtime: Runtime.PYTHON_3_12,
|
|
56
|
+
architecture: Architecture.ARM_64,
|
|
57
|
+
handler: "handler.handler",
|
|
58
|
+
timeout: Duration.seconds(RENDITIONS_TIMEOUT_SECONDS),
|
|
59
|
+
memorySize: RENDITIONS_MEMORY_MB,
|
|
60
|
+
ephemeralStorageSize: Size.mebibytes(RENDITIONS_EPHEMERAL_STORAGE_MB),
|
|
61
|
+
retryAttempts: 2,
|
|
62
|
+
deadLetterQueue,
|
|
63
|
+
environment: renditionsEnvironment(props.siteBucket.bucketName),
|
|
64
|
+
code: props.code ?? pythonLambdaCode(
|
|
65
|
+
{
|
|
66
|
+
functionDir: "renditions",
|
|
67
|
+
modules: ["papyrus_content"],
|
|
68
|
+
requirements: renditionsRequirements,
|
|
69
|
+
noDeps: true,
|
|
70
|
+
checkoutExtras: installCheckoutRequirements,
|
|
71
|
+
siteFiles: props.siteFiles,
|
|
72
|
+
},
|
|
73
|
+
[
|
|
74
|
+
"bash",
|
|
75
|
+
"-c",
|
|
76
|
+
[
|
|
77
|
+
"set -euo pipefail",
|
|
78
|
+
"mkdir -p /asset-output/papyrus_content",
|
|
79
|
+
"cp amplify/functions/renditions/handler.py /asset-output/handler.py",
|
|
80
|
+
"cp -R src/papyrus_content/. /asset-output/papyrus_content/",
|
|
81
|
+
"python3 -m pip install -r amplify/functions/renditions/requirements.txt -t /asset-output --no-cache-dir",
|
|
82
|
+
].join(" && "),
|
|
83
|
+
],
|
|
84
|
+
),
|
|
85
|
+
});
|
|
86
|
+
renditions.addToRolePolicy(
|
|
87
|
+
new PolicyStatement({
|
|
88
|
+
effect: Effect.ALLOW,
|
|
89
|
+
actions: ["s3:GetObject"],
|
|
90
|
+
resources: [props.mediaBucket.arnForObjects(`${RENDITIONS_MEDIA_PREFIX}*`)],
|
|
91
|
+
}),
|
|
92
|
+
);
|
|
93
|
+
renditions.addToRolePolicy(
|
|
94
|
+
new PolicyStatement({
|
|
95
|
+
effect: Effect.ALLOW,
|
|
96
|
+
actions: ["s3:PutObject", "s3:GetObject"],
|
|
97
|
+
resources: [props.siteBucket.arnForObjects("*")],
|
|
98
|
+
}),
|
|
99
|
+
);
|
|
100
|
+
renditions.addToRolePolicy(
|
|
101
|
+
new PolicyStatement({
|
|
102
|
+
effect: Effect.ALLOW,
|
|
103
|
+
actions: ["s3:ListBucket"],
|
|
104
|
+
resources: [props.siteBucket.bucketArn],
|
|
105
|
+
}),
|
|
106
|
+
);
|
|
107
|
+
const rule = new events.Rule(scope, "MediaObjectCreatedRule", {
|
|
108
|
+
eventPattern: renditionsEventPattern(props.mediaBucket.bucketName),
|
|
109
|
+
});
|
|
110
|
+
rule.addTarget(new targets.LambdaFunction(renditions, { retryAttempts: 2 }));
|
|
111
|
+
return { function: renditions, rule, deadLetterQueue };
|
|
112
|
+
}
|