@anthusai/papyrus 5.132.0-next.2 → 5.133.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/amplify/agents/change-feeds.js +128 -0
- package/amplify/agents/change-feeds.ts +159 -0
- package/amplify/agents/item-change-feed.js +2 -14
- package/amplify/agents/item-change-feed.ts +2 -14
- package/amplify/agents/pieces.js +7 -1
- package/amplify/agents/pieces.ts +8 -2
- package/amplify/agents/stream-readers.js +89 -0
- package/amplify/agents/stream-readers.ts +106 -0
- package/amplify/functions/change-feed/handler.py +56 -0
- package/amplify/functions/reference-word-index/handler.py +6 -1
- package/amplify/functions/reference-word-index/wire.js +3 -1
- package/amplify/functions/reference-word-index/wire.ts +3 -1
- package/amplify/functions/slack-delivery/handler.py +5 -0
- package/amplify/plugins/home-page/function/handler.py +6 -0
- package/amplify/site-backend.js +52 -1
- package/amplify/site-backend.ts +50 -1
- package/package.json +1 -1
- package/routes.manifest.json +1 -1
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { Duration, Stack } from "aws-cdk-lib";
|
|
2
|
+
import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
3
|
+
import { Architecture, CfnEventSourceMapping, Function as LambdaFunction, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
4
|
+
import { pythonLambdaCode } from "../functions/shared/python-bundle.js";
|
|
5
|
+
import { connectToSiteApi } from "../site-api-access.js";
|
|
6
|
+
import { assertLiteralResourceName, sanitizeAwsName } from "../site-backend-names.js";
|
|
7
|
+
import { LISTENER_INDEX_KIND } from "./item-change-feed.js";
|
|
8
|
+
import { FEED_STREAM_VIEW, FEED_TABLES, tableStream } from "./stream-readers.js";
|
|
9
|
+
import { allowWakeSending } from "./wake-queue.js";
|
|
10
|
+
/**
|
|
11
|
+
* The change feeds for Reference, ReviewRequest, Message, LocalizationRequest and Delivery (docs/design/always-on-agents-architecture.md, slice B3, PPY-9b9e64):
|
|
12
|
+
* one function that turns each change to a row of those tables into one wake per listening agent on the site's wake queue. All the logic is
|
|
13
|
+
* src/papyrus_content/change_feeds.py (the normalized change and the wake are the Item feed's, src/papyrus_content/item_change_feed.py); the handler is
|
|
14
|
+
* amplify/functions/change-feed/handler.py. One function, not one per table: the feeds share every line of code and all they differ in is the table of the stream
|
|
15
|
+
* a record came from, which the record's stream ARN names (`PAPYRUS_CHANGE_FEED_STREAMS`). That costs the function, its role and its policy once, and one mapping
|
|
16
|
+
* for each stream the feed reads itself.
|
|
17
|
+
*
|
|
18
|
+
* Where a table's stream is read by the feed itself (`placements` says `own`) this piece holds a mapping with a filter. Where the table already has two readers
|
|
19
|
+
* (`behind`) it holds no mapping: the reader that stays on the stream passes its batch to this function (`allowChangeFeedRelay`, src/papyrus_content/change_feed_relay.py),
|
|
20
|
+
* so the stream never gets a third reader.
|
|
21
|
+
*
|
|
22
|
+
* What the function may do, and nothing else: read the streams it owns; read the site's `SiteDocument` rows (the live version of the listener index) and one
|
|
23
|
+
* Message thread by id through its own site's API, and that index's objects in the site's bucket; look up and send to its own site's wake queue. It reads no table
|
|
24
|
+
* by scan or list, receives and deletes no wake, and calls no model.
|
|
25
|
+
*/
|
|
26
|
+
export const CHANGE_FEED_API_FIELDS = ["Query/fields/getSiteDocument", "Query/fields/getMessageThread"];
|
|
27
|
+
export const CHANGE_FEED_STREAMS_ENV = "PAPYRUS_CHANGE_FEED_STREAMS";
|
|
28
|
+
export const CHANGE_FEED_FUNCTION_ENV = "PAPYRUS_CHANGE_FEED_FUNCTION";
|
|
29
|
+
export const CHANGE_FEED_RELAY_STREAMS_ENV = "PAPYRUS_CHANGE_FEED_RELAY_STREAMS";
|
|
30
|
+
const STREAM_READ_ACTIONS = ["dynamodb:DescribeStream", "dynamodb:GetRecords", "dynamodb:GetShardIterator", "dynamodb:ListStreams"];
|
|
31
|
+
export const CHANGE_FEED_NAME_SUFFIX = "-change-feed";
|
|
32
|
+
const SITE_ID_LENGTH = 40;
|
|
33
|
+
/** The feed function's name: literal, from the site's brand id, so a reader that relays to it names it without a reference into the `agents` stack. */
|
|
34
|
+
export function changeFeedFunctionName(brandId) {
|
|
35
|
+
const siteId = sanitizeAwsName(brandId, SITE_ID_LENGTH);
|
|
36
|
+
if (siteId === "")
|
|
37
|
+
throw new Error(`The change feed is named after the site's brand id, which must contain a letter or digit (got ${JSON.stringify(brandId)}).`);
|
|
38
|
+
return assertLiteralResourceName("Change feed function name", `papyrus-${siteId}${CHANGE_FEED_NAME_SUFFIX}`);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* The mapping's filter for each table: the stream records the feed can act on, so a record that cannot be a change never starts the function. The feed's own
|
|
42
|
+
* normalizer decides again (a relayed batch is not filtered), so a filter only saves invocations and never decides what a change is.
|
|
43
|
+
*/
|
|
44
|
+
export const CHANGE_FEED_FILTERS = {
|
|
45
|
+
Reference: { eventName: ["INSERT", "MODIFY"] },
|
|
46
|
+
ReviewRequest: { eventName: ["MODIFY"], dynamodb: { NewImage: { state: { S: ["answered"] } } } },
|
|
47
|
+
Message: { eventName: ["INSERT"], dynamodb: { NewImage: { messageKind: { S: ["comment"] } } } },
|
|
48
|
+
LocalizationRequest: { eventName: ["INSERT", "MODIFY"], dynamodb: { NewImage: { state: { S: ["wanted"] } } } },
|
|
49
|
+
Delivery: { eventName: ["INSERT", "MODIFY"] },
|
|
50
|
+
};
|
|
51
|
+
/** The streams of every feed table the schema has. The Reference, Message and LocalizationRequest streams are the readers' own where they exist; ReviewRequest and Delivery had none. */
|
|
52
|
+
export function enableFeedStreams(backend) {
|
|
53
|
+
const streams = {};
|
|
54
|
+
for (const table of FEED_TABLES) {
|
|
55
|
+
const arn = tableStream(backend, table, FEED_STREAM_VIEW);
|
|
56
|
+
if (arn !== undefined)
|
|
57
|
+
streams[table] = arn;
|
|
58
|
+
}
|
|
59
|
+
return streams;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Lets a stream reader pass its batches to this site's change feed: it may invoke that one function, by its literal name, and the function's name and the streams it
|
|
63
|
+
* hands over are its settings. Adds no resource. A function that reads two streams (the home page's) hands over only the ones whose feed sits behind it.
|
|
64
|
+
*/
|
|
65
|
+
export function allowChangeFeedRelay(carrier, brandId, streamArns) {
|
|
66
|
+
const stack = Stack.of(carrier);
|
|
67
|
+
carrier.addEnvironment(CHANGE_FEED_FUNCTION_ENV, changeFeedFunctionName(brandId));
|
|
68
|
+
carrier.addEnvironment(CHANGE_FEED_RELAY_STREAMS_ENV, stack.toJsonString([...streamArns]));
|
|
69
|
+
carrier.addToRolePolicy(new PolicyStatement({
|
|
70
|
+
effect: Effect.ALLOW,
|
|
71
|
+
actions: ["lambda:InvokeFunction"],
|
|
72
|
+
resources: [`arn:${stack.partition}:lambda:${stack.region}:${stack.account}:function:${changeFeedFunctionName(brandId)}`],
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
export function changeFeedsPiece(brandId, source) {
|
|
76
|
+
return {
|
|
77
|
+
id: "change-feeds",
|
|
78
|
+
add(scope) {
|
|
79
|
+
const stack = Stack.of(scope);
|
|
80
|
+
const tables = FEED_TABLES.filter((table) => source.streams[table] !== undefined);
|
|
81
|
+
const feed = new LambdaFunction(scope, "ChangeFeed", {
|
|
82
|
+
functionName: changeFeedFunctionName(brandId),
|
|
83
|
+
runtime: Runtime.PYTHON_3_12,
|
|
84
|
+
architecture: Architecture.ARM_64,
|
|
85
|
+
handler: "handler.handler",
|
|
86
|
+
timeout: Duration.seconds(60),
|
|
87
|
+
memorySize: 256,
|
|
88
|
+
description: "Puts a wake on the site's agent wake queue for each agent that listens for a change to a Reference, review request, Message, translation request or Delivery (PPY-9b9e64).",
|
|
89
|
+
code: pythonLambdaCode({ functionDir: "change-feed", modules: ["papyrus_content"], noDeps: true }, [
|
|
90
|
+
"bash",
|
|
91
|
+
"-c",
|
|
92
|
+
[
|
|
93
|
+
"set -euo pipefail",
|
|
94
|
+
"mkdir -p /asset-output/papyrus_content",
|
|
95
|
+
"cp amplify/functions/change-feed/handler.py /asset-output/handler.py",
|
|
96
|
+
"cp -R src/papyrus_content/. /asset-output/papyrus_content/",
|
|
97
|
+
].join(" && "),
|
|
98
|
+
]),
|
|
99
|
+
});
|
|
100
|
+
connectToSiteApi(feed, source.graphqlUrl);
|
|
101
|
+
feed.addEnvironment("PAPYRUS_AGENT_PUBLICATION_ID", brandId);
|
|
102
|
+
feed.addEnvironment("PAPYRUS_TENANT_ID", source.tenantId);
|
|
103
|
+
feed.addEnvironment("PAPYRUS_SITE_BUCKET", source.bucketName);
|
|
104
|
+
feed.addEnvironment(CHANGE_FEED_STREAMS_ENV, stack.toJsonString(tables.map((table) => ({ stream: source.streams[table], table }))));
|
|
105
|
+
feed.addToRolePolicy(new PolicyStatement({
|
|
106
|
+
effect: Effect.ALLOW,
|
|
107
|
+
actions: ["appsync:GraphQL"],
|
|
108
|
+
resources: CHANGE_FEED_API_FIELDS.map((field) => `arn:${stack.partition}:appsync:${stack.region}:${stack.account}:apis/${source.apiId}/types/${field}`),
|
|
109
|
+
}));
|
|
110
|
+
feed.addToRolePolicy(new PolicyStatement({ effect: Effect.ALLOW, actions: ["s3:GetObject"], resources: [`${source.bucketArn}/tenants/${source.tenantId}/site/${LISTENER_INDEX_KIND}/*`] }));
|
|
111
|
+
allowWakeSending(feed, brandId);
|
|
112
|
+
for (const table of tables.filter((candidate) => source.placements[candidate].mode === "own")) {
|
|
113
|
+
const streamArn = source.streams[table];
|
|
114
|
+
feed.addToRolePolicy(new PolicyStatement({ effect: Effect.ALLOW, actions: STREAM_READ_ACTIONS, resources: [streamArn] }));
|
|
115
|
+
new CfnEventSourceMapping(scope, `${table}StreamMapping`, {
|
|
116
|
+
eventSourceArn: streamArn,
|
|
117
|
+
functionName: feed.functionArn,
|
|
118
|
+
startingPosition: "LATEST",
|
|
119
|
+
batchSize: 10,
|
|
120
|
+
maximumBatchingWindowInSeconds: 2,
|
|
121
|
+
maximumRetryAttempts: 5,
|
|
122
|
+
functionResponseTypes: ["ReportBatchItemFailures"],
|
|
123
|
+
filterCriteria: { filters: [{ pattern: JSON.stringify(CHANGE_FEED_FILTERS[table]) }] },
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { Duration, Stack } from "aws-cdk-lib";
|
|
2
|
+
import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
3
|
+
import { Architecture, CfnEventSourceMapping, Function as LambdaFunction, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
4
|
+
import type { Construct } from "constructs";
|
|
5
|
+
import { pythonLambdaCode } from "../functions/shared/python-bundle";
|
|
6
|
+
import { connectToSiteApi } from "../site-api-access";
|
|
7
|
+
import { assertLiteralResourceName, sanitizeAwsName } from "../site-backend-names";
|
|
8
|
+
import { LISTENER_INDEX_KIND } from "./item-change-feed";
|
|
9
|
+
import { FEED_STREAM_VIEW, FEED_TABLES, tableStream, type FeedPlacement, type FeedTable, type StreamReaderId } from "./stream-readers";
|
|
10
|
+
import { allowWakeSending } from "./wake-queue";
|
|
11
|
+
import type { AgentPiece } from "./stack";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The change feeds for Reference, ReviewRequest, Message, LocalizationRequest and Delivery (docs/design/always-on-agents-architecture.md, slice B3, PPY-9b9e64):
|
|
15
|
+
* one function that turns each change to a row of those tables into one wake per listening agent on the site's wake queue. All the logic is
|
|
16
|
+
* src/papyrus_content/change_feeds.py (the normalized change and the wake are the Item feed's, src/papyrus_content/item_change_feed.py); the handler is
|
|
17
|
+
* amplify/functions/change-feed/handler.py. One function, not one per table: the feeds share every line of code and all they differ in is the table of the stream
|
|
18
|
+
* a record came from, which the record's stream ARN names (`PAPYRUS_CHANGE_FEED_STREAMS`). That costs the function, its role and its policy once, and one mapping
|
|
19
|
+
* for each stream the feed reads itself.
|
|
20
|
+
*
|
|
21
|
+
* Where a table's stream is read by the feed itself (`placements` says `own`) this piece holds a mapping with a filter. Where the table already has two readers
|
|
22
|
+
* (`behind`) it holds no mapping: the reader that stays on the stream passes its batch to this function (`allowChangeFeedRelay`, src/papyrus_content/change_feed_relay.py),
|
|
23
|
+
* so the stream never gets a third reader.
|
|
24
|
+
*
|
|
25
|
+
* What the function may do, and nothing else: read the streams it owns; read the site's `SiteDocument` rows (the live version of the listener index) and one
|
|
26
|
+
* Message thread by id through its own site's API, and that index's objects in the site's bucket; look up and send to its own site's wake queue. It reads no table
|
|
27
|
+
* by scan or list, receives and deletes no wake, and calls no model.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
export const CHANGE_FEED_API_FIELDS = ["Query/fields/getSiteDocument", "Query/fields/getMessageThread"];
|
|
31
|
+
export const CHANGE_FEED_STREAMS_ENV = "PAPYRUS_CHANGE_FEED_STREAMS";
|
|
32
|
+
export const CHANGE_FEED_FUNCTION_ENV = "PAPYRUS_CHANGE_FEED_FUNCTION";
|
|
33
|
+
export const CHANGE_FEED_RELAY_STREAMS_ENV = "PAPYRUS_CHANGE_FEED_RELAY_STREAMS";
|
|
34
|
+
const STREAM_READ_ACTIONS = ["dynamodb:DescribeStream", "dynamodb:GetRecords", "dynamodb:GetShardIterator", "dynamodb:ListStreams"];
|
|
35
|
+
export const CHANGE_FEED_NAME_SUFFIX = "-change-feed";
|
|
36
|
+
const SITE_ID_LENGTH = 40;
|
|
37
|
+
|
|
38
|
+
/** The feed function's name: literal, from the site's brand id, so a reader that relays to it names it without a reference into the `agents` stack. */
|
|
39
|
+
export function changeFeedFunctionName(brandId: string): string {
|
|
40
|
+
const siteId = sanitizeAwsName(brandId, SITE_ID_LENGTH);
|
|
41
|
+
if (siteId === "") throw new Error(`The change feed is named after the site's brand id, which must contain a letter or digit (got ${JSON.stringify(brandId)}).`);
|
|
42
|
+
return assertLiteralResourceName("Change feed function name", `papyrus-${siteId}${CHANGE_FEED_NAME_SUFFIX}`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The mapping's filter for each table: the stream records the feed can act on, so a record that cannot be a change never starts the function. The feed's own
|
|
47
|
+
* normalizer decides again (a relayed batch is not filtered), so a filter only saves invocations and never decides what a change is.
|
|
48
|
+
*/
|
|
49
|
+
export const CHANGE_FEED_FILTERS: Record<FeedTable, Record<string, unknown>> = {
|
|
50
|
+
Reference: { eventName: ["INSERT", "MODIFY"] },
|
|
51
|
+
ReviewRequest: { eventName: ["MODIFY"], dynamodb: { NewImage: { state: { S: ["answered"] } } } },
|
|
52
|
+
Message: { eventName: ["INSERT"], dynamodb: { NewImage: { messageKind: { S: ["comment"] } } } },
|
|
53
|
+
LocalizationRequest: { eventName: ["INSERT", "MODIFY"], dynamodb: { NewImage: { state: { S: ["wanted"] } } } },
|
|
54
|
+
Delivery: { eventName: ["INSERT", "MODIFY"] },
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
/** The streams the stack switched on or found: a table's stream ARN, only for a table the site's schema has. */
|
|
58
|
+
export type FeedStreams = Partial<Record<FeedTable, string>>;
|
|
59
|
+
|
|
60
|
+
export type ChangeFeedsSource = {
|
|
61
|
+
streams: FeedStreams;
|
|
62
|
+
placements: Record<FeedTable, FeedPlacement>;
|
|
63
|
+
graphqlUrl: string;
|
|
64
|
+
apiId: string;
|
|
65
|
+
tenantId: string;
|
|
66
|
+
bucketName: string;
|
|
67
|
+
bucketArn: string;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/** The streams of every feed table the schema has. The Reference, Message and LocalizationRequest streams are the readers' own where they exist; ReviewRequest and Delivery had none. */
|
|
71
|
+
export function enableFeedStreams(backend: any): FeedStreams {
|
|
72
|
+
const streams: FeedStreams = {};
|
|
73
|
+
for (const table of FEED_TABLES) {
|
|
74
|
+
const arn = tableStream(backend, table, FEED_STREAM_VIEW);
|
|
75
|
+
if (arn !== undefined) streams[table] = arn;
|
|
76
|
+
}
|
|
77
|
+
return streams;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Lets a stream reader pass its batches to this site's change feed: it may invoke that one function, by its literal name, and the function's name and the streams it
|
|
82
|
+
* hands over are its settings. Adds no resource. A function that reads two streams (the home page's) hands over only the ones whose feed sits behind it.
|
|
83
|
+
*/
|
|
84
|
+
export function allowChangeFeedRelay(carrier: LambdaFunction, brandId: string, streamArns: readonly string[]): void {
|
|
85
|
+
const stack = Stack.of(carrier);
|
|
86
|
+
carrier.addEnvironment(CHANGE_FEED_FUNCTION_ENV, changeFeedFunctionName(brandId));
|
|
87
|
+
carrier.addEnvironment(CHANGE_FEED_RELAY_STREAMS_ENV, stack.toJsonString([...streamArns]));
|
|
88
|
+
carrier.addToRolePolicy(
|
|
89
|
+
new PolicyStatement({
|
|
90
|
+
effect: Effect.ALLOW,
|
|
91
|
+
actions: ["lambda:InvokeFunction"],
|
|
92
|
+
resources: [`arn:${stack.partition}:lambda:${stack.region}:${stack.account}:function:${changeFeedFunctionName(brandId)}`],
|
|
93
|
+
}),
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export type RelayCarrier = { reader: StreamReaderId; lambda: LambdaFunction };
|
|
98
|
+
|
|
99
|
+
export function changeFeedsPiece(brandId: string, source: ChangeFeedsSource): AgentPiece {
|
|
100
|
+
return {
|
|
101
|
+
id: "change-feeds",
|
|
102
|
+
add(scope: Construct) {
|
|
103
|
+
const stack = Stack.of(scope);
|
|
104
|
+
const tables = FEED_TABLES.filter((table) => source.streams[table] !== undefined);
|
|
105
|
+
const feed = new LambdaFunction(scope, "ChangeFeed", {
|
|
106
|
+
functionName: changeFeedFunctionName(brandId),
|
|
107
|
+
runtime: Runtime.PYTHON_3_12,
|
|
108
|
+
architecture: Architecture.ARM_64,
|
|
109
|
+
handler: "handler.handler",
|
|
110
|
+
timeout: Duration.seconds(60),
|
|
111
|
+
memorySize: 256,
|
|
112
|
+
description: "Puts a wake on the site's agent wake queue for each agent that listens for a change to a Reference, review request, Message, translation request or Delivery (PPY-9b9e64).",
|
|
113
|
+
code: pythonLambdaCode(
|
|
114
|
+
{ functionDir: "change-feed", modules: ["papyrus_content"], noDeps: true },
|
|
115
|
+
[
|
|
116
|
+
"bash",
|
|
117
|
+
"-c",
|
|
118
|
+
[
|
|
119
|
+
"set -euo pipefail",
|
|
120
|
+
"mkdir -p /asset-output/papyrus_content",
|
|
121
|
+
"cp amplify/functions/change-feed/handler.py /asset-output/handler.py",
|
|
122
|
+
"cp -R src/papyrus_content/. /asset-output/papyrus_content/",
|
|
123
|
+
].join(" && "),
|
|
124
|
+
],
|
|
125
|
+
),
|
|
126
|
+
});
|
|
127
|
+
connectToSiteApi(feed, source.graphqlUrl);
|
|
128
|
+
feed.addEnvironment("PAPYRUS_AGENT_PUBLICATION_ID", brandId);
|
|
129
|
+
feed.addEnvironment("PAPYRUS_TENANT_ID", source.tenantId);
|
|
130
|
+
feed.addEnvironment("PAPYRUS_SITE_BUCKET", source.bucketName);
|
|
131
|
+
feed.addEnvironment(CHANGE_FEED_STREAMS_ENV, stack.toJsonString(tables.map((table) => ({ stream: source.streams[table], table }))));
|
|
132
|
+
feed.addToRolePolicy(
|
|
133
|
+
new PolicyStatement({
|
|
134
|
+
effect: Effect.ALLOW,
|
|
135
|
+
actions: ["appsync:GraphQL"],
|
|
136
|
+
resources: CHANGE_FEED_API_FIELDS.map((field) => `arn:${stack.partition}:appsync:${stack.region}:${stack.account}:apis/${source.apiId}/types/${field}`),
|
|
137
|
+
}),
|
|
138
|
+
);
|
|
139
|
+
feed.addToRolePolicy(
|
|
140
|
+
new PolicyStatement({ effect: Effect.ALLOW, actions: ["s3:GetObject"], resources: [`${source.bucketArn}/tenants/${source.tenantId}/site/${LISTENER_INDEX_KIND}/*`] }),
|
|
141
|
+
);
|
|
142
|
+
allowWakeSending(feed, brandId);
|
|
143
|
+
for (const table of tables.filter((candidate) => source.placements[candidate].mode === "own")) {
|
|
144
|
+
const streamArn = source.streams[table]!;
|
|
145
|
+
feed.addToRolePolicy(new PolicyStatement({ effect: Effect.ALLOW, actions: STREAM_READ_ACTIONS, resources: [streamArn] }));
|
|
146
|
+
new CfnEventSourceMapping(scope, `${table}StreamMapping`, {
|
|
147
|
+
eventSourceArn: streamArn,
|
|
148
|
+
functionName: feed.functionArn,
|
|
149
|
+
startingPosition: "LATEST",
|
|
150
|
+
batchSize: 10,
|
|
151
|
+
maximumBatchingWindowInSeconds: 2,
|
|
152
|
+
maximumRetryAttempts: 5,
|
|
153
|
+
functionResponseTypes: ["ReportBatchItemFailures"],
|
|
154
|
+
filterCriteria: { filters: [{ pattern: JSON.stringify(CHANGE_FEED_FILTERS[table]) }] },
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
}
|
|
@@ -4,6 +4,7 @@ import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
|
4
4
|
import { Architecture, CfnEventSourceMapping, Function as LambdaFunction, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
5
5
|
import { pythonLambdaCode } from "../functions/shared/python-bundle.js";
|
|
6
6
|
import { connectToSiteApi } from "../site-api-access.js";
|
|
7
|
+
import { tableStream } from "./stream-readers.js";
|
|
7
8
|
import { allowWakeSending } from "./wake-queue.js";
|
|
8
9
|
/**
|
|
9
10
|
* The Item change feed (docs/design/always-on-agents-architecture.md, slice B2, PPY-8a374e): a DynamoDB stream consumer on the Item table that turns each
|
|
@@ -27,20 +28,7 @@ const STREAM_READ_ACTIONS = ["dynamodb:DescribeStream", "dynamodb:GetRecords", "
|
|
|
27
28
|
* The Item table had no stream before. Returns undefined when the site's schema has no Item table.
|
|
28
29
|
*/
|
|
29
30
|
export function enableItemStream(backend) {
|
|
30
|
-
|
|
31
|
-
const wrapper = cfnResources.amplifyDynamoDbTables?.Item;
|
|
32
|
-
if (wrapper !== undefined) {
|
|
33
|
-
wrapper.streamSpecification = { streamViewType: ITEM_STREAM_VIEW };
|
|
34
|
-
return wrapper.resource.getAtt("TableStreamArn").toString();
|
|
35
|
-
}
|
|
36
|
-
const cfnTables = cfnResources.cfnTables ?? {};
|
|
37
|
-
const found = cfnTables.Item ??
|
|
38
|
-
cfnTables.ItemTable ??
|
|
39
|
-
Object.entries(cfnTables).find(([key]) => key.toLowerCase().replace(/table$/, "") === "item")?.[1];
|
|
40
|
-
if (!found)
|
|
41
|
-
return undefined;
|
|
42
|
-
found.streamSpecification = { streamViewType: ITEM_STREAM_VIEW };
|
|
43
|
-
return found.attrStreamArn;
|
|
31
|
+
return tableStream(backend, "Item", ITEM_STREAM_VIEW);
|
|
44
32
|
}
|
|
45
33
|
export function itemChangeFeedPiece(brandId, source) {
|
|
46
34
|
return {
|
|
@@ -5,6 +5,7 @@ import { Architecture, CfnEventSourceMapping, Function as LambdaFunction, Runtim
|
|
|
5
5
|
import type { Construct } from "constructs";
|
|
6
6
|
import { pythonLambdaCode } from "../functions/shared/python-bundle";
|
|
7
7
|
import { connectToSiteApi } from "../site-api-access";
|
|
8
|
+
import { tableStream } from "./stream-readers";
|
|
8
9
|
import { allowWakeSending } from "./wake-queue";
|
|
9
10
|
import type { AgentPiece } from "./stack";
|
|
10
11
|
|
|
@@ -43,20 +44,7 @@ export type ItemChangeFeedSource = {
|
|
|
43
44
|
* The Item table had no stream before. Returns undefined when the site's schema has no Item table.
|
|
44
45
|
*/
|
|
45
46
|
export function enableItemStream(backend: any): string | undefined {
|
|
46
|
-
|
|
47
|
-
const wrapper = cfnResources.amplifyDynamoDbTables?.Item;
|
|
48
|
-
if (wrapper !== undefined) {
|
|
49
|
-
wrapper.streamSpecification = { streamViewType: ITEM_STREAM_VIEW };
|
|
50
|
-
return wrapper.resource.getAtt("TableStreamArn").toString();
|
|
51
|
-
}
|
|
52
|
-
const cfnTables = cfnResources.cfnTables ?? {};
|
|
53
|
-
const found =
|
|
54
|
-
cfnTables.Item ??
|
|
55
|
-
cfnTables.ItemTable ??
|
|
56
|
-
Object.entries(cfnTables).find(([key]) => key.toLowerCase().replace(/table$/, "") === "item")?.[1];
|
|
57
|
-
if (!found) return undefined;
|
|
58
|
-
(found as any).streamSpecification = { streamViewType: ITEM_STREAM_VIEW };
|
|
59
|
-
return (found as any).attrStreamArn;
|
|
47
|
+
return tableStream(backend, "Item", ITEM_STREAM_VIEW);
|
|
60
48
|
}
|
|
61
49
|
|
|
62
50
|
export function itemChangeFeedPiece(brandId: string, source: ItemChangeFeedSource): AgentPiece {
|
package/amplify/agents/pieces.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { changeFeedsPiece } from "./change-feeds.js";
|
|
1
2
|
import { itemChangeFeedPiece } from "./item-change-feed.js";
|
|
2
3
|
import { scheduleGroupPiece } from "./schedule-group.js";
|
|
3
4
|
import { wakeQueuePiece } from "./wake-queue.js";
|
|
@@ -8,5 +9,10 @@ import { wakeQueuePiece } from "./wake-queue.js";
|
|
|
8
9
|
* site, so they are built for one site's brand id.
|
|
9
10
|
*/
|
|
10
11
|
export function agentPieces(brandId, sources = {}) {
|
|
11
|
-
return [
|
|
12
|
+
return [
|
|
13
|
+
wakeQueuePiece(brandId),
|
|
14
|
+
scheduleGroupPiece(brandId),
|
|
15
|
+
...(sources.itemChangeFeed ? [itemChangeFeedPiece(brandId, sources.itemChangeFeed)] : []),
|
|
16
|
+
...(sources.changeFeeds ? [changeFeedsPiece(brandId, sources.changeFeeds)] : []),
|
|
17
|
+
];
|
|
12
18
|
}
|
package/amplify/agents/pieces.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { changeFeedsPiece, type ChangeFeedsSource } from "./change-feeds";
|
|
1
2
|
import { itemChangeFeedPiece, type ItemChangeFeedSource } from "./item-change-feed";
|
|
2
3
|
import type { AgentPiece } from "./stack";
|
|
3
4
|
import { scheduleGroupPiece } from "./schedule-group";
|
|
4
5
|
import { wakeQueuePiece } from "./wake-queue";
|
|
5
6
|
|
|
6
7
|
/** What a site's backend gives the pieces that read its own resources. A piece whose source is absent is not added (a shared stack acts for no one tenant). */
|
|
7
|
-
export type AgentPieceSources = { itemChangeFeed?: ItemChangeFeedSource };
|
|
8
|
+
export type AgentPieceSources = { itemChangeFeed?: ItemChangeFeedSource; changeFeeds?: ChangeFeedsSource };
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* The agent pieces a site's `agents` stack holds. Each slice of docs/design/always-on-agents-architecture.md adds its
|
|
@@ -13,5 +14,10 @@ export type AgentPieceSources = { itemChangeFeed?: ItemChangeFeedSource };
|
|
|
13
14
|
* site, so they are built for one site's brand id.
|
|
14
15
|
*/
|
|
15
16
|
export function agentPieces(brandId: string, sources: AgentPieceSources = {}): readonly AgentPiece[] {
|
|
16
|
-
return [
|
|
17
|
+
return [
|
|
18
|
+
wakeQueuePiece(brandId),
|
|
19
|
+
scheduleGroupPiece(brandId),
|
|
20
|
+
...(sources.itemChangeFeed ? [itemChangeFeedPiece(brandId, sources.itemChangeFeed)] : []),
|
|
21
|
+
...(sources.changeFeeds ? [changeFeedsPiece(brandId, sources.changeFeeds)] : []),
|
|
22
|
+
];
|
|
17
23
|
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import * as dynamodb from "aws-cdk-lib/aws-dynamodb";
|
|
2
|
+
/**
|
|
3
|
+
* Who reads which DynamoDB stream, and where each change feed sits (docs/design/always-on-agents-architecture.md, section 8.1 and slice B3, PPY-9b9e64).
|
|
4
|
+
*
|
|
5
|
+
* A DynamoDB stream should have at most two readers [AWS guidance]. The Reference table has up to two before any feed (the word index, the home page) and the
|
|
6
|
+
* Message table up to three (the console responder, Slack delivery, the home page). So the change feed for a table is placed by a rule, in one function that
|
|
7
|
+
* the test calls with every combination of switches:
|
|
8
|
+
*
|
|
9
|
+
* - a table with fewer than two readers gets the feed as a reader of its own (an event source mapping with a filter);
|
|
10
|
+
* - a table with two or more gets no new reader: the feed sits behind one that stays on the stream, which passes its batch to the feed function and reports what
|
|
11
|
+
* the feed could not do (src/papyrus_content/change_feed_relay.py). Only a reader whose handler is Python and whose mapping reads every change the feed needs
|
|
12
|
+
* can carry it; `RELAY_CARRIERS` lists them, in the order they are preferred.
|
|
13
|
+
*
|
|
14
|
+
* This is a finding about readers that exist today, not something this fixes for them: the readers are named here only so the rule and the test can count them.
|
|
15
|
+
*/
|
|
16
|
+
/** The tables the change feeds of slice B3 cover (the Item table's feed is its own function, amplify/agents/item-change-feed.ts). */
|
|
17
|
+
export const FEED_TABLES = ["Reference", "ReviewRequest", "Message", "LocalizationRequest", "Delivery"];
|
|
18
|
+
export const MAX_STREAM_READERS = 2;
|
|
19
|
+
/** Every stream reader that is not a change feed: the table it reads. */
|
|
20
|
+
export const STREAM_READER_TABLES = {
|
|
21
|
+
"reference-word-index": "Reference",
|
|
22
|
+
"home-page-references": "Reference",
|
|
23
|
+
"console-responder": "Message",
|
|
24
|
+
"slack-delivery": "Message",
|
|
25
|
+
"home-page-threads": "Message",
|
|
26
|
+
"localization-dispatch": "LocalizationRequest",
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* The readers whose handler can carry a feed, per table, best first. The word index reads every Reference change; Slack delivery reads every Message change; the
|
|
30
|
+
* home page's thread reader reads comments, which is the only Message change the feed acts on. The home page's Reference reader sees only curated references, so it
|
|
31
|
+
* cannot carry the Reference feed. The console responder is not Python.
|
|
32
|
+
*/
|
|
33
|
+
export const RELAY_CARRIERS = {
|
|
34
|
+
Reference: ["reference-word-index"],
|
|
35
|
+
Message: ["slack-delivery", "home-page-threads"],
|
|
36
|
+
};
|
|
37
|
+
/** The readers on `table` among the `active` ones. */
|
|
38
|
+
export function readersOn(table, active) {
|
|
39
|
+
return active.filter((reader) => STREAM_READER_TABLES[reader] === table);
|
|
40
|
+
}
|
|
41
|
+
/** Where the feed for `table` sits, given the readers switched on. Throws when no reader can carry it: the feed is never added as a third reader. */
|
|
42
|
+
export function planFeed(table, active) {
|
|
43
|
+
const readers = readersOn(table, active);
|
|
44
|
+
if (readers.length < MAX_STREAM_READERS)
|
|
45
|
+
return { mode: "own" };
|
|
46
|
+
const carrier = (RELAY_CARRIERS[table] ?? []).find((candidate) => readers.includes(candidate));
|
|
47
|
+
if (carrier === undefined) {
|
|
48
|
+
throw new Error(`The ${table} stream already has ${readers.length} readers (${readers.join(", ")}) and none of them can carry the change feed; move one behind another first.`);
|
|
49
|
+
}
|
|
50
|
+
return { mode: "behind", reader: carrier };
|
|
51
|
+
}
|
|
52
|
+
export function planFeeds(active) {
|
|
53
|
+
return Object.fromEntries(FEED_TABLES.map((table) => [table, planFeed(table, active)]));
|
|
54
|
+
}
|
|
55
|
+
/** How many readers each table's stream has once the feeds are placed: the existing readers plus a feed that is a reader of its own. */
|
|
56
|
+
export function readerCounts(active, placements) {
|
|
57
|
+
const counts = {};
|
|
58
|
+
for (const reader of active)
|
|
59
|
+
counts[STREAM_READER_TABLES[reader]] = (counts[STREAM_READER_TABLES[reader]] ?? 0) + 1;
|
|
60
|
+
for (const table of FEED_TABLES)
|
|
61
|
+
if (placements[table].mode === "own")
|
|
62
|
+
counts[table] = (counts[table] ?? 0) + 1;
|
|
63
|
+
return counts;
|
|
64
|
+
}
|
|
65
|
+
/** A stream view the feed needs: both images (it compares the old and new row to tell what changed). */
|
|
66
|
+
export const FEED_STREAM_VIEW = dynamodb.StreamViewType.NEW_AND_OLD_IMAGES;
|
|
67
|
+
/**
|
|
68
|
+
* The stream of `table` (its ARN), switched on with `view` when nothing has switched it on yet, or undefined when the site's schema has no such table (an optional
|
|
69
|
+
* model group left out). A stream another feature switched on keeps its own view type: the feed needs the new image of an inserted row and, for the tables it
|
|
70
|
+
* enables itself, both.
|
|
71
|
+
*/
|
|
72
|
+
export function tableStream(backend, table, view) {
|
|
73
|
+
const cfnResources = backend.data.resources.cfnResources;
|
|
74
|
+
const wrapper = cfnResources.amplifyDynamoDbTables?.[table];
|
|
75
|
+
if (wrapper !== undefined) {
|
|
76
|
+
if (wrapper.streamSpecification?.streamViewType === undefined)
|
|
77
|
+
wrapper.streamSpecification = { streamViewType: view };
|
|
78
|
+
return wrapper.resource.getAtt("TableStreamArn").toString();
|
|
79
|
+
}
|
|
80
|
+
const cfnTables = cfnResources.cfnTables ?? {};
|
|
81
|
+
const found = cfnTables[table] ??
|
|
82
|
+
cfnTables[`${table}Table`] ??
|
|
83
|
+
Object.entries(cfnTables).find(([key]) => key.toLowerCase().replace(/table$/, "") === table.toLowerCase())?.[1];
|
|
84
|
+
if (!found)
|
|
85
|
+
return undefined;
|
|
86
|
+
if (found.streamSpecification?.streamViewType === undefined)
|
|
87
|
+
found.streamSpecification = { streamViewType: view };
|
|
88
|
+
return found.attrStreamArn;
|
|
89
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import * as dynamodb from "aws-cdk-lib/aws-dynamodb";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Who reads which DynamoDB stream, and where each change feed sits (docs/design/always-on-agents-architecture.md, section 8.1 and slice B3, PPY-9b9e64).
|
|
5
|
+
*
|
|
6
|
+
* A DynamoDB stream should have at most two readers [AWS guidance]. The Reference table has up to two before any feed (the word index, the home page) and the
|
|
7
|
+
* Message table up to three (the console responder, Slack delivery, the home page). So the change feed for a table is placed by a rule, in one function that
|
|
8
|
+
* the test calls with every combination of switches:
|
|
9
|
+
*
|
|
10
|
+
* - a table with fewer than two readers gets the feed as a reader of its own (an event source mapping with a filter);
|
|
11
|
+
* - a table with two or more gets no new reader: the feed sits behind one that stays on the stream, which passes its batch to the feed function and reports what
|
|
12
|
+
* the feed could not do (src/papyrus_content/change_feed_relay.py). Only a reader whose handler is Python and whose mapping reads every change the feed needs
|
|
13
|
+
* can carry it; `RELAY_CARRIERS` lists them, in the order they are preferred.
|
|
14
|
+
*
|
|
15
|
+
* This is a finding about readers that exist today, not something this fixes for them: the readers are named here only so the rule and the test can count them.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The tables the change feeds of slice B3 cover (the Item table's feed is its own function, amplify/agents/item-change-feed.ts). */
|
|
19
|
+
export const FEED_TABLES = ["Reference", "ReviewRequest", "Message", "LocalizationRequest", "Delivery"] as const;
|
|
20
|
+
export type FeedTable = (typeof FEED_TABLES)[number];
|
|
21
|
+
|
|
22
|
+
export const MAX_STREAM_READERS = 2;
|
|
23
|
+
|
|
24
|
+
export type StreamReaderId =
|
|
25
|
+
| "reference-word-index"
|
|
26
|
+
| "home-page-references"
|
|
27
|
+
| "console-responder"
|
|
28
|
+
| "slack-delivery"
|
|
29
|
+
| "home-page-threads"
|
|
30
|
+
| "localization-dispatch";
|
|
31
|
+
|
|
32
|
+
/** Every stream reader that is not a change feed: the table it reads. */
|
|
33
|
+
export const STREAM_READER_TABLES: Record<StreamReaderId, string> = {
|
|
34
|
+
"reference-word-index": "Reference",
|
|
35
|
+
"home-page-references": "Reference",
|
|
36
|
+
"console-responder": "Message",
|
|
37
|
+
"slack-delivery": "Message",
|
|
38
|
+
"home-page-threads": "Message",
|
|
39
|
+
"localization-dispatch": "LocalizationRequest",
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The readers whose handler can carry a feed, per table, best first. The word index reads every Reference change; Slack delivery reads every Message change; the
|
|
44
|
+
* home page's thread reader reads comments, which is the only Message change the feed acts on. The home page's Reference reader sees only curated references, so it
|
|
45
|
+
* cannot carry the Reference feed. The console responder is not Python.
|
|
46
|
+
*/
|
|
47
|
+
export const RELAY_CARRIERS: Partial<Record<FeedTable, readonly StreamReaderId[]>> = {
|
|
48
|
+
Reference: ["reference-word-index"],
|
|
49
|
+
Message: ["slack-delivery", "home-page-threads"],
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export type FeedPlacement = { mode: "own" } | { mode: "behind"; reader: StreamReaderId };
|
|
53
|
+
|
|
54
|
+
/** The readers on `table` among the `active` ones. */
|
|
55
|
+
export function readersOn(table: string, active: readonly StreamReaderId[]): StreamReaderId[] {
|
|
56
|
+
return active.filter((reader) => STREAM_READER_TABLES[reader] === table);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Where the feed for `table` sits, given the readers switched on. Throws when no reader can carry it: the feed is never added as a third reader. */
|
|
60
|
+
export function planFeed(table: FeedTable, active: readonly StreamReaderId[]): FeedPlacement {
|
|
61
|
+
const readers = readersOn(table, active);
|
|
62
|
+
if (readers.length < MAX_STREAM_READERS) return { mode: "own" };
|
|
63
|
+
const carrier = (RELAY_CARRIERS[table] ?? []).find((candidate) => readers.includes(candidate));
|
|
64
|
+
if (carrier === undefined) {
|
|
65
|
+
throw new Error(`The ${table} stream already has ${readers.length} readers (${readers.join(", ")}) and none of them can carry the change feed; move one behind another first.`);
|
|
66
|
+
}
|
|
67
|
+
return { mode: "behind", reader: carrier };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function planFeeds(active: readonly StreamReaderId[]): Record<FeedTable, FeedPlacement> {
|
|
71
|
+
return Object.fromEntries(FEED_TABLES.map((table) => [table, planFeed(table, active)])) as Record<FeedTable, FeedPlacement>;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** How many readers each table's stream has once the feeds are placed: the existing readers plus a feed that is a reader of its own. */
|
|
75
|
+
export function readerCounts(active: readonly StreamReaderId[], placements: Record<FeedTable, FeedPlacement>): Record<string, number> {
|
|
76
|
+
const counts: Record<string, number> = {};
|
|
77
|
+
for (const reader of active) counts[STREAM_READER_TABLES[reader]] = (counts[STREAM_READER_TABLES[reader]] ?? 0) + 1;
|
|
78
|
+
for (const table of FEED_TABLES) if (placements[table].mode === "own") counts[table] = (counts[table] ?? 0) + 1;
|
|
79
|
+
return counts;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** A stream view the feed needs: both images (it compares the old and new row to tell what changed). */
|
|
83
|
+
export const FEED_STREAM_VIEW = dynamodb.StreamViewType.NEW_AND_OLD_IMAGES;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The stream of `table` (its ARN), switched on with `view` when nothing has switched it on yet, or undefined when the site's schema has no such table (an optional
|
|
87
|
+
* model group left out). A stream another feature switched on keeps its own view type: the feed needs the new image of an inserted row and, for the tables it
|
|
88
|
+
* enables itself, both.
|
|
89
|
+
*/
|
|
90
|
+
export function tableStream(backend: any, table: string, view: dynamodb.StreamViewType): string | undefined {
|
|
91
|
+
const cfnResources = backend.data.resources.cfnResources;
|
|
92
|
+
const wrapper = cfnResources.amplifyDynamoDbTables?.[table];
|
|
93
|
+
if (wrapper !== undefined) {
|
|
94
|
+
if (wrapper.streamSpecification?.streamViewType === undefined) wrapper.streamSpecification = { streamViewType: view };
|
|
95
|
+
return wrapper.resource.getAtt("TableStreamArn").toString();
|
|
96
|
+
}
|
|
97
|
+
const cfnTables = cfnResources.cfnTables ?? {};
|
|
98
|
+
const found =
|
|
99
|
+
cfnTables[table] ??
|
|
100
|
+
cfnTables[`${table}Table`] ??
|
|
101
|
+
Object.entries(cfnTables).find(([key]) => key.toLowerCase().replace(/table$/, "") === table.toLowerCase())?.[1];
|
|
102
|
+
if (!found) return undefined;
|
|
103
|
+
if ((found as any).streamSpecification?.streamViewType === undefined) (found as any).streamSpecification = { streamViewType: view };
|
|
104
|
+
return (found as any).attrStreamArn;
|
|
105
|
+
}
|
|
106
|
+
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""The change feed for Reference, ReviewRequest, Message, LocalizationRequest and Delivery (PPY-9b9e64).
|
|
2
|
+
|
|
3
|
+
All the logic is ``papyrus_content.change_feeds``; this builds its collaborators with the function's own role, as the Item feed's handler does: the site's listener
|
|
4
|
+
index (one point read of the live version, the body from S3 only when the version moved), the site's wake queue, and one point read of a Message thread by id,
|
|
5
|
+
made only when some agent listens for a posted Message. Nothing here lists a table.
|
|
6
|
+
|
|
7
|
+
The records come from this function's own stream mappings or, for a table that already has two stream readers, from the reader that stays on the stream and
|
|
8
|
+
hands its batch over (``papyrus_content.change_feed_relay``).
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
_INDEX: Any = None # held across invocations of one container: the listener index re-reads the body only when the live version moves
|
|
17
|
+
_CLIENT: Any = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _client(site) -> Any:
|
|
21
|
+
global _CLIENT
|
|
22
|
+
if _CLIENT is None:
|
|
23
|
+
from papyrus_content.graphql_authoring import PapyrusGraphQLAuthoringClient
|
|
24
|
+
|
|
25
|
+
os.environ.setdefault("PAPYRUS_GRAPHQL_AUTH", "iam") # the function's own role signs the calls
|
|
26
|
+
_CLIENT = PapyrusGraphQLAuthoringClient(endpoint=site.appsync_endpoint)
|
|
27
|
+
return _CLIENT
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def listener_index(site) -> Any:
|
|
31
|
+
global _INDEX
|
|
32
|
+
if _INDEX is None:
|
|
33
|
+
import boto3
|
|
34
|
+
|
|
35
|
+
from papyrus_content.agent_listeners import ListenerIndex
|
|
36
|
+
from papyrus_content.agent_policy.run_gate import SiteCatalogue
|
|
37
|
+
from papyrus_content.site_store import SiteStore
|
|
38
|
+
|
|
39
|
+
if not (site.tenant_id and site.site_bucket):
|
|
40
|
+
raise RuntimeError("the change feed needs the site's tenant id and bucket (PAPYRUS_TENANT_ID, PAPYRUS_SITE_BUCKET) to find who listens")
|
|
41
|
+
_INDEX = ListenerIndex(SiteStore(boto3.client("s3"), site.site_bucket, SiteCatalogue(_client(site))), site.tenant_id)
|
|
42
|
+
return _INDEX
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def handler(event: dict[str, Any], context: Any = None, *, index=None, send=None, environment=None, thread_of=None) -> dict[str, Any]:
|
|
46
|
+
from papyrus_content import change_feeds, item_change_feed
|
|
47
|
+
from papyrus_content.agent_wakes import wake_sender
|
|
48
|
+
from papyrus_content.localize.dispatch import decode_image
|
|
49
|
+
|
|
50
|
+
env = environment if environment is not None else os.environ
|
|
51
|
+
site = item_change_feed.Site.from_environment(env)
|
|
52
|
+
result = change_feeds.handle(event.get("Records", []), streams=change_feeds.Streams.from_environment(env), decode=decode_image,
|
|
53
|
+
listeners=lambda: (index or listener_index(site)).document(), send=send or wake_sender(), site=site,
|
|
54
|
+
thread_of=thread_of or (lambda thread_id: _client(site).get_record("MessageThread", thread_id)))
|
|
55
|
+
print(json.dumps({"event": "papyrus_change_feed", "records": len(event.get("Records", [])), "wakesSent": result["wakesSent"], "failed": len(result["batchItemFailures"])}))
|
|
56
|
+
return {"batchItemFailures": result["batchItemFailures"]}
|
|
@@ -2,4 +2,9 @@ from papyrus_knowledge_query.reference_words import stream_handler
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
def handler(event, context=None):
|
|
5
|
-
|
|
5
|
+
"""The word index, then the change feed: where the Reference stream already has two readers the feed is not a third, this reader passes it the batch and
|
|
6
|
+
reports what the feed could not do (papyrus_content.change_feed_relay; PPY-9b9e64)."""
|
|
7
|
+
from papyrus_content.change_feed_relay import relay
|
|
8
|
+
|
|
9
|
+
failures = {item["itemIdentifier"]: item for item in [*stream_handler(event, context)["batchItemFailures"], *relay(event)]}
|
|
10
|
+
return {"batchItemFailures": list(failures.values())}
|
|
@@ -69,9 +69,11 @@ export function wireReferenceWordIndex(backend, knowledgeQueryLambda) {
|
|
|
69
69
|
"-c",
|
|
70
70
|
[
|
|
71
71
|
"set -euo pipefail",
|
|
72
|
-
"mkdir -p /asset-output/papyrus_knowledge_query",
|
|
72
|
+
"mkdir -p /asset-output/papyrus_knowledge_query /asset-output/papyrus_content",
|
|
73
73
|
"cp amplify/functions/reference-word-index/handler.py /asset-output/handler.py",
|
|
74
74
|
"cp -R src/papyrus_knowledge_query/. /asset-output/papyrus_knowledge_query/",
|
|
75
|
+
// The one file that passes a batch to the change feed (PPY-9b9e64), alone: it needs nothing else of papyrus_content.
|
|
76
|
+
"cp src/papyrus_content/change_feed_relay.py /asset-output/papyrus_content/change_feed_relay.py",
|
|
75
77
|
].join(" && "),
|
|
76
78
|
]),
|
|
77
79
|
});
|
|
@@ -81,9 +81,11 @@ export function wireReferenceWordIndex(backend: any, knowledgeQueryLambda: Lambd
|
|
|
81
81
|
"-c",
|
|
82
82
|
[
|
|
83
83
|
"set -euo pipefail",
|
|
84
|
-
"mkdir -p /asset-output/papyrus_knowledge_query",
|
|
84
|
+
"mkdir -p /asset-output/papyrus_knowledge_query /asset-output/papyrus_content",
|
|
85
85
|
"cp amplify/functions/reference-word-index/handler.py /asset-output/handler.py",
|
|
86
86
|
"cp -R src/papyrus_knowledge_query/. /asset-output/papyrus_knowledge_query/",
|
|
87
|
+
// The one file that passes a batch to the change feed (PPY-9b9e64), alone: it needs nothing else of papyrus_content.
|
|
88
|
+
"cp src/papyrus_content/change_feed_relay.py /asset-output/papyrus_content/change_feed_relay.py",
|
|
87
89
|
].join(" && "),
|
|
88
90
|
],
|
|
89
91
|
),
|
|
@@ -107,6 +107,11 @@ def handler(event: dict[str, Any], _context: Any = None) -> dict[str, Any]:
|
|
|
107
107
|
)
|
|
108
108
|
failures.append({"itemIdentifier": sequence, "error": str(error)})
|
|
109
109
|
|
|
110
|
+
# Where the Message stream already has two readers the change feed is not a third: this reader passes it the batch and reports what the feed could not do
|
|
111
|
+
# (papyrus_content.change_feed_relay; PPY-9b9e64).
|
|
112
|
+
from papyrus_content.change_feed_relay import relay
|
|
113
|
+
|
|
114
|
+
failures.extend(relay(event))
|
|
110
115
|
if failures:
|
|
111
116
|
return {"batchItemFailures": failures}
|
|
112
117
|
return {"ok": True, "results": results}
|
|
@@ -265,6 +265,12 @@ def schedule_for_changes(
|
|
|
265
265
|
|
|
266
266
|
def handler(event: Mapping[str, Any], context: Any = None, *, dynamodb_client: Any = None, scheduler_client: Any = None, now_seconds: float | None = None) -> dict[str, Any]:
|
|
267
267
|
records = event.get("Records") or []
|
|
268
|
+
# Where the Message stream already has two readers the change feed is not a third: this reader passes it the batch (papyrus_content.change_feed_relay; PPY-9b9e64).
|
|
269
|
+
# This mapping reports no single record, so a record the feed could not finish fails the batch, which is retried whole (every wake key is the same on the retry).
|
|
270
|
+
from papyrus_content.change_feed_relay import relay
|
|
271
|
+
|
|
272
|
+
if relay(event):
|
|
273
|
+
raise RuntimeError("the change feed could not finish every record of the batch")
|
|
268
274
|
changes = [change for change in (record_change(record) for record in records) if change is not None]
|
|
269
275
|
if not changes:
|
|
270
276
|
return {"records": len(records), "changed": 0, "scheduled": False}
|
package/amplify/site-backend.js
CHANGED
|
@@ -35,6 +35,8 @@ import { cognitoAuthorizeEndpoint } from "./cli-login/cognito-endpoint.js";
|
|
|
35
35
|
import { defineSiteData } from "./data/resource.js";
|
|
36
36
|
import { agentPieces } from "./agents/pieces.js";
|
|
37
37
|
import { enableItemStream } from "./agents/item-change-feed.js";
|
|
38
|
+
import { allowChangeFeedRelay, enableFeedStreams } from "./agents/change-feeds.js";
|
|
39
|
+
import { planFeeds } from "./agents/stream-readers.js";
|
|
38
40
|
import { ADMIN_AGENT_POLICY_GRANT_STACK_NAME, ADMIN_AGENT_POLICY_GROUP, addAdminAgentPolicyGrant } from "./agents/admin-policy-grant.js";
|
|
39
41
|
import { addAgentsStack } from "./agents/stack.js";
|
|
40
42
|
import { isBootstrapStage, resolveExcludedModelGroups } from "./data/schema.js";
|
|
@@ -789,7 +791,21 @@ export function defineSiteBackend(papyrusSite, options = {}) {
|
|
|
789
791
|
// The Item change feed (slice B2) reads the Item table's stream, so the stream is switched on with the stack that reads it, never in the first deploy. A dedicated site
|
|
790
792
|
// with a tenant id only: a shared stack's rows carry tenants it does not act for, and the listener index is one site's document.
|
|
791
793
|
const bootstrapStage = isBootstrapStage(process.env);
|
|
792
|
-
const
|
|
794
|
+
const feedsRun = tenancyMode !== "shared" && backendTenantId !== "" && !bootstrapStage;
|
|
795
|
+
const itemStreamArn = feedsRun ? enableItemStream(backend) : undefined;
|
|
796
|
+
// The change feeds of the other tables (slice B3) follow the same rule, and no table gets a third stream reader: the readers that exist on each stream are counted
|
|
797
|
+
// here (amplify/agents/stream-readers.ts), a table with fewer than two gets a feed of its own, and one with two gets the feed behind a reader that stays on the stream.
|
|
798
|
+
const homePagePlugin = plugins.find(({ plugin }) => plugin.manifest.name === "home-page");
|
|
799
|
+
const streamReaders = [
|
|
800
|
+
"reference-word-index",
|
|
801
|
+
...(homePagePlugin ? ["home-page-references"] : []),
|
|
802
|
+
...(homePagePlugin && homePagePlugin.config.threads ? ["home-page-threads"] : []),
|
|
803
|
+
...(enableConsoleResponder ? ["console-responder"] : []),
|
|
804
|
+
...(enableSlackAgent ? ["slack-delivery"] : []),
|
|
805
|
+
...(tenancyMode !== "shared" && backend.data.resources.tables.LocalizationRequest ? ["localization-dispatch"] : []),
|
|
806
|
+
];
|
|
807
|
+
const feedStreams = feedsRun ? enableFeedStreams(backend) : undefined;
|
|
808
|
+
const feedPlacements = planFeeds(streamReaders);
|
|
793
809
|
addAgentsStack((name) => backend.createStack(name), agentPieces(siteBackendIdentity.brandId, {
|
|
794
810
|
itemChangeFeed: itemStreamArn === undefined
|
|
795
811
|
? undefined
|
|
@@ -801,6 +817,17 @@ export function defineSiteBackend(papyrusSite, options = {}) {
|
|
|
801
817
|
bucketName: storageBucket.bucketName,
|
|
802
818
|
bucketArn: storageBucket.bucketArn,
|
|
803
819
|
},
|
|
820
|
+
changeFeeds: feedStreams === undefined
|
|
821
|
+
? undefined
|
|
822
|
+
: {
|
|
823
|
+
streams: feedStreams,
|
|
824
|
+
placements: feedPlacements,
|
|
825
|
+
graphqlUrl: backend.data.resources.cfnResources.cfnGraphqlApi.attrGraphQlUrl,
|
|
826
|
+
apiId: backend.data.resources.graphqlApi.apiId,
|
|
827
|
+
tenantId: backendTenantId,
|
|
828
|
+
bucketName: storageBucket.bucketName,
|
|
829
|
+
bucketArn: storageBucket.bucketArn,
|
|
830
|
+
},
|
|
804
831
|
}), bootstrapStage);
|
|
805
832
|
// The signed-in admin may put and get the agent policy document's versions (`papyrus agents set-rung --apply`). A dedicated site only: the grant names every
|
|
806
833
|
// tenant's prefix, which a shared stack's admin must not hold.
|
|
@@ -808,6 +835,30 @@ export function defineSiteBackend(papyrusSite, options = {}) {
|
|
|
808
835
|
addAdminAgentPolicyGrant(backend.createStack(ADMIN_AGENT_POLICY_GRANT_STACK_NAME), backend.auth.resources.groups[ADMIN_AGENT_POLICY_GROUP].role, storageBucket.bucketArn);
|
|
809
836
|
}
|
|
810
837
|
wirePlugins(backend, plugins, contentActionsLambda, s3ProjectionPublication !== undefined);
|
|
838
|
+
if (feedStreams !== undefined) {
|
|
839
|
+
// The readers that carry a feed pass their batches to it: they may invoke the one function, and it is named in their settings. No resource is added.
|
|
840
|
+
const carriers = {
|
|
841
|
+
"reference-word-index": referenceWordIndex.consumer,
|
|
842
|
+
"slack-delivery": enableSlackAgent ? backend.slackDelivery.resources.lambda : undefined,
|
|
843
|
+
"home-page-threads": homePagePlugin ? backend.homePageTrigger?.resources.lambda : undefined,
|
|
844
|
+
"home-page-references": undefined,
|
|
845
|
+
"console-responder": undefined,
|
|
846
|
+
"localization-dispatch": undefined,
|
|
847
|
+
};
|
|
848
|
+
const relayed = new Map();
|
|
849
|
+
for (const table of Object.keys(feedPlacements)) {
|
|
850
|
+
const placement = feedPlacements[table];
|
|
851
|
+
const streamArn = feedStreams[table];
|
|
852
|
+
if (placement.mode !== "behind" || streamArn === undefined)
|
|
853
|
+
continue;
|
|
854
|
+
const carrier = carriers[placement.reader];
|
|
855
|
+
if (carrier === undefined)
|
|
856
|
+
throw new Error(`The ${table} change feed sits behind ${placement.reader}, which this backend does not have.`);
|
|
857
|
+
relayed.set(carrier, [...(relayed.get(carrier) ?? []), streamArn]);
|
|
858
|
+
}
|
|
859
|
+
for (const [carrier, streamArns] of relayed)
|
|
860
|
+
allowChangeFeedRelay(carrier, siteBackendIdentity.brandId, streamArns);
|
|
861
|
+
}
|
|
811
862
|
if (tenancyMode === "shared") {
|
|
812
863
|
// The loop at the top reaches the functions Amplify defines. A shared stack also has functions that wires and plugins make with `new Function`
|
|
813
864
|
// (the console chat responder, the reference word index consumer, the public forms handler); each must know it is shared.
|
package/amplify/site-backend.ts
CHANGED
|
@@ -42,6 +42,8 @@ import { cognitoAuthorizeEndpoint } from "./cli-login/cognito-endpoint";
|
|
|
42
42
|
import { defineSiteData } from "./data/resource";
|
|
43
43
|
import { agentPieces } from "./agents/pieces";
|
|
44
44
|
import { enableItemStream } from "./agents/item-change-feed";
|
|
45
|
+
import { allowChangeFeedRelay, enableFeedStreams } from "./agents/change-feeds";
|
|
46
|
+
import { planFeeds, type FeedTable, type StreamReaderId } from "./agents/stream-readers";
|
|
45
47
|
import { ADMIN_AGENT_POLICY_GRANT_STACK_NAME, ADMIN_AGENT_POLICY_GROUP, addAdminAgentPolicyGrant } from "./agents/admin-policy-grant";
|
|
46
48
|
import { addAgentsStack } from "./agents/stack";
|
|
47
49
|
import { isBootstrapStage, resolveExcludedModelGroups } from "./data/schema";
|
|
@@ -947,7 +949,21 @@ export function defineSiteBackend(papyrusSite: PapyrusSite, options: DefineSiteB
|
|
|
947
949
|
// The Item change feed (slice B2) reads the Item table's stream, so the stream is switched on with the stack that reads it, never in the first deploy. A dedicated site
|
|
948
950
|
// with a tenant id only: a shared stack's rows carry tenants it does not act for, and the listener index is one site's document.
|
|
949
951
|
const bootstrapStage = isBootstrapStage(process.env);
|
|
950
|
-
const
|
|
952
|
+
const feedsRun = tenancyMode !== "shared" && backendTenantId !== "" && !bootstrapStage;
|
|
953
|
+
const itemStreamArn = feedsRun ? enableItemStream(backend) : undefined;
|
|
954
|
+
// The change feeds of the other tables (slice B3) follow the same rule, and no table gets a third stream reader: the readers that exist on each stream are counted
|
|
955
|
+
// here (amplify/agents/stream-readers.ts), a table with fewer than two gets a feed of its own, and one with two gets the feed behind a reader that stays on the stream.
|
|
956
|
+
const homePagePlugin = plugins.find(({ plugin }) => plugin.manifest.name === "home-page");
|
|
957
|
+
const streamReaders: StreamReaderId[] = [
|
|
958
|
+
"reference-word-index",
|
|
959
|
+
...(homePagePlugin ? (["home-page-references"] as const) : []),
|
|
960
|
+
...(homePagePlugin && (homePagePlugin.config as { threads?: boolean }).threads ? (["home-page-threads"] as const) : []),
|
|
961
|
+
...(enableConsoleResponder ? (["console-responder"] as const) : []),
|
|
962
|
+
...(enableSlackAgent ? (["slack-delivery"] as const) : []),
|
|
963
|
+
...(tenancyMode !== "shared" && backend.data.resources.tables.LocalizationRequest ? (["localization-dispatch"] as const) : []),
|
|
964
|
+
];
|
|
965
|
+
const feedStreams = feedsRun ? enableFeedStreams(backend) : undefined;
|
|
966
|
+
const feedPlacements = planFeeds(streamReaders);
|
|
951
967
|
addAgentsStack(
|
|
952
968
|
(name) => backend.createStack(name),
|
|
953
969
|
agentPieces(siteBackendIdentity.brandId, {
|
|
@@ -962,6 +978,18 @@ export function defineSiteBackend(papyrusSite: PapyrusSite, options: DefineSiteB
|
|
|
962
978
|
bucketName: storageBucket.bucketName,
|
|
963
979
|
bucketArn: storageBucket.bucketArn,
|
|
964
980
|
},
|
|
981
|
+
changeFeeds:
|
|
982
|
+
feedStreams === undefined
|
|
983
|
+
? undefined
|
|
984
|
+
: {
|
|
985
|
+
streams: feedStreams,
|
|
986
|
+
placements: feedPlacements,
|
|
987
|
+
graphqlUrl: backend.data.resources.cfnResources.cfnGraphqlApi.attrGraphQlUrl as string,
|
|
988
|
+
apiId: backend.data.resources.graphqlApi.apiId as string,
|
|
989
|
+
tenantId: backendTenantId,
|
|
990
|
+
bucketName: storageBucket.bucketName,
|
|
991
|
+
bucketArn: storageBucket.bucketArn,
|
|
992
|
+
},
|
|
965
993
|
}),
|
|
966
994
|
bootstrapStage,
|
|
967
995
|
);
|
|
@@ -975,6 +1003,27 @@ export function defineSiteBackend(papyrusSite: PapyrusSite, options: DefineSiteB
|
|
|
975
1003
|
);
|
|
976
1004
|
}
|
|
977
1005
|
wirePlugins(backend, plugins, contentActionsLambda, s3ProjectionPublication !== undefined);
|
|
1006
|
+
if (feedStreams !== undefined) {
|
|
1007
|
+
// The readers that carry a feed pass their batches to it: they may invoke the one function, and it is named in their settings. No resource is added.
|
|
1008
|
+
const carriers: Record<StreamReaderId, LambdaFunction | undefined> = {
|
|
1009
|
+
"reference-word-index": referenceWordIndex.consumer,
|
|
1010
|
+
"slack-delivery": enableSlackAgent ? (backend.slackDelivery.resources.lambda as LambdaFunction) : undefined,
|
|
1011
|
+
"home-page-threads": homePagePlugin ? ((backend as any).homePageTrigger?.resources.lambda as LambdaFunction | undefined) : undefined,
|
|
1012
|
+
"home-page-references": undefined,
|
|
1013
|
+
"console-responder": undefined,
|
|
1014
|
+
"localization-dispatch": undefined,
|
|
1015
|
+
};
|
|
1016
|
+
const relayed = new Map<LambdaFunction, string[]>();
|
|
1017
|
+
for (const table of Object.keys(feedPlacements) as FeedTable[]) {
|
|
1018
|
+
const placement = feedPlacements[table];
|
|
1019
|
+
const streamArn = feedStreams[table];
|
|
1020
|
+
if (placement.mode !== "behind" || streamArn === undefined) continue;
|
|
1021
|
+
const carrier = carriers[placement.reader];
|
|
1022
|
+
if (carrier === undefined) throw new Error(`The ${table} change feed sits behind ${placement.reader}, which this backend does not have.`);
|
|
1023
|
+
relayed.set(carrier, [...(relayed.get(carrier) ?? []), streamArn]);
|
|
1024
|
+
}
|
|
1025
|
+
for (const [carrier, streamArns] of relayed) allowChangeFeedRelay(carrier, siteBackendIdentity.brandId, streamArns);
|
|
1026
|
+
}
|
|
978
1027
|
|
|
979
1028
|
if (tenancyMode === "shared") {
|
|
980
1029
|
// The loop at the top reaches the functions Amplify defines. A shared stack also has functions that wires and plugins make with `new Function`
|
package/package.json
CHANGED
package/routes.manifest.json
CHANGED