@anthusai/papyrus 5.20.0 → 5.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/amplify/data/schema.js +11 -25
- package/amplify/data/schema.ts +13 -26
- package/amplify/functions/content-actions/handler.py +17 -2
- package/amplify/plugins/newsletter/config.js +195 -0
- package/amplify/plugins/newsletter/config.ts +240 -0
- package/amplify/plugins/newsletter/function/handler.py +6 -0
- package/amplify/plugins/newsletter/function/requirements.txt +5 -0
- package/amplify/plugins/newsletter/ops/handler.py +12 -0
- package/amplify/plugins/newsletter/ops/resource.js +27 -0
- package/amplify/plugins/newsletter/ops/resource.ts +37 -0
- package/amplify/plugins/newsletter/papyrus.plugin.js +260 -0
- package/amplify/plugins/newsletter/papyrus.plugin.ts +308 -0
- package/amplify/plugins/newsletter/schema.js +56 -0
- package/amplify/plugins/newsletter/schema.ts +61 -0
- package/amplify/plugins/newsletter/secure-string-parameter.js +107 -0
- package/amplify/plugins/newsletter/secure-string-parameter.ts +120 -0
- package/amplify/plugins/newsletter/sending.js +140 -0
- package/amplify/plugins/newsletter/sending.ts +186 -0
- package/amplify/plugins/reader-steering/config.js +52 -1
- package/amplify/plugins/reader-steering/config.ts +91 -1
- package/amplify/plugins/reader-steering/function/ai_moderation.py +230 -0
- package/amplify/plugins/reader-steering/function/handler.py +829 -10
- package/amplify/plugins/reader-steering/function/moderation-prompt.md +27 -0
- package/amplify/plugins/reader-steering/function/moderation.py +316 -0
- package/amplify/plugins/reader-steering/function/resource.js +1 -1
- package/amplify/plugins/reader-steering/function/resource.ts +1 -1
- package/amplify/plugins/reader-steering/papyrus.plugin.js +40 -4
- package/amplify/plugins/reader-steering/papyrus.plugin.ts +43 -4
- package/amplify/plugins/reader-steering/schema.js +101 -1
- package/amplify/plugins/reader-steering/schema.ts +101 -1
- package/amplify/plugins/registry.js +2 -0
- package/amplify/plugins/registry.ts +2 -0
- package/amplify/plugins/types.ts +2 -0
- package/app/api/newsletter/confirm/route.ts +13 -0
- package/app/api/newsletter/preferences/route.ts +13 -0
- package/app/api/newsletter/subscribe/route.ts +9 -0
- package/app/api/newsletter/unsubscribe/route.ts +13 -0
- package/app/api/reader/comment/delete/route.ts +9 -0
- package/app/api/reader/comment/edit/route.ts +9 -0
- package/app/api/reader/comment/route.ts +9 -0
- package/app/api/reader/report/route.ts +9 -0
- package/app/api/reader/submit/route.ts +9 -0
- package/components/moderation-panel.tsx +248 -0
- package/components/newsletter-panel.tsx +474 -0
- package/components/newsletter-sections.tsx +163 -0
- package/components/newsroom-organizer.css +14 -0
- package/components/topic-steering-workspace.tsx +42 -5
- package/lib/newsletter-api.ts +391 -0
- package/lib/newsletter-client.ts +46 -0
- package/lib/newsletter-fixture.ts +184 -0
- package/lib/newsletter-model.ts +202 -0
- package/lib/newsletter-pages.ts +264 -0
- package/lib/reader-api.ts +7 -5
- package/lib/reader-moderation.ts +220 -0
- package/package.json +19 -1
- package/routes.manifest.json +103 -1
|
@@ -12,15 +12,24 @@ development or test (docs/reader-steering.md).
|
|
|
12
12
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
|
+
import hashlib
|
|
15
16
|
import json
|
|
16
17
|
import os
|
|
17
18
|
import re
|
|
19
|
+
import sys
|
|
18
20
|
import time
|
|
19
21
|
import uuid
|
|
20
22
|
from dataclasses import dataclass
|
|
21
23
|
from datetime import datetime, timezone
|
|
22
24
|
from decimal import Decimal
|
|
23
25
|
from typing import Any, Callable, Protocol
|
|
26
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
27
|
+
|
|
28
|
+
_FUNCTION_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
|
|
29
|
+
if _FUNCTION_DIRECTORY not in sys.path: # a Lambda already has its code directory on the path; a test loading this file by path does not
|
|
30
|
+
sys.path.insert(0, _FUNCTION_DIRECTORY)
|
|
31
|
+
import ai_moderation # noqa: E402
|
|
32
|
+
import moderation as moderation_rules # noqa: E402
|
|
24
33
|
|
|
25
34
|
SITE_SUBJECT = re.compile(r"ss1_[A-Za-z0-9_-]{43}")
|
|
26
35
|
INTERIM_SUBJECT = re.compile(r"ssdev_[A-Za-z0-9_-]{43}")
|
|
@@ -42,8 +51,50 @@ ACTION_ME = "me"
|
|
|
42
51
|
ACTION_HANDLE = "handle"
|
|
43
52
|
ACTION_VOTE = "vote"
|
|
44
53
|
ACTION_RATING = "rating"
|
|
54
|
+
ACTION_SUBMIT = "submit"
|
|
55
|
+
ACTION_COMMENT = "comment"
|
|
56
|
+
ACTION_REPORT = "report"
|
|
57
|
+
RATE_BUCKET_REPORT = "report"
|
|
58
|
+
REPORT_ID_PREFIX = "rep-"
|
|
59
|
+
MAXIMUM_REPORT_REASON = 500
|
|
60
|
+
MODERATION_MODES = ("off", "human", "ai")
|
|
61
|
+
AI_WORKER = "ai-review"
|
|
62
|
+
AI_TASK = {"worker": AI_WORKER}
|
|
63
|
+
AI_DEADLINE_MARGIN_SECONDS = 12
|
|
64
|
+
ACTION_COMMENT_EDIT = "comment-edit"
|
|
65
|
+
ACTION_COMMENT_DELETE = "comment-delete"
|
|
66
|
+
RATE_BUCKET_COMMENT = "comment"
|
|
67
|
+
COMMENT_ID_PREFIX = "cmt-"
|
|
68
|
+
COMMENT_ID = re.compile(r"cmt-[0-9a-f]{32}")
|
|
69
|
+
COMMENT_VISIBLE = "visible"
|
|
70
|
+
COMMENT_HIDDEN = "hidden"
|
|
71
|
+
COMMENT_DELETED = "deleted"
|
|
72
|
+
COMMENT_PAYLOAD_KEYS = frozenset({"subject", "body", "parent", "ratingContext"})
|
|
73
|
+
COMMENT_EDIT_KEYS = frozenset({"id", "body"})
|
|
74
|
+
COMMENT_DELETE_KEYS = frozenset({"id"})
|
|
75
|
+
LINK = re.compile(r"(?:https?://|www\.)\S+", re.IGNORECASE)
|
|
76
|
+
TEXT_CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
|
45
77
|
RATE_BUCKET_REACTIONS = "react"
|
|
46
78
|
RATE_BUCKET_HANDLE = "handle"
|
|
79
|
+
RATE_BUCKET_SUBMIT = "submit"
|
|
80
|
+
DAY_SECONDS = 24 * HOUR_SECONDS
|
|
81
|
+
SUBMISSION_ID_PREFIX = "sub-"
|
|
82
|
+
SUBMISSION_ID = re.compile(r"sub-[0-9a-f]{24}")
|
|
83
|
+
SUBMISSION_DOMAIN_INDEX = "submissionsByDomain"
|
|
84
|
+
PROFILE_READER_INDEX = "profilesByReader"
|
|
85
|
+
REPORT_TARGET_INDEX = "reportsByTarget"
|
|
86
|
+
SUBMISSION_PENDING = "pending"
|
|
87
|
+
SUBMISSION_PROMOTED = "promoted"
|
|
88
|
+
SUBMISSION_HIDDEN = "hidden"
|
|
89
|
+
VISIBLE_SUBMISSION_STATES = (SUBMISSION_PENDING, SUBMISSION_PROMOTED)
|
|
90
|
+
EXTRACTION_PENDING = "pending"
|
|
91
|
+
SUBMISSION_PAYLOAD_KEYS = frozenset({"url", "title", "kind", "note"})
|
|
92
|
+
MAXIMUM_URL_LENGTH = 2048
|
|
93
|
+
MAXIMUM_TITLE_LENGTH = 200
|
|
94
|
+
MAXIMUM_NOTE_LENGTH = 500
|
|
95
|
+
MAXIMUM_SIMILAR = 5
|
|
96
|
+
EMAIL_ADDRESS = re.compile(r"[A-Za-z0-9._%+-]+@(?:[A-Za-z0-9-]*[A-Za-z][A-Za-z0-9-]*\.)+[A-Za-z]{2,}")
|
|
97
|
+
CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
|
47
98
|
SNAPSHOT_TASK = {"projectionTask": {"task": "reader-snapshot"}}
|
|
48
99
|
SNAPSHOT_SCHEDULE_PREFIX = "papyrus-reader-snapshot-"
|
|
49
100
|
RESERVED_HANDLES = frozenset(
|
|
@@ -62,6 +113,20 @@ class ReaderSettings:
|
|
|
62
113
|
reserved_handles: frozenset[str] = frozenset()
|
|
63
114
|
allow_interim_subjects: bool = False
|
|
64
115
|
snapshot_debounce_seconds: int = 60
|
|
116
|
+
submissions_per_day: int = 10
|
|
117
|
+
similar_hours: int = 48
|
|
118
|
+
submission_kinds: tuple[str, ...] = ("link",)
|
|
119
|
+
comments_per_hour: int = 30
|
|
120
|
+
comment_edit_minutes: int = 10
|
|
121
|
+
comment_maximum_length: int = 2000
|
|
122
|
+
comment_maximum_links: int = 3
|
|
123
|
+
moderation_mode: str = "human"
|
|
124
|
+
pre_moderation: bool = False
|
|
125
|
+
reports_per_day: int = 20
|
|
126
|
+
promote_item_type: str = "brief"
|
|
127
|
+
promote_section: str = "unsectioned"
|
|
128
|
+
ai_model: str = ai_moderation.DEFAULT_MODEL
|
|
129
|
+
ai_key_parameter: str = ""
|
|
65
130
|
|
|
66
131
|
|
|
67
132
|
def settings_from_environment(environment: dict[str, str] | None = None) -> ReaderSettings:
|
|
@@ -77,6 +142,20 @@ def settings_from_environment(environment: dict[str, str] | None = None) -> Read
|
|
|
77
142
|
reserved_handles=frozenset(raw.get("reservedHandles") or ()),
|
|
78
143
|
allow_interim_subjects=bool(raw.get("allowInterimSubjects", False)),
|
|
79
144
|
snapshot_debounce_seconds=int(raw.get("snapshotDebounceSeconds", defaults.snapshot_debounce_seconds)),
|
|
145
|
+
submissions_per_day=int((raw.get("submissions") or {}).get("perDay", defaults.submissions_per_day)),
|
|
146
|
+
similar_hours=int((raw.get("submissions") or {}).get("similarHours", defaults.similar_hours)),
|
|
147
|
+
submission_kinds=tuple((raw.get("submissions") or {}).get("kinds") or defaults.submission_kinds),
|
|
148
|
+
comments_per_hour=int((raw.get("comments") or {}).get("perHour", defaults.comments_per_hour)),
|
|
149
|
+
comment_edit_minutes=int((raw.get("comments") or {}).get("editWindowMinutes", defaults.comment_edit_minutes)),
|
|
150
|
+
comment_maximum_length=int((raw.get("comments") or {}).get("maximumLength", defaults.comment_maximum_length)),
|
|
151
|
+
comment_maximum_links=int((raw.get("comments") or {}).get("maximumLinks", defaults.comment_maximum_links)),
|
|
152
|
+
moderation_mode=str((raw.get("moderation") or {}).get("mode", defaults.moderation_mode)),
|
|
153
|
+
pre_moderation=bool((raw.get("moderation") or {}).get("preModeration", defaults.pre_moderation)),
|
|
154
|
+
reports_per_day=int((raw.get("moderation") or {}).get("reportsPerDay", defaults.reports_per_day)),
|
|
155
|
+
ai_model=str(((raw.get("moderation") or {}).get("ai") or {}).get("model", defaults.ai_model)),
|
|
156
|
+
ai_key_parameter=str(((raw.get("moderation") or {}).get("ai") or {}).get("apiKeyParameter", defaults.ai_key_parameter)),
|
|
157
|
+
promote_item_type=str((raw.get("submissions") or {}).get("itemType", defaults.promote_item_type)),
|
|
158
|
+
promote_section=str((raw.get("submissions") or {}).get("section", defaults.promote_section)),
|
|
80
159
|
)
|
|
81
160
|
|
|
82
161
|
|
|
@@ -95,6 +174,32 @@ class ReaderStore(Protocol):
|
|
|
95
174
|
|
|
96
175
|
def is_published(self, subject_key: str) -> bool: ...
|
|
97
176
|
|
|
177
|
+
def get_submission(self, submission_id: str) -> dict[str, Any] | None: ...
|
|
178
|
+
|
|
179
|
+
def create_submission(self, submission: dict[str, Any]) -> bool: ...
|
|
180
|
+
|
|
181
|
+
def recent_submissions_by_domain(self, domain: str, since_iso: str) -> list[dict[str, Any]]: ...
|
|
182
|
+
|
|
183
|
+
def get_comment(self, comment_id: str) -> dict[str, Any] | None: ...
|
|
184
|
+
|
|
185
|
+
def create_comment(self, comment: dict[str, Any]) -> None: ...
|
|
186
|
+
|
|
187
|
+
def update_comment(self, comment_id: str, changes: dict[str, Any], now_iso: str, *, expected_state: str) -> bool: ...
|
|
188
|
+
|
|
189
|
+
def put_report(self, report: dict[str, Any], target_key: str, target_table: str, now_iso: str) -> bool: ...
|
|
190
|
+
|
|
191
|
+
def get_profile_by_reader(self, reader_id: str) -> dict[str, Any] | None: ...
|
|
192
|
+
|
|
193
|
+
def get_action(self, action_id: str) -> dict[str, Any] | None: ...
|
|
194
|
+
|
|
195
|
+
def transact_moderation(self, action: dict[str, Any], target_type: str, target_key: str, plan: Any, *, undoes: str | None = None) -> str: ...
|
|
196
|
+
|
|
197
|
+
def close_reports(self, target: str, now_iso: str) -> int: ...
|
|
198
|
+
|
|
199
|
+
def new_content(self) -> list[tuple[str, dict[str, Any]]]: ...
|
|
200
|
+
|
|
201
|
+
def human_acted_on(self) -> set[str]: ...
|
|
202
|
+
|
|
98
203
|
def count_action(self, reader_id: str, bucket: str, window_start: int, limit: int, expires_at: int) -> bool: ...
|
|
99
204
|
|
|
100
205
|
def claim_snapshot_slot(self, now_seconds: float, debounce_seconds: int) -> int | None: ...
|
|
@@ -111,11 +216,13 @@ def refusal(status: int, error: str, message: str, **extra: Any) -> dict[str, An
|
|
|
111
216
|
|
|
112
217
|
|
|
113
218
|
def iso(now_seconds: float) -> str:
|
|
114
|
-
|
|
219
|
+
"""UTC with milliseconds, so rows written in the same second still sort in the order they were written."""
|
|
220
|
+
moment = datetime.fromtimestamp(now_seconds, tz=timezone.utc)
|
|
221
|
+
return moment.strftime("%Y-%m-%dT%H:%M:%S.") + f"{moment.microsecond // 1000:03d}Z"
|
|
115
222
|
|
|
116
223
|
|
|
117
224
|
def seconds_of(iso_text: str) -> float:
|
|
118
|
-
return datetime.strptime(iso_text, "%Y-%m-%dT%H:%M:%S
|
|
225
|
+
return datetime.strptime(iso_text, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc).timestamp()
|
|
119
226
|
|
|
120
227
|
|
|
121
228
|
def vote_id(subject_key: str, reader_id: str, dimension: str) -> str:
|
|
@@ -138,15 +245,52 @@ def action_limit(profile: dict[str, Any], settings: ReaderSettings, now_seconds:
|
|
|
138
245
|
|
|
139
246
|
|
|
140
247
|
def within_rate_limit(
|
|
141
|
-
store: ReaderStore, profile: dict[str, Any], bucket: str, limit: int, now_seconds: float
|
|
248
|
+
store: ReaderStore, profile: dict[str, Any], bucket: str, limit: int, now_seconds: float, window_seconds: int = HOUR_SECONDS
|
|
142
249
|
) -> dict[str, Any] | None:
|
|
143
|
-
window_start = int(now_seconds //
|
|
144
|
-
window_end = window_start +
|
|
250
|
+
window_start = int(now_seconds // window_seconds) * window_seconds
|
|
251
|
+
window_end = window_start + window_seconds
|
|
145
252
|
if store.count_action(profile["readerId"], bucket, window_start, limit, window_end + HOUR_SECONDS):
|
|
146
253
|
return None
|
|
147
254
|
return refusal(429, "rate_limited", "Too many actions. Try again later.", retryAfter=max(1, int(window_end - now_seconds)))
|
|
148
255
|
|
|
149
256
|
|
|
257
|
+
def new_account_limit(profile: dict[str, Any], settings: ReaderSettings, now_seconds: float, full_limit: int) -> int:
|
|
258
|
+
"""The limit for an action: a third (``new_account_divisor``) of it while the profile is younger than ``new_account_hours``."""
|
|
259
|
+
age_hours = (now_seconds - seconds_of(profile["createdAt"])) / HOUR_SECONDS
|
|
260
|
+
return max(1, full_limit // settings.new_account_divisor) if age_hours < settings.new_account_hours else full_limit
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class InvalidUrl(ValueError):
|
|
264
|
+
pass
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def canonical_source_url(value: Any) -> str:
|
|
268
|
+
"""The canonical form of a source URL. Identical to ``papyrus_content.reference_intake.canonical_source_url`` (a test pins them together)."""
|
|
269
|
+
if not isinstance(value, str) or len(value) > 4096 or any(ord(c) <= 32 for c in value):
|
|
270
|
+
raise InvalidUrl("source_uri must be an HTTP(S) URL without whitespace.")
|
|
271
|
+
try:
|
|
272
|
+
parts = urlsplit(value)
|
|
273
|
+
if parts.scheme.lower() not in {"https", "http"} or not parts.hostname or parts.username or parts.password:
|
|
274
|
+
raise ValueError()
|
|
275
|
+
host = parts.hostname.encode("idna").decode("ascii").lower()
|
|
276
|
+
if ":" in host:
|
|
277
|
+
host = f"[{host}]"
|
|
278
|
+
port = parts.port
|
|
279
|
+
if port is not None and port != {"http": 80, "https": 443}[parts.scheme.lower()]:
|
|
280
|
+
host += f":{port}"
|
|
281
|
+
return urlunsplit((parts.scheme.lower(), host, parts.path or "/", parts.query, ""))
|
|
282
|
+
except (ValueError, UnicodeError) as error:
|
|
283
|
+
raise InvalidUrl("source_uri must be a valid HTTP(S) URL without credentials.") from error
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def submission_id_for(canonical_url: str) -> str:
|
|
287
|
+
return SUBMISSION_ID_PREFIX + hashlib.sha256(canonical_url.encode("utf-8")).hexdigest()[:24]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def domain_of(canonical_url: str) -> str:
|
|
291
|
+
return (urlsplit(canonical_url).hostname or "").lower()
|
|
292
|
+
|
|
293
|
+
|
|
150
294
|
def ensure_profile(store: ReaderStore, site_subject: str, now_seconds: float) -> dict[str, Any]:
|
|
151
295
|
profile = store.get_profile(site_subject)
|
|
152
296
|
if profile is not None:
|
|
@@ -242,7 +386,7 @@ def react(
|
|
|
242
386
|
limited = within_rate_limit(store, profile, RATE_BUCKET_REACTIONS, action_limit(profile, settings, now_seconds), now_seconds)
|
|
243
387
|
if limited:
|
|
244
388
|
return limited
|
|
245
|
-
if not store
|
|
389
|
+
if not subject_is_open(store, subject):
|
|
246
390
|
return refusal(404, "unknown_subject", "That item is not published.")
|
|
247
391
|
stored_id = vote_id(subject, profile["readerId"], dimension)
|
|
248
392
|
if value is None:
|
|
@@ -263,6 +407,339 @@ def react(
|
|
|
263
407
|
return reply(200, {"subject": subject, "dimension": dimension, "value": value})
|
|
264
408
|
|
|
265
409
|
|
|
410
|
+
def subject_is_open(store: ReaderStore, subject: str) -> bool:
|
|
411
|
+
"""A published item, or a submission that is visible (pending or promoted); hidden submissions cannot be voted on."""
|
|
412
|
+
if SUBMISSION_ID.fullmatch(subject):
|
|
413
|
+
submission = store.get_submission(subject)
|
|
414
|
+
return submission is not None and submission.get("state") in VISIBLE_SUBMISSION_STATES
|
|
415
|
+
return store.is_published(subject)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def clean_text(value: Any, label: str, maximum: int, *, required: bool) -> tuple[str | None, dict[str, Any] | None]:
|
|
419
|
+
if value is None or value == "":
|
|
420
|
+
if required:
|
|
421
|
+
return None, refusal(422, f"invalid_{label}", f"Send a {label}.")
|
|
422
|
+
return None, None
|
|
423
|
+
if not isinstance(value, str) or CONTROL_CHARACTERS.search(value):
|
|
424
|
+
return None, refusal(422, f"invalid_{label}", f"The {label} must be plain text.")
|
|
425
|
+
if EMAIL_ADDRESS.search(value):
|
|
426
|
+
return None, refusal(422, f"invalid_{label}", f"Public text cannot contain an email address. Remove it from the {label}.")
|
|
427
|
+
text = " ".join(value.split()) if label == "title" else value.strip()
|
|
428
|
+
if not text and required:
|
|
429
|
+
return None, refusal(422, f"invalid_{label}", f"Send a {label}.")
|
|
430
|
+
if len(text) > maximum:
|
|
431
|
+
return None, refusal(422, f"invalid_{label}", f"The {label} is at most {maximum} characters.")
|
|
432
|
+
return text or None, None
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def public_submission(submission: dict[str, Any]) -> dict[str, Any]:
|
|
436
|
+
promoted = submission.get("state") == SUBMISSION_PROMOTED and submission.get("itemId")
|
|
437
|
+
return {"subject": submission["itemId"] if promoted else submission["id"], "url": submission["canonicalUrl"], "title": submission["title"], "state": submission["state"]}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def duplicate(existing: dict[str, Any]) -> dict[str, Any]:
|
|
441
|
+
if existing.get("state") not in VISIBLE_SUBMISSION_STATES:
|
|
442
|
+
return refusal(409, "duplicate_submission", "That link was already submitted.")
|
|
443
|
+
return refusal(409, "duplicate_submission", "That link was already submitted.", existing=public_submission(existing))
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def new_content_fields(settings: ReaderSettings, visible: str) -> dict[str, Any]:
|
|
447
|
+
"""``state`` and ``moderation`` for a new submission or comment.
|
|
448
|
+
|
|
449
|
+
With moderation off nothing waits in a queue. With pre-moderation the item starts hidden ("held") until a moderator approves it.
|
|
450
|
+
"""
|
|
451
|
+
if settings.moderation_mode == "off":
|
|
452
|
+
return {"state": visible, "moderation": moderation_rules.MODERATION_APPROVED}
|
|
453
|
+
if settings.pre_moderation:
|
|
454
|
+
return {"state": "hidden", "moderation": moderation_rules.MODERATION_NEW}
|
|
455
|
+
return {"state": visible, "moderation": moderation_rules.MODERATION_NEW}
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def submit_action(
|
|
459
|
+
store: ReaderStore,
|
|
460
|
+
profile: dict[str, Any],
|
|
461
|
+
payload: dict[str, Any],
|
|
462
|
+
settings: ReaderSettings,
|
|
463
|
+
now_seconds: float,
|
|
464
|
+
schedule_snapshot: Callable[[], None],
|
|
465
|
+
) -> dict[str, Any]:
|
|
466
|
+
if profile.get("state") == STATE_LIMITED:
|
|
467
|
+
return refusal(403, "reader_limited", "This reader cannot submit right now.")
|
|
468
|
+
if not profile.get("handle"):
|
|
469
|
+
return refusal(422, "handle_required", "Choose a handle before submitting a link.")
|
|
470
|
+
if set(payload) - SUBMISSION_PAYLOAD_KEYS:
|
|
471
|
+
return refusal(422, "invalid_submission", "Send only url, title, kind and note.")
|
|
472
|
+
raw_url = payload.get("url")
|
|
473
|
+
if not isinstance(raw_url, str) or not raw_url.strip() or len(raw_url) > MAXIMUM_URL_LENGTH:
|
|
474
|
+
return refusal(422, "invalid_url", "Send the link as an http or https address.")
|
|
475
|
+
try:
|
|
476
|
+
canonical = canonical_source_url(raw_url.strip())
|
|
477
|
+
except InvalidUrl:
|
|
478
|
+
return refusal(422, "invalid_url", "Send the link as an http or https address.")
|
|
479
|
+
if len(canonical) > MAXIMUM_URL_LENGTH or EMAIL_ADDRESS.search(canonical):
|
|
480
|
+
return refusal(422, "invalid_url", "Send the link as an http or https address without an email address in it.")
|
|
481
|
+
title, problem = clean_text(payload.get("title"), "title", MAXIMUM_TITLE_LENGTH, required=True)
|
|
482
|
+
if problem:
|
|
483
|
+
return problem
|
|
484
|
+
note, problem = clean_text(payload.get("note"), "note", MAXIMUM_NOTE_LENGTH, required=False)
|
|
485
|
+
if problem:
|
|
486
|
+
return problem
|
|
487
|
+
kind = payload.get("kind", settings.submission_kinds[0])
|
|
488
|
+
if not isinstance(kind, str) or kind not in settings.submission_kinds:
|
|
489
|
+
return refusal(422, "invalid_kind", "That kind is not available.")
|
|
490
|
+
limit = new_account_limit(profile, settings, now_seconds, settings.submissions_per_day)
|
|
491
|
+
limited = within_rate_limit(store, profile, RATE_BUCKET_SUBMIT, limit, now_seconds, DAY_SECONDS)
|
|
492
|
+
if limited:
|
|
493
|
+
return limited
|
|
494
|
+
submission_id = submission_id_for(canonical)
|
|
495
|
+
existing = store.get_submission(submission_id)
|
|
496
|
+
if existing is not None:
|
|
497
|
+
return duplicate(existing)
|
|
498
|
+
domain = domain_of(canonical)
|
|
499
|
+
since = iso(now_seconds - settings.similar_hours * HOUR_SECONDS)
|
|
500
|
+
similar_rows = [
|
|
501
|
+
row
|
|
502
|
+
for row in store.recent_submissions_by_domain(domain, since)
|
|
503
|
+
if row.get("state") in VISIBLE_SUBMISSION_STATES and row["id"] != submission_id
|
|
504
|
+
]
|
|
505
|
+
similar_rows.sort(key=lambda row: row["createdAt"], reverse=True)
|
|
506
|
+
similar_rows = similar_rows[:MAXIMUM_SIMILAR]
|
|
507
|
+
submission = {
|
|
508
|
+
"id": submission_id,
|
|
509
|
+
"canonicalUrl": canonical,
|
|
510
|
+
"domain": domain,
|
|
511
|
+
"title": title,
|
|
512
|
+
"kind": kind,
|
|
513
|
+
"readerId": profile["readerId"],
|
|
514
|
+
**new_content_fields(settings, SUBMISSION_PENDING),
|
|
515
|
+
"similar": [row["id"] for row in similar_rows],
|
|
516
|
+
"extractionState": EXTRACTION_PENDING,
|
|
517
|
+
"createdAt": iso(now_seconds),
|
|
518
|
+
}
|
|
519
|
+
if note:
|
|
520
|
+
submission["note"] = note
|
|
521
|
+
if not store.create_submission(submission):
|
|
522
|
+
return duplicate(store.get_submission(submission_id) or submission)
|
|
523
|
+
schedule_snapshot()
|
|
524
|
+
return reply(
|
|
525
|
+
201,
|
|
526
|
+
{
|
|
527
|
+
"subject": submission_id,
|
|
528
|
+
"url": canonical,
|
|
529
|
+
"title": title,
|
|
530
|
+
"kind": kind,
|
|
531
|
+
"state": "held" if submission["state"] == SUBMISSION_HIDDEN else SUBMISSION_PENDING,
|
|
532
|
+
"extraction": EXTRACTION_PENDING,
|
|
533
|
+
"similar": [{"subject": row["id"], "url": row["canonicalUrl"], "title": row["title"]} for row in similar_rows],
|
|
534
|
+
},
|
|
535
|
+
)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def clean_comment_body(value: Any, settings: ReaderSettings) -> tuple[str | None, dict[str, Any] | None]:
|
|
539
|
+
"""Plain text, trimmed, at most ``comment_maximum_length`` characters, ``comment_maximum_links`` links, no email address."""
|
|
540
|
+
if not isinstance(value, str):
|
|
541
|
+
return None, refusal(422, "invalid_body", "Send the comment as text.")
|
|
542
|
+
text = value.replace("\r\n", "\n").replace("\r", "\n").strip()
|
|
543
|
+
if not text:
|
|
544
|
+
return None, refusal(422, "invalid_body", "Write something first.")
|
|
545
|
+
if TEXT_CONTROL_CHARACTERS.search(text):
|
|
546
|
+
return None, refusal(422, "invalid_body", "A comment is plain text.")
|
|
547
|
+
if len(text) > settings.comment_maximum_length:
|
|
548
|
+
return None, refusal(422, "body_too_long", f"A comment is at most {settings.comment_maximum_length} characters.")
|
|
549
|
+
if len(LINK.findall(text)) > settings.comment_maximum_links:
|
|
550
|
+
return None, refusal(422, "too_many_links", f"A comment has at most {settings.comment_maximum_links} links.")
|
|
551
|
+
if EMAIL_ADDRESS.search(text):
|
|
552
|
+
return None, refusal(422, "invalid_body", "Public text cannot contain an email address. Remove it from the comment.")
|
|
553
|
+
return text, None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def writes_comments(profile: dict[str, Any]) -> dict[str, Any] | None:
|
|
557
|
+
if profile.get("state") == STATE_LIMITED:
|
|
558
|
+
return refusal(403, "reader_limited", "This reader cannot comment right now.")
|
|
559
|
+
if not profile.get("handle"):
|
|
560
|
+
return refusal(422, "handle_required", "Choose a handle before commenting.")
|
|
561
|
+
return None
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def rating_context_for(store: ReaderStore, profile: dict[str, Any], subject: str, raw: Any, settings: ReaderSettings) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
565
|
+
"""The rating a comment explains must be the reader's own current vote, so a comment cannot claim a rating nobody gave."""
|
|
566
|
+
if raw is None:
|
|
567
|
+
return None, None
|
|
568
|
+
if not isinstance(raw, dict) or set(raw) != {"dimension", "value"}:
|
|
569
|
+
return None, refusal(422, "invalid_rating_context", "Send ratingContext as {dimension, value}.")
|
|
570
|
+
dimension, value = raw["dimension"], raw["value"]
|
|
571
|
+
if not isinstance(dimension, str) or dimension not in (*TOGGLE_DIMENSIONS, *settings.dimensions) or isinstance(value, bool) or not isinstance(value, int):
|
|
572
|
+
return None, refusal(422, "invalid_rating_context", "That rating context is not available.")
|
|
573
|
+
stored = store.get_vote_values([vote_id(subject, profile["readerId"], dimension)])
|
|
574
|
+
if stored.get(vote_id(subject, profile["readerId"], dimension)) != value:
|
|
575
|
+
return None, refusal(422, "rating_context_mismatch", "Rate or vote first; a comment can only explain your own current rating.")
|
|
576
|
+
return {"dimension": dimension, "value": value}, None
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def comment_action(
|
|
580
|
+
store: ReaderStore,
|
|
581
|
+
profile: dict[str, Any],
|
|
582
|
+
payload: dict[str, Any],
|
|
583
|
+
settings: ReaderSettings,
|
|
584
|
+
now_seconds: float,
|
|
585
|
+
schedule_snapshot: Callable[[], None],
|
|
586
|
+
) -> dict[str, Any]:
|
|
587
|
+
problem = writes_comments(profile)
|
|
588
|
+
if problem:
|
|
589
|
+
return problem
|
|
590
|
+
if set(payload) - COMMENT_PAYLOAD_KEYS:
|
|
591
|
+
return refusal(422, "invalid_comment", "Send only subject, body, parent and ratingContext.")
|
|
592
|
+
subject = payload.get("subject")
|
|
593
|
+
if not isinstance(subject, str) or not SUBJECT_KEY.fullmatch(subject):
|
|
594
|
+
return refusal(422, "invalid_subject", "Send the key of a published item.")
|
|
595
|
+
body, problem = clean_comment_body(payload.get("body"), settings)
|
|
596
|
+
if problem:
|
|
597
|
+
return problem
|
|
598
|
+
parent_id = payload.get("parent")
|
|
599
|
+
if parent_id is not None and (not isinstance(parent_id, str) or not COMMENT_ID.fullmatch(parent_id)):
|
|
600
|
+
return refusal(422, "invalid_parent", "That comment cannot be replied to.")
|
|
601
|
+
context, problem = rating_context_for(store, profile, subject, payload.get("ratingContext"), settings)
|
|
602
|
+
if problem:
|
|
603
|
+
return problem
|
|
604
|
+
if context is not None and parent_id is not None:
|
|
605
|
+
return refusal(422, "invalid_rating_context", "A reply does not explain a rating.")
|
|
606
|
+
limited = within_rate_limit(store, profile, RATE_BUCKET_COMMENT, new_account_limit(profile, settings, now_seconds, settings.comments_per_hour), now_seconds)
|
|
607
|
+
if limited:
|
|
608
|
+
return limited
|
|
609
|
+
if not subject_is_open(store, subject):
|
|
610
|
+
return refusal(404, "unknown_subject", "That item is not published.")
|
|
611
|
+
if parent_id is not None:
|
|
612
|
+
parent = store.get_comment(parent_id)
|
|
613
|
+
if parent is None or parent.get("subjectKey") != subject or parent.get("parentId") or parent.get("state") != COMMENT_VISIBLE:
|
|
614
|
+
return refusal(422, "invalid_parent", "That comment cannot be replied to.")
|
|
615
|
+
comment_id = COMMENT_ID_PREFIX + uuid.uuid4().hex
|
|
616
|
+
created = iso(now_seconds)
|
|
617
|
+
comment = {
|
|
618
|
+
"id": comment_id,
|
|
619
|
+
"subjectKey": subject,
|
|
620
|
+
"readerId": profile["readerId"],
|
|
621
|
+
"body": body,
|
|
622
|
+
**new_content_fields(settings, COMMENT_VISIBLE),
|
|
623
|
+
"createdAt": created,
|
|
624
|
+
}
|
|
625
|
+
if parent_id is not None:
|
|
626
|
+
comment["parentId"] = parent_id
|
|
627
|
+
if context is not None:
|
|
628
|
+
comment["ratingContext"] = json.dumps(context, separators=(",", ":"))
|
|
629
|
+
store.create_comment(comment)
|
|
630
|
+
schedule_snapshot()
|
|
631
|
+
return reply(
|
|
632
|
+
201,
|
|
633
|
+
{
|
|
634
|
+
"id": comment_id, "subject": subject, "parent": parent_id, "body": body, "createdAt": created, "ratingContext": context,
|
|
635
|
+
"state": "held" if comment["state"] == COMMENT_HIDDEN else COMMENT_VISIBLE,
|
|
636
|
+
},
|
|
637
|
+
)
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def report_action(
|
|
641
|
+
store: ReaderStore,
|
|
642
|
+
profile: dict[str, Any],
|
|
643
|
+
payload: dict[str, Any],
|
|
644
|
+
settings: ReaderSettings,
|
|
645
|
+
now_seconds: float,
|
|
646
|
+
schedule_snapshot: Callable[[], None],
|
|
647
|
+
) -> dict[str, Any]:
|
|
648
|
+
"""A reader reports a visible submission or comment; it goes back to the moderation queue. One report per reader and target."""
|
|
649
|
+
if set(payload) - {"target", "reason"}:
|
|
650
|
+
return refusal(422, "invalid_report", "Send only target and reason.")
|
|
651
|
+
target = payload.get("target")
|
|
652
|
+
target_type = moderation_rules.target_type_of(target) if isinstance(target, str) else None
|
|
653
|
+
if target_type not in (moderation_rules.TARGET_SUBMISSION, moderation_rules.TARGET_COMMENT):
|
|
654
|
+
return refusal(422, "invalid_target", "Report a submission or a comment.")
|
|
655
|
+
reason = payload.get("reason")
|
|
656
|
+
if not isinstance(reason, str) or not reason.strip() or len(reason.strip()) > MAXIMUM_REPORT_REASON or CONTROL_CHARACTERS.search(reason):
|
|
657
|
+
return refusal(422, "invalid_reason", f"Give a reason in plain text of at most {MAXIMUM_REPORT_REASON} characters.")
|
|
658
|
+
limited = within_rate_limit(
|
|
659
|
+
store, profile, RATE_BUCKET_REPORT, new_account_limit(profile, settings, now_seconds, settings.reports_per_day), now_seconds, DAY_SECONDS
|
|
660
|
+
)
|
|
661
|
+
if limited:
|
|
662
|
+
return limited
|
|
663
|
+
row = store.get_submission(target) if target_type == moderation_rules.TARGET_SUBMISSION else store.get_comment(target)
|
|
664
|
+
visible = VISIBLE_SUBMISSION_STATES if target_type == moderation_rules.TARGET_SUBMISSION else (COMMENT_VISIBLE,)
|
|
665
|
+
if row is None or row.get("state") not in visible:
|
|
666
|
+
return refusal(404, "unknown_target", "That cannot be reported.")
|
|
667
|
+
report = {
|
|
668
|
+
"id": REPORT_ID_PREFIX + hashlib.sha256(f"{profile['readerId']}#{target}".encode("utf-8")).hexdigest()[:24],
|
|
669
|
+
"targetType": target_type,
|
|
670
|
+
"target": target,
|
|
671
|
+
"reporterReaderId": profile["readerId"],
|
|
672
|
+
"reason": reason.strip(),
|
|
673
|
+
"state": "open",
|
|
674
|
+
"createdAt": iso(now_seconds),
|
|
675
|
+
}
|
|
676
|
+
store.put_report(report, target, "submission" if target_type == moderation_rules.TARGET_SUBMISSION else "comment", iso(now_seconds))
|
|
677
|
+
return reply(200, {"reported": True})
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def own_comment(store: ReaderStore, profile: dict[str, Any], payload: dict[str, Any], allowed: frozenset[str]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
681
|
+
if set(payload) - allowed:
|
|
682
|
+
return None, refusal(422, "invalid_comment", "Send only " + ", ".join(sorted(allowed)) + ".")
|
|
683
|
+
comment_id = payload.get("id")
|
|
684
|
+
if not isinstance(comment_id, str) or not COMMENT_ID.fullmatch(comment_id):
|
|
685
|
+
return None, refusal(404, "unknown_comment", "That comment does not exist.")
|
|
686
|
+
comment = store.get_comment(comment_id)
|
|
687
|
+
if comment is None or comment.get("readerId") != profile["readerId"] or comment.get("state") == COMMENT_HIDDEN:
|
|
688
|
+
return None, refusal(404, "unknown_comment", "That comment does not exist.")
|
|
689
|
+
return comment, None
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def edit_comment_action(
|
|
693
|
+
store: ReaderStore,
|
|
694
|
+
profile: dict[str, Any],
|
|
695
|
+
payload: dict[str, Any],
|
|
696
|
+
settings: ReaderSettings,
|
|
697
|
+
now_seconds: float,
|
|
698
|
+
schedule_snapshot: Callable[[], None],
|
|
699
|
+
) -> dict[str, Any]:
|
|
700
|
+
problem = writes_comments(profile)
|
|
701
|
+
if problem:
|
|
702
|
+
return problem
|
|
703
|
+
comment, problem = own_comment(store, profile, payload, COMMENT_EDIT_KEYS)
|
|
704
|
+
if problem:
|
|
705
|
+
return problem
|
|
706
|
+
if comment.get("state") != COMMENT_VISIBLE:
|
|
707
|
+
return refusal(409, "comment_deleted", "A deleted comment cannot be edited.")
|
|
708
|
+
if now_seconds - seconds_of(comment["createdAt"]) > settings.comment_edit_minutes * 60:
|
|
709
|
+
return refusal(403, "edit_window_closed", f"A comment can be edited for {settings.comment_edit_minutes} minutes after it is posted.")
|
|
710
|
+
body, problem = clean_comment_body(payload.get("body"), settings)
|
|
711
|
+
if problem:
|
|
712
|
+
return problem
|
|
713
|
+
limited = within_rate_limit(store, profile, RATE_BUCKET_COMMENT, new_account_limit(profile, settings, now_seconds, settings.comments_per_hour), now_seconds)
|
|
714
|
+
if limited:
|
|
715
|
+
return limited
|
|
716
|
+
edited = iso(now_seconds)
|
|
717
|
+
if not store.update_comment(comment["id"], {"body": body, "editedAt": edited, "reviewedAt": None, "signals": None}, edited, expected_state=COMMENT_VISIBLE):
|
|
718
|
+
return refusal(409, "comment_changed", "That comment changed. Reload and try again.")
|
|
719
|
+
schedule_snapshot()
|
|
720
|
+
return reply(200, {"id": comment["id"], "body": body, "editedAt": edited})
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def delete_comment_action(
|
|
724
|
+
store: ReaderStore,
|
|
725
|
+
profile: dict[str, Any],
|
|
726
|
+
payload: dict[str, Any],
|
|
727
|
+
settings: ReaderSettings,
|
|
728
|
+
now_seconds: float,
|
|
729
|
+
schedule_snapshot: Callable[[], None],
|
|
730
|
+
) -> dict[str, Any]:
|
|
731
|
+
comment, problem = own_comment(store, profile, payload, COMMENT_DELETE_KEYS)
|
|
732
|
+
if problem:
|
|
733
|
+
return problem
|
|
734
|
+
if comment.get("state") == COMMENT_DELETED:
|
|
735
|
+
return reply(200, {"id": comment["id"], "state": COMMENT_DELETED})
|
|
736
|
+
deleted = iso(now_seconds)
|
|
737
|
+
if not store.update_comment(comment["id"], {"body": "", "deletedAt": deleted, "state": COMMENT_DELETED, "ratingContext": None}, deleted, expected_state=COMMENT_VISIBLE):
|
|
738
|
+
return refusal(409, "comment_changed", "That comment changed. Reload and try again.")
|
|
739
|
+
schedule_snapshot()
|
|
740
|
+
return reply(200, {"id": comment["id"], "state": COMMENT_DELETED})
|
|
741
|
+
|
|
742
|
+
|
|
266
743
|
def vote_action(store, profile, payload, settings, is_staff, now_seconds, schedule_snapshot):
|
|
267
744
|
on = payload.get("on")
|
|
268
745
|
if not isinstance(on, bool):
|
|
@@ -307,6 +784,16 @@ def handle_event(
|
|
|
307
784
|
return vote_action(store, profile, payload, settings, event["isStaff"], now_seconds, schedule_snapshot)
|
|
308
785
|
if action == ACTION_RATING:
|
|
309
786
|
return rating_action(store, profile, payload, settings, event["isStaff"], now_seconds, schedule_snapshot)
|
|
787
|
+
if action == ACTION_SUBMIT:
|
|
788
|
+
return submit_action(store, profile, payload, settings, now_seconds, schedule_snapshot)
|
|
789
|
+
if action == ACTION_COMMENT:
|
|
790
|
+
return comment_action(store, profile, payload, settings, now_seconds, schedule_snapshot)
|
|
791
|
+
if action == ACTION_COMMENT_EDIT:
|
|
792
|
+
return edit_comment_action(store, profile, payload, settings, now_seconds, schedule_snapshot)
|
|
793
|
+
if action == ACTION_COMMENT_DELETE:
|
|
794
|
+
return delete_comment_action(store, profile, payload, settings, now_seconds, schedule_snapshot)
|
|
795
|
+
if action == ACTION_REPORT:
|
|
796
|
+
return report_action(store, profile, payload, settings, now_seconds, schedule_snapshot)
|
|
310
797
|
return refusal(400, "unknown_action", "Unknown action.")
|
|
311
798
|
|
|
312
799
|
|
|
@@ -320,6 +807,10 @@ def plain(value: Any) -> Any:
|
|
|
320
807
|
return value
|
|
321
808
|
|
|
322
809
|
|
|
810
|
+
def _without_none(item: dict[str, Any]) -> dict[str, Any]:
|
|
811
|
+
return {key: value for key, value in item.items() if value is not None}
|
|
812
|
+
|
|
813
|
+
|
|
323
814
|
def is_conditional_failure(error: Exception) -> bool:
|
|
324
815
|
return str(getattr(error, "response", {}).get("Error", {}).get("Code", "")) == "ConditionalCheckFailedException"
|
|
325
816
|
|
|
@@ -327,9 +818,34 @@ def is_conditional_failure(error: Exception) -> bool:
|
|
|
327
818
|
class DynamoReaderStore:
|
|
328
819
|
"""The Amplify model tables written directly: items carry ``__typename``, ``createdAt`` and ``updatedAt`` so staff can list them."""
|
|
329
820
|
|
|
330
|
-
def __init__(
|
|
821
|
+
def __init__(
|
|
822
|
+
self,
|
|
823
|
+
*,
|
|
824
|
+
resource: Any,
|
|
825
|
+
profile_table: str,
|
|
826
|
+
handle_table: str,
|
|
827
|
+
vote_table: str,
|
|
828
|
+
published_item_table: str,
|
|
829
|
+
rate_table: str,
|
|
830
|
+
submission_table: str = "",
|
|
831
|
+
comment_table: str = "",
|
|
832
|
+
report_table: str = "",
|
|
833
|
+
action_table: str = "",
|
|
834
|
+
item_table: str = "",
|
|
835
|
+
) -> None:
|
|
331
836
|
self._resource = resource
|
|
332
|
-
self._names = {
|
|
837
|
+
self._names = {
|
|
838
|
+
"profile": profile_table,
|
|
839
|
+
"handle": handle_table,
|
|
840
|
+
"vote": vote_table,
|
|
841
|
+
"published": published_item_table,
|
|
842
|
+
"rate": rate_table,
|
|
843
|
+
"submission": submission_table,
|
|
844
|
+
"comment": comment_table,
|
|
845
|
+
"report": report_table,
|
|
846
|
+
"action": action_table,
|
|
847
|
+
"item": item_table,
|
|
848
|
+
}
|
|
333
849
|
|
|
334
850
|
def _table(self, key: str) -> Any:
|
|
335
851
|
return self._resource.Table(self._names[key])
|
|
@@ -420,6 +936,208 @@ class DynamoReaderStore:
|
|
|
420
936
|
item = self._table("published").get_item(Key={"id": PUBLISHED_ID_PREFIX + subject_key}, ProjectionExpression="#s", ExpressionAttributeNames={"#s": "status"}).get("Item")
|
|
421
937
|
return bool(item) and item.get("status") == "published"
|
|
422
938
|
|
|
939
|
+
def get_submission(self, submission_id: str) -> dict[str, Any] | None:
|
|
940
|
+
item = self._table("submission").get_item(Key={"id": submission_id}, ConsistentRead=True).get("Item")
|
|
941
|
+
return plain(item) if item else None
|
|
942
|
+
|
|
943
|
+
def create_submission(self, submission: dict[str, Any]) -> bool:
|
|
944
|
+
item = {**submission, "__typename": "ReaderSubmission", "updatedAt": submission["createdAt"]}
|
|
945
|
+
try:
|
|
946
|
+
self._table("submission").put_item(Item=item, ConditionExpression="attribute_not_exists(id)")
|
|
947
|
+
except Exception as error:
|
|
948
|
+
if is_conditional_failure(error):
|
|
949
|
+
return False
|
|
950
|
+
raise
|
|
951
|
+
return True
|
|
952
|
+
|
|
953
|
+
def recent_submissions_by_domain(self, domain: str, since_iso: str) -> list[dict[str, Any]]:
|
|
954
|
+
from boto3.dynamodb.conditions import Key
|
|
955
|
+
|
|
956
|
+
rows: list[dict[str, Any]] = []
|
|
957
|
+
kwargs: dict[str, Any] = {
|
|
958
|
+
"IndexName": SUBMISSION_DOMAIN_INDEX,
|
|
959
|
+
"KeyConditionExpression": Key("domain").eq(domain) & Key("createdAt").gte(since_iso),
|
|
960
|
+
}
|
|
961
|
+
while True:
|
|
962
|
+
page = self._table("submission").query(**kwargs)
|
|
963
|
+
rows.extend(plain(item) for item in page.get("Items", []))
|
|
964
|
+
if "LastEvaluatedKey" not in page:
|
|
965
|
+
return rows
|
|
966
|
+
kwargs["ExclusiveStartKey"] = page["LastEvaluatedKey"]
|
|
967
|
+
|
|
968
|
+
def get_comment(self, comment_id: str) -> dict[str, Any] | None:
|
|
969
|
+
item = self._table("comment").get_item(Key={"id": comment_id}, ConsistentRead=True).get("Item")
|
|
970
|
+
return plain(item) if item else None
|
|
971
|
+
|
|
972
|
+
def create_comment(self, comment: dict[str, Any]) -> None:
|
|
973
|
+
self._table("comment").put_item(
|
|
974
|
+
Item={**comment, "__typename": "ReaderComment", "updatedAt": comment["createdAt"]}, ConditionExpression="attribute_not_exists(id)"
|
|
975
|
+
)
|
|
976
|
+
|
|
977
|
+
def update_comment(self, comment_id: str, changes: dict[str, Any], now_iso: str, *, expected_state: str) -> bool:
|
|
978
|
+
"""Sets the changed fields (``None`` removes one) while the comment is still in ``expected_state``."""
|
|
979
|
+
names = {"#state": "state"}
|
|
980
|
+
values: dict[str, Any] = {":expected": expected_state, ":now": now_iso}
|
|
981
|
+
sets, removes = ["updatedAt = :now"], []
|
|
982
|
+
for number, (field, value) in enumerate(changes.items()):
|
|
983
|
+
names[f"#f{number}"] = field
|
|
984
|
+
if value is None:
|
|
985
|
+
removes.append(f"#f{number}")
|
|
986
|
+
else:
|
|
987
|
+
values[f":v{number}"] = value
|
|
988
|
+
sets.append(f"#f{number} = :v{number}")
|
|
989
|
+
expression = "SET " + ", ".join(sets) + (" REMOVE " + ", ".join(removes) if removes else "")
|
|
990
|
+
try:
|
|
991
|
+
self._table("comment").update_item(
|
|
992
|
+
Key={"id": comment_id},
|
|
993
|
+
UpdateExpression=expression,
|
|
994
|
+
ConditionExpression="attribute_exists(id) AND #state = :expected",
|
|
995
|
+
ExpressionAttributeNames=names,
|
|
996
|
+
ExpressionAttributeValues=values,
|
|
997
|
+
)
|
|
998
|
+
except Exception as error:
|
|
999
|
+
if is_conditional_failure(error):
|
|
1000
|
+
return False
|
|
1001
|
+
raise
|
|
1002
|
+
return True
|
|
1003
|
+
|
|
1004
|
+
def put_report(self, report: dict[str, Any], target_key: str, target_table: str, now_iso: str) -> bool:
|
|
1005
|
+
"""Stores the report and puts the target back in the queue in one transaction. A second report by the same reader changes nothing."""
|
|
1006
|
+
item = {**report, "__typename": "ReaderReport", "updatedAt": report["createdAt"]}
|
|
1007
|
+
try:
|
|
1008
|
+
self._resource.meta.client.transact_write_items(
|
|
1009
|
+
TransactItems=[
|
|
1010
|
+
{"Put": {"TableName": self._names["report"], "Item": _without_none(item), "ConditionExpression": "attribute_not_exists(id)"}},
|
|
1011
|
+
{
|
|
1012
|
+
"Update": {
|
|
1013
|
+
"TableName": self._names[target_table],
|
|
1014
|
+
"Key": {"id": target_key},
|
|
1015
|
+
"UpdateExpression": "SET moderation = :new, updatedAt = :now ADD reportCount :one",
|
|
1016
|
+
"ConditionExpression": "attribute_exists(id)",
|
|
1017
|
+
"ExpressionAttributeValues": {":new": moderation_rules.MODERATION_NEW, ":now": now_iso, ":one": 1},
|
|
1018
|
+
}
|
|
1019
|
+
},
|
|
1020
|
+
]
|
|
1021
|
+
)
|
|
1022
|
+
except Exception as error:
|
|
1023
|
+
reasons = getattr(error, "response", {}).get("CancellationReasons") or []
|
|
1024
|
+
if [str(reason.get("Code", "")) for reason in reasons][:1] == ["ConditionalCheckFailed"]:
|
|
1025
|
+
return False
|
|
1026
|
+
raise
|
|
1027
|
+
return True
|
|
1028
|
+
|
|
1029
|
+
def get_profile_by_reader(self, reader_id: str) -> dict[str, Any] | None:
|
|
1030
|
+
from boto3.dynamodb.conditions import Key
|
|
1031
|
+
|
|
1032
|
+
page = self._table("profile").query(IndexName=PROFILE_READER_INDEX, KeyConditionExpression=Key("readerId").eq(reader_id))
|
|
1033
|
+
rows = page.get("Items", [])
|
|
1034
|
+
return plain(rows[0]) if rows else None
|
|
1035
|
+
|
|
1036
|
+
def get_action(self, action_id: str) -> dict[str, Any] | None:
|
|
1037
|
+
item = self._table("action").get_item(Key={"id": action_id}, ConsistentRead=True).get("Item")
|
|
1038
|
+
return plain(item) if item else None
|
|
1039
|
+
|
|
1040
|
+
def transact_moderation(self, action: dict[str, Any], target_type: str, target_key: str, plan: Any, *, undoes: str | None = None) -> str:
|
|
1041
|
+
"""Writes the log row, the target change (only if the target still has the expected state) and, for an undo, the pointer on the original."""
|
|
1042
|
+
table_for = {moderation_rules.TARGET_SUBMISSION: "submission", moderation_rules.TARGET_COMMENT: "comment", moderation_rules.TARGET_READER: "profile"}
|
|
1043
|
+
names: dict[str, str] = {}
|
|
1044
|
+
values: dict[str, Any] = {":now": action["at"]}
|
|
1045
|
+
sets = ["updatedAt = :now"]
|
|
1046
|
+
conditions = ["attribute_exists(id)"]
|
|
1047
|
+
for number, (field, value) in enumerate(plan.changes.items()):
|
|
1048
|
+
names[f"#c{number}"] = field
|
|
1049
|
+
values[f":c{number}"] = value
|
|
1050
|
+
sets.append(f"#c{number} = :c{number}")
|
|
1051
|
+
for number, (field, value) in enumerate(plan.expected.items()):
|
|
1052
|
+
names[f"#e{number}"] = field
|
|
1053
|
+
values[f":e{number}"] = value
|
|
1054
|
+
conditions.append(f"#e{number} = :e{number}")
|
|
1055
|
+
items: list[dict[str, Any]] = [
|
|
1056
|
+
{"Put": {"TableName": self._names["action"], "Item": _without_none({**action, "__typename": "ModerationAction", "updatedAt": action["at"]}), "ConditionExpression": "attribute_not_exists(id)"}},
|
|
1057
|
+
{
|
|
1058
|
+
"Update": {
|
|
1059
|
+
"TableName": self._names[table_for[target_type]],
|
|
1060
|
+
"Key": {"id": target_key},
|
|
1061
|
+
"UpdateExpression": "SET " + ", ".join(sets),
|
|
1062
|
+
"ConditionExpression": " AND ".join(conditions),
|
|
1063
|
+
"ExpressionAttributeNames": names,
|
|
1064
|
+
"ExpressionAttributeValues": values,
|
|
1065
|
+
}
|
|
1066
|
+
},
|
|
1067
|
+
]
|
|
1068
|
+
if undoes:
|
|
1069
|
+
items.append(
|
|
1070
|
+
{
|
|
1071
|
+
"Update": {
|
|
1072
|
+
"TableName": self._names["action"],
|
|
1073
|
+
"Key": {"id": undoes},
|
|
1074
|
+
"UpdateExpression": "SET undoneBy = :by, updatedAt = :now",
|
|
1075
|
+
"ConditionExpression": "attribute_exists(id) AND attribute_not_exists(undoneBy)",
|
|
1076
|
+
"ExpressionAttributeValues": {":by": action["id"], ":now": action["at"]},
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
)
|
|
1080
|
+
if plan.item is not None:
|
|
1081
|
+
items.append({"Put": {"TableName": self._names["item"], "Item": _without_none(plan.item), "ConditionExpression": "attribute_not_exists(id)"}})
|
|
1082
|
+
try:
|
|
1083
|
+
self._resource.meta.client.transact_write_items(TransactItems=items)
|
|
1084
|
+
except Exception as error:
|
|
1085
|
+
reasons = getattr(error, "response", {}).get("CancellationReasons") or []
|
|
1086
|
+
codes = [str(reason.get("Code", "")) for reason in reasons]
|
|
1087
|
+
if codes[1:2] == ["ConditionalCheckFailed"]:
|
|
1088
|
+
return "target_changed"
|
|
1089
|
+
if codes[2:3] == ["ConditionalCheckFailed"] and undoes:
|
|
1090
|
+
return "already_undone"
|
|
1091
|
+
raise
|
|
1092
|
+
return "ok"
|
|
1093
|
+
|
|
1094
|
+
def close_reports(self, target: str, now_iso: str) -> int:
|
|
1095
|
+
from boto3.dynamodb.conditions import Key
|
|
1096
|
+
|
|
1097
|
+
table = self._table("report")
|
|
1098
|
+
closed = 0
|
|
1099
|
+
kwargs: dict[str, Any] = {"IndexName": REPORT_TARGET_INDEX, "KeyConditionExpression": Key("target").eq(target)}
|
|
1100
|
+
while True:
|
|
1101
|
+
page = table.query(**kwargs)
|
|
1102
|
+
for row in page.get("Items", []):
|
|
1103
|
+
if row.get("state") == "open":
|
|
1104
|
+
table.update_item(
|
|
1105
|
+
Key={"id": row["id"]},
|
|
1106
|
+
UpdateExpression="SET #s = :closed, closedAt = :now, updatedAt = :now",
|
|
1107
|
+
ExpressionAttributeNames={"#s": "state"},
|
|
1108
|
+
ExpressionAttributeValues={":closed": "closed", ":now": now_iso},
|
|
1109
|
+
)
|
|
1110
|
+
closed += 1
|
|
1111
|
+
if "LastEvaluatedKey" not in page:
|
|
1112
|
+
return closed
|
|
1113
|
+
kwargs["ExclusiveStartKey"] = page["LastEvaluatedKey"]
|
|
1114
|
+
|
|
1115
|
+
def _scan(self, table_key: str, **arguments: Any) -> list[dict[str, Any]]:
|
|
1116
|
+
rows: list[dict[str, Any]] = []
|
|
1117
|
+
while True:
|
|
1118
|
+
page = self._table(table_key).scan(**arguments)
|
|
1119
|
+
rows.extend(plain(item) for item in page.get("Items", []))
|
|
1120
|
+
if "LastEvaluatedKey" not in page:
|
|
1121
|
+
return rows
|
|
1122
|
+
arguments["ExclusiveStartKey"] = page["LastEvaluatedKey"]
|
|
1123
|
+
|
|
1124
|
+
def new_content(self) -> list[tuple[str, dict[str, Any]]]:
|
|
1125
|
+
"""Submissions and comments waiting for a decision (``moderation: new``). A scan: the queue of a pilot is small."""
|
|
1126
|
+
from boto3.dynamodb.conditions import Attr
|
|
1127
|
+
|
|
1128
|
+
waiting = Attr("moderation").eq(moderation_rules.MODERATION_NEW)
|
|
1129
|
+
return [
|
|
1130
|
+
*((moderation_rules.TARGET_SUBMISSION, row) for row in self._scan("submission", FilterExpression=waiting)),
|
|
1131
|
+
*((moderation_rules.TARGET_COMMENT, row) for row in self._scan("comment", FilterExpression=waiting)),
|
|
1132
|
+
]
|
|
1133
|
+
|
|
1134
|
+
def human_acted_on(self) -> set[str]:
|
|
1135
|
+
"""Targets a person has decided on; the model never judges those again."""
|
|
1136
|
+
from boto3.dynamodb.conditions import Attr
|
|
1137
|
+
|
|
1138
|
+
rows = self._scan("action", FilterExpression=Attr("by").eq(moderation_rules.BY_HUMAN))
|
|
1139
|
+
return {row["target"] for row in rows}
|
|
1140
|
+
|
|
423
1141
|
def count_action(self, reader_id: str, bucket: str, window_start: int, limit: int, expires_at: int) -> bool:
|
|
424
1142
|
try:
|
|
425
1143
|
self._table("rate").update_item(
|
|
@@ -492,7 +1210,7 @@ def snapshot_scheduler(
|
|
|
492
1210
|
Target={
|
|
493
1211
|
"Arn": target.function_arn,
|
|
494
1212
|
"RoleArn": target.role_arn,
|
|
495
|
-
"Input":
|
|
1213
|
+
"Input": target.task_json,
|
|
496
1214
|
"RetryPolicy": {"MaximumRetryAttempts": 0},
|
|
497
1215
|
},
|
|
498
1216
|
)
|
|
@@ -511,6 +1229,11 @@ def snapshot_scheduler(
|
|
|
511
1229
|
class SnapshotTarget:
|
|
512
1230
|
function_arn: str
|
|
513
1231
|
role_arn: str
|
|
1232
|
+
task_json: str = json.dumps(SNAPSHOT_TASK)
|
|
1233
|
+
|
|
1234
|
+
def for_ai_review(self, own_function_arn: str) -> "SnapshotTarget":
|
|
1235
|
+
"""In AI mode the debounced run is the AI worker (this function), which asks for the snapshot when it has decided."""
|
|
1236
|
+
return SnapshotTarget(own_function_arn, self.role_arn, json.dumps(AI_TASK))
|
|
514
1237
|
|
|
515
1238
|
|
|
516
1239
|
def snapshot_target_from_environment(environment: dict[str, str] | None = None) -> SnapshotTarget | None:
|
|
@@ -531,14 +1254,110 @@ def store_from_environment(environment: dict[str, str] | None = None) -> DynamoR
|
|
|
531
1254
|
vote_table=environment["PAPYRUS_READER_VOTE_TABLE"],
|
|
532
1255
|
published_item_table=environment["PAPYRUS_READER_PUBLISHED_ITEM_TABLE"],
|
|
533
1256
|
rate_table=environment["PAPYRUS_READER_RATE_TABLE"],
|
|
1257
|
+
submission_table=environment.get("PAPYRUS_READER_SUBMISSION_TABLE", ""),
|
|
1258
|
+
comment_table=environment.get("PAPYRUS_READER_COMMENT_TABLE", ""),
|
|
1259
|
+
report_table=environment.get("PAPYRUS_READER_REPORT_TABLE", ""),
|
|
1260
|
+
action_table=environment.get("PAPYRUS_READER_ACTION_TABLE", ""),
|
|
1261
|
+
item_table=environment.get("PAPYRUS_READER_ITEM_TABLE", ""),
|
|
534
1262
|
)
|
|
535
1263
|
|
|
536
1264
|
|
|
1265
|
+
def read_api_key(settings: ReaderSettings, ssm_client: Any) -> str:
|
|
1266
|
+
"""The model's API key from its SSM SecureString (never Secrets Manager). The caller keeps it out of logs and results."""
|
|
1267
|
+
name = settings.ai_key_parameter
|
|
1268
|
+
if not ai_moderation.PARAMETER_NAME.fullmatch(name):
|
|
1269
|
+
raise ValueError("moderation.ai.apiKeyParameter is not a /papyrus/<site>/openai-api-key name")
|
|
1270
|
+
value = ssm_client.get_parameter(Name=name, WithDecryption=True)["Parameter"]["Value"].strip()
|
|
1271
|
+
if not value:
|
|
1272
|
+
raise ValueError("the API key parameter is empty")
|
|
1273
|
+
return value
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
def run_ai_worker(
|
|
1277
|
+
*,
|
|
1278
|
+
store: Any,
|
|
1279
|
+
settings: ReaderSettings,
|
|
1280
|
+
now: Callable[[], float],
|
|
1281
|
+
deadline: float,
|
|
1282
|
+
read_key: Callable[[], str],
|
|
1283
|
+
model_factory: Callable[[str, str], Any],
|
|
1284
|
+
invoke_snapshot: Callable[[], None],
|
|
1285
|
+
) -> dict[str, Any]:
|
|
1286
|
+
"""The scheduled AI review: judge new items, then ask for a snapshot so decisions reach the public files.
|
|
1287
|
+
|
|
1288
|
+
Anything that goes wrong is reported by error class name. The snapshot is requested whatever happens, so votes still show.
|
|
1289
|
+
"""
|
|
1290
|
+
if settings.moderation_mode != "ai":
|
|
1291
|
+
return {"ok": False, "error": "ai_moderation_off"}
|
|
1292
|
+
result: dict[str, Any]
|
|
1293
|
+
try:
|
|
1294
|
+
model = model_factory(read_key(), settings.ai_model)
|
|
1295
|
+
report = ai_moderation.run_ai_moderation(
|
|
1296
|
+
store,
|
|
1297
|
+
model,
|
|
1298
|
+
now=now,
|
|
1299
|
+
settings=moderation_rules.ModerationSettings(item_type=settings.promote_item_type, section=settings.promote_section),
|
|
1300
|
+
deadline=deadline,
|
|
1301
|
+
iso=iso,
|
|
1302
|
+
)
|
|
1303
|
+
result = {"ok": True, **report.to_dict()}
|
|
1304
|
+
except Exception as error: # noqa: BLE001 - never put the message in the result: it could echo a key or reader text
|
|
1305
|
+
result = {"ok": False, "error": type(error).__name__}
|
|
1306
|
+
try:
|
|
1307
|
+
invoke_snapshot()
|
|
1308
|
+
except Exception as error: # noqa: BLE001
|
|
1309
|
+
result["snapshotError"] = type(error).__name__
|
|
1310
|
+
return result
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def handle_staff_event(event: dict[str, Any], *, store: Any, settings: ReaderSettings, now_seconds: float) -> dict[str, Any]:
|
|
1314
|
+
"""``moderateReader`` from AppSync: an editor or admin hides, approves, promotes, bans or undoes. Answers a receipt, never an exception."""
|
|
1315
|
+
identity = event.get("identity") or {}
|
|
1316
|
+
claims = identity.get("claims") or {}
|
|
1317
|
+
groups = claims.get("cognito:groups") or identity.get("groups") or []
|
|
1318
|
+
if isinstance(groups, str):
|
|
1319
|
+
groups = [group.strip() for group in groups.strip("[]").replace(",", " ").split()]
|
|
1320
|
+
subject = identity.get("sub")
|
|
1321
|
+
if not isinstance(subject, str) or not subject or not moderation_rules.STAFF_GROUPS.intersection(groups):
|
|
1322
|
+
return moderation_rules.refused("not_staff", "Only editors and admins can moderate.").to_dict()
|
|
1323
|
+
arguments = event.get("arguments") or {}
|
|
1324
|
+
outcome = moderation_rules.apply_action(
|
|
1325
|
+
store,
|
|
1326
|
+
action=arguments.get("action"),
|
|
1327
|
+
target_type=arguments.get("targetType"),
|
|
1328
|
+
target=arguments.get("target"),
|
|
1329
|
+
reason=arguments.get("reason"),
|
|
1330
|
+
undo_of=arguments.get("undoOf"),
|
|
1331
|
+
actor_sub=subject,
|
|
1332
|
+
by=moderation_rules.BY_HUMAN,
|
|
1333
|
+
now_iso=iso(now_seconds),
|
|
1334
|
+
settings=moderation_rules.ModerationSettings(item_type=settings.promote_item_type, section=settings.promote_section),
|
|
1335
|
+
)
|
|
1336
|
+
return outcome.to_dict()
|
|
1337
|
+
|
|
1338
|
+
|
|
537
1339
|
def handler(event: dict[str, Any], _context: Any = None) -> dict[str, Any]:
|
|
538
1340
|
import boto3
|
|
539
1341
|
|
|
540
1342
|
settings = settings_from_environment()
|
|
541
1343
|
store = store_from_environment()
|
|
542
1344
|
now_seconds = time.time()
|
|
543
|
-
|
|
1345
|
+
if "arguments" in event and "info" in event:
|
|
1346
|
+
return handle_staff_event(event, store=store, settings=settings, now_seconds=now_seconds)
|
|
1347
|
+
target = snapshot_target_from_environment()
|
|
1348
|
+
if settings.moderation_mode == "ai" and target is not None and _context is not None:
|
|
1349
|
+
if event.get("worker") == AI_WORKER:
|
|
1350
|
+
remaining = _context.get_remaining_time_in_millis() / 1000 - AI_DEADLINE_MARGIN_SECONDS
|
|
1351
|
+
lambda_client = boto3.client("lambda")
|
|
1352
|
+
return run_ai_worker(
|
|
1353
|
+
store=store,
|
|
1354
|
+
settings=settings,
|
|
1355
|
+
now=time.time,
|
|
1356
|
+
deadline=time.time() + max(remaining, 5),
|
|
1357
|
+
read_key=lambda: read_api_key(settings, boto3.client("ssm")),
|
|
1358
|
+
model_factory=lambda key, model: ai_moderation.OpenAIModerationModel(key, model=model),
|
|
1359
|
+
invoke_snapshot=lambda: lambda_client.invoke(FunctionName=target.function_arn, InvocationType="Event", Payload=json.dumps(SNAPSHOT_TASK).encode("utf-8")),
|
|
1360
|
+
)
|
|
1361
|
+
target = target.for_ai_review(_context.invoked_function_arn)
|
|
1362
|
+
scheduler = snapshot_scheduler(store, settings, now_seconds, target, boto3.client("scheduler"))
|
|
544
1363
|
return handle_event(event, store=store, settings=settings, now_seconds=now_seconds, schedule_snapshot=scheduler)
|