@bygghammar/cms-platform 0.95.0 → 0.97.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/docs/PAGES.md +45 -2
- package/package.json +3 -1
- package/src/admin/describe-log.ts +2 -0
- package/src/backoffice/BookingLandingSection.tsx +672 -0
- package/src/backoffice/BookingSettingsCard.tsx +34 -4
- package/src/backoffice/PageEditorView.tsx +47 -24
- package/src/backoffice/routes/host.ts +7 -0
- package/src/backoffice/routes/index.tsx +22 -3
- package/src/blocks/components/CountdownBlock.tsx +3 -1
- package/src/blocks/components/CtaBannerBlock.tsx +8 -2
- package/src/blocks/components/HeroFullBlock.tsx +7 -1
- package/src/blocks/components/IconGridBlock.tsx +5 -1
- package/src/blocks/components/InstagramBlock.tsx +5 -1
- package/src/blocks/components/InstagramTiles.tsx +3 -1
- package/src/blocks/components/LinkTilesBlock.tsx +8 -1
- package/src/blocks/components/MediaSplitBlock.tsx +8 -3
- package/src/blocks/components/PriceTableBlock.tsx +6 -5
- package/src/blocks/components/TripListBlock.tsx +5 -1
- package/src/cms/booking-landing.ts +359 -0
- package/src/cms/booking-links.ts +24 -0
- package/src/cms/page-visibility.ts +67 -1
- package/src/forms/form-audit.ts +22 -2
- package/src/html/html-tokenizer.ts +254 -0
- package/src/html/sanitize-html.ts +71 -36
- package/src/navigation/visibility.ts +65 -3
- package/src/popups/runtime.tsx +75 -32
- package/src/server/forms-core.ts +9 -0
- package/src/server/forms-delivery-core.ts +29 -22
- package/src/server/forms-submissions-core.ts +8 -5
- package/src/server/media-usage-core.ts +53 -13
- package/src/server/outbound-address.ts +131 -0
- package/src/server/outbound-http.ts +565 -0
- package/src/server/page-visibility-core.ts +231 -5
- package/src/server/popup-optin-webhook-core.ts +79 -140
- package/src/server/schemas.ts +45 -0
- package/src/site/BookingOffNotice.tsx +73 -0
- package/src/site/PageRenderer.tsx +59 -3
- package/src/site/link-visibility.tsx +21 -2
- package/src/site/preview-messaging.ts +232 -0
- package/src/site/routes/preview-page.tsx +37 -29
- package/src/util/redact-url.ts +18 -0
package/src/popups/runtime.tsx
CHANGED
|
@@ -25,7 +25,8 @@ import type {
|
|
|
25
25
|
import { ManagedForm } from "../forms/FormRenderer";
|
|
26
26
|
import { onFormSubmissionSuccess } from "../forms/success-listeners";
|
|
27
27
|
import { siteConfig } from "../config/site-config";
|
|
28
|
-
import { containsBookingTarget, isBookingHref, stripBookingDeep } from "../cms/booking-links";
|
|
28
|
+
import { containsBookingTarget, containsLinkTo, isBookingHref, stripBookingDeep } from "../cms/booking-links";
|
|
29
|
+
import { isHiddenFromDiscovery, type LinkVisibilityPolicy } from "../navigation/visibility";
|
|
29
30
|
import { useLinkVisibility } from "../site/link-visibility";
|
|
30
31
|
import type { PopupPublicHost } from "./host";
|
|
31
32
|
import { popupPublicHost } from "./host";
|
|
@@ -111,14 +112,25 @@ export function PopupModuleRuntime(): ReactNode {
|
|
|
111
112
|
* it found through the same rules, and only the latest read counts, so one
|
|
112
113
|
* begun under the old policy cannot bring booking back. Without a policy
|
|
113
114
|
* provider booking is visible and nothing changes.
|
|
115
|
+
*
|
|
116
|
+
* The same goes for a popup that leads to a page hidden from the website
|
|
117
|
+
* while booking is hidden (a booking-off landing page): not offered, and
|
|
118
|
+
* retired if it is the one on screen, until the page is shown again.
|
|
114
119
|
*/
|
|
115
|
-
const
|
|
120
|
+
const linkPolicy = useLinkVisibility();
|
|
121
|
+
const bookingVisible = linkPolicy.bookingVisible;
|
|
122
|
+
const hiddenKey = hiddenPagesKey(linkPolicy);
|
|
116
123
|
const [loaded, setLoaded] = useState<readonly PopupConfig[] | null>(null);
|
|
117
124
|
const offered = useRef<Offered>(NOTHING_OFFERED);
|
|
118
125
|
const offer = useMemo(
|
|
119
|
-
() =>
|
|
120
|
-
|
|
121
|
-
|
|
126
|
+
() =>
|
|
127
|
+
popupsToOffer(loaded ?? [], offered.current, bookingVisible, (href) =>
|
|
128
|
+
isHiddenFromDiscovery(href, linkPolicy),
|
|
129
|
+
),
|
|
130
|
+
// `offered` is what the runtime was last handed, read for identity only;
|
|
131
|
+
// the policy matters only through what `hiddenKey` names.
|
|
132
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
133
|
+
[loaded, bookingVisible, hiddenKey],
|
|
122
134
|
);
|
|
123
135
|
useEffect(() => {
|
|
124
136
|
offered.current = offer;
|
|
@@ -126,14 +138,23 @@ export function PopupModuleRuntime(): ReactNode {
|
|
|
126
138
|
|
|
127
139
|
// The repository is asked again when the host or the policy changes — and
|
|
128
140
|
// only once the runtime asks for its popups, so never where it does not run.
|
|
129
|
-
const askedFor = useRef<{
|
|
141
|
+
const askedFor = useRef<{
|
|
142
|
+
configs: PopupPublicHost["configs"];
|
|
143
|
+
bookingVisible: boolean;
|
|
144
|
+
hiddenKey: string;
|
|
145
|
+
} | null>(null);
|
|
130
146
|
const latestRead = useRef(0);
|
|
131
147
|
const configs = useMemo<RuntimeAdapters["configs"]>(
|
|
132
148
|
() => ({
|
|
133
149
|
listActive: async () => {
|
|
134
150
|
const asked = askedFor.current;
|
|
135
|
-
if (
|
|
136
|
-
|
|
151
|
+
if (
|
|
152
|
+
asked === null ||
|
|
153
|
+
asked.configs !== host.configs ||
|
|
154
|
+
asked.bookingVisible !== bookingVisible ||
|
|
155
|
+
asked.hiddenKey !== hiddenKey
|
|
156
|
+
) {
|
|
157
|
+
askedFor.current = { configs: host.configs, bookingVisible, hiddenKey };
|
|
137
158
|
const read = ++latestRead.current;
|
|
138
159
|
Promise.resolve()
|
|
139
160
|
.then(() => host.configs.listActive())
|
|
@@ -152,7 +173,7 @@ export function PopupModuleRuntime(): ReactNode {
|
|
|
152
173
|
return offer.list;
|
|
153
174
|
},
|
|
154
175
|
}),
|
|
155
|
-
[host.configs, bookingVisible, offer],
|
|
176
|
+
[host.configs, bookingVisible, hiddenKey, offer],
|
|
156
177
|
);
|
|
157
178
|
|
|
158
179
|
// The argument type comes from the port, not a copy of it: a package that
|
|
@@ -189,6 +210,15 @@ type ContentBlock = PopupConfig["steps"][number]["blocks"][number];
|
|
|
189
210
|
type PopupButton = Extract<ContentBlock, { type: "buttons" }>["buttons"][number];
|
|
190
211
|
type BlockAction = PopupButton["action"];
|
|
191
212
|
|
|
213
|
+
/** The booking-off pages a policy hides, as one comparable string. */
|
|
214
|
+
function hiddenPagesKey(policy: LinkVisibilityPolicy): string {
|
|
215
|
+
return Object.entries(policy.hiddenPaths)
|
|
216
|
+
.filter(([, reason]) => reason === "booking-off")
|
|
217
|
+
.map(([path]) => path)
|
|
218
|
+
.sort()
|
|
219
|
+
.join("\n");
|
|
220
|
+
}
|
|
221
|
+
|
|
192
222
|
/** The list last handed to the runtime, and each popup's content, by key. */
|
|
193
223
|
type Offered = { list: readonly PopupConfig[]; content: ReadonlyMap<string, string> };
|
|
194
224
|
const NOTHING_OFFERED: Offered = { list: [], content: new Map() };
|
|
@@ -197,7 +227,8 @@ const NOTHING_OFFERED: Offered = { list: [], content: new Map() };
|
|
|
197
227
|
* What the runtime is handed from a load, given what it already holds.
|
|
198
228
|
*
|
|
199
229
|
* - While booking is hidden, a popup that leads to booking is not offered: it
|
|
200
|
-
* is handed over retired (see {@link retire}).
|
|
230
|
+
* is handed over retired (see {@link retire}). So is a popup that leads to
|
|
231
|
+
* a page hidden while booking is hidden (`hiddenPage`).
|
|
201
232
|
* - A popup the runtime already holds is never dropped, even when the load no
|
|
202
233
|
* longer has it (a server that withholds booking popups, a popup paused or
|
|
203
234
|
* deleted since). The runtime tracks the open popup by key, so dropping the
|
|
@@ -213,7 +244,11 @@ function popupsToOffer(
|
|
|
213
244
|
loaded: readonly PopupConfig[],
|
|
214
245
|
previous: Offered,
|
|
215
246
|
bookingVisible: boolean,
|
|
247
|
+
hiddenPage: (href: string) => boolean,
|
|
216
248
|
): Offered {
|
|
249
|
+
const hideBooking = !bookingVisible;
|
|
250
|
+
const withheld = (popup: PopupConfig) =>
|
|
251
|
+
(hideBooking && containsBookingTarget(popup)) || containsLinkTo(popup, hiddenPage);
|
|
217
252
|
const held = new Map(previous.list.map((popup) => [popup.key, popup]));
|
|
218
253
|
const content = new Map<string, string>();
|
|
219
254
|
const list: PopupConfig[] = [];
|
|
@@ -224,10 +259,10 @@ function popupsToOffer(
|
|
|
224
259
|
list.push(same !== undefined && previous.content.get(popup.key) === text ? same : popup);
|
|
225
260
|
};
|
|
226
261
|
for (const popup of loaded) {
|
|
227
|
-
offer(
|
|
262
|
+
offer(withheld(popup) ? retire(popup, hideBooking, hiddenPage) : popup);
|
|
228
263
|
}
|
|
229
264
|
for (const popup of previous.list) {
|
|
230
|
-
if (!content.has(popup.key)) offer(retire(popup,
|
|
265
|
+
if (!content.has(popup.key)) offer(retire(popup, hideBooking, hiddenPage));
|
|
231
266
|
}
|
|
232
267
|
const unchanged =
|
|
233
268
|
list.length === previous.list.length && list.every((popup, index) => popup === previous.list[index]);
|
|
@@ -237,43 +272,51 @@ function popupsToOffer(
|
|
|
237
272
|
/**
|
|
238
273
|
* A popup the runtime keeps but never offers again.
|
|
239
274
|
*
|
|
240
|
-
* Paused, which the runtime neither shows nor reports as suppressed.
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
275
|
+
* Paused, which the runtime neither shows nor reports as suppressed. Every
|
|
276
|
+
* place in it that leads to booking (with `hideBooking`) or to a page hidden
|
|
277
|
+
* while booking is hidden (`hiddenPage`) is taken out too: links and
|
|
278
|
+
* buttons go, an answer or a form that would open it now simply closes. So if
|
|
279
|
+
* it is the one on screen, the visitor keeps what they were reading, cannot
|
|
280
|
+
* follow it anywhere it must not lead, and closes it the usual way. The
|
|
281
|
+
* stored popup is not touched; this is the copy the visitor sees.
|
|
246
282
|
*/
|
|
247
|
-
function retire(
|
|
283
|
+
function retire(
|
|
284
|
+
popup: PopupConfig,
|
|
285
|
+
hideBooking: boolean,
|
|
286
|
+
hiddenPage: (href: string) => boolean,
|
|
287
|
+
): PopupConfig {
|
|
288
|
+
const blocked = (href: string) => (hideBooking && isBookingHref(href)) || hiddenPage(href);
|
|
248
289
|
const paused: PopupConfig = { ...popup, status: "paused" };
|
|
249
|
-
|
|
250
|
-
return stripBookingDeep({
|
|
290
|
+
const cleaned: PopupConfig = {
|
|
251
291
|
...paused,
|
|
252
|
-
steps: paused.steps.map((step) => ({
|
|
253
|
-
|
|
292
|
+
steps: paused.steps.map((step) => ({
|
|
293
|
+
...step,
|
|
294
|
+
blocks: step.blocks.flatMap((block) => withoutTargets(block, blocked)),
|
|
295
|
+
})),
|
|
296
|
+
};
|
|
297
|
+
// Booking links inside text are removed as before, and only for booking.
|
|
298
|
+
return hideBooking ? stripBookingDeep(cleaned) : cleaned;
|
|
254
299
|
}
|
|
255
300
|
|
|
256
|
-
function
|
|
301
|
+
function withoutTargets(block: ContentBlock, blocked: (href: string) => boolean): ContentBlock[] {
|
|
302
|
+
const opens = (action: BlockAction) => action.kind === "openUrl" && blocked(action.href);
|
|
303
|
+
const closingInstead = (button: PopupButton): PopupButton =>
|
|
304
|
+
opens(button.action) ? { ...button, action: { kind: "close" } } : button;
|
|
257
305
|
switch (block.type) {
|
|
258
306
|
case "link":
|
|
259
|
-
return
|
|
307
|
+
return blocked(block.href) ? [] : [block];
|
|
260
308
|
case "buttons": {
|
|
261
|
-
const buttons = block.buttons.filter((button) => !
|
|
309
|
+
const buttons = block.buttons.filter((button) => !opens(button.action));
|
|
262
310
|
return buttons.length === 0 ? [] : [{ ...block, buttons }];
|
|
263
311
|
}
|
|
264
312
|
case "choice":
|
|
265
313
|
return [{ ...block, yes: closingInstead(block.yes), no: closingInstead(block.no) }];
|
|
266
314
|
case "form":
|
|
267
|
-
return [
|
|
315
|
+
return [opens(block.onSuccess) ? { ...block, onSuccess: { kind: "close" } } : block];
|
|
268
316
|
default:
|
|
269
317
|
return [block];
|
|
270
318
|
}
|
|
271
319
|
}
|
|
272
320
|
|
|
273
|
-
const opensBooking = (action: BlockAction) => action.kind === "openUrl" && isBookingHref(action.href);
|
|
274
|
-
|
|
275
|
-
const closingInstead = (button: PopupButton): PopupButton =>
|
|
276
|
-
opensBooking(button.action) ? { ...button, action: { kind: "close" } } : button;
|
|
277
|
-
|
|
278
321
|
// Default export so the module host can load this file lazily.
|
|
279
322
|
export default PopupModuleRuntime;
|
package/src/server/forms-core.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* - a field key that already has stored answers can never be removed
|
|
12
12
|
* - snapshots and audit entries never block the write they describe
|
|
13
13
|
*/
|
|
14
|
+
import { checkWebhookUrl } from "./webhook-url-guard";
|
|
14
15
|
import {
|
|
15
16
|
defaultFieldsFor,
|
|
16
17
|
mergeFields,
|
|
@@ -410,6 +411,14 @@ export async function saveFormCore(
|
|
|
410
411
|
}
|
|
411
412
|
}
|
|
412
413
|
|
|
414
|
+
// A webhook address is checked when it is saved, not first when a visitor's
|
|
415
|
+
// submission tries to reach it (where it is checked again, on every hop).
|
|
416
|
+
const webhookUrl = (data.private_settings as { webhookUrl?: unknown }).webhookUrl;
|
|
417
|
+
if (typeof webhookUrl === "string" && webhookUrl.trim() !== "") {
|
|
418
|
+
const checked = checkWebhookUrl(webhookUrl);
|
|
419
|
+
if (!checked.ok) throw new Error(`Webhook URL: ${checked.message}`);
|
|
420
|
+
}
|
|
421
|
+
|
|
413
422
|
const row = {
|
|
414
423
|
key: data.key,
|
|
415
424
|
name: data.name,
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import type { FormFieldDef } from "../forms/field-sets";
|
|
16
|
+
import { safeRequest, WEBHOOK_POLICY, type Resolver, type Transport } from "./outbound-http";
|
|
17
|
+
import { checkWebhookUrl } from "./webhook-url-guard";
|
|
16
18
|
import { sendViaProvider } from "./email-send-core";
|
|
17
19
|
import { logEmail } from "./email-log-core";
|
|
18
20
|
import { appendGoogleSheetRow, DEFAULT_WORKSHEET } from "./google-sheets-core";
|
|
@@ -489,6 +491,11 @@ export async function appendToSheetCore(opts: {
|
|
|
489
491
|
* POST one submission to an external endpoint (CRM, Zapier, Make…).
|
|
490
492
|
* Signed with an optional shared secret so the receiver can verify origin.
|
|
491
493
|
* Never throws: the submission is already stored and must never be lost.
|
|
494
|
+
*
|
|
495
|
+
* The endpoint is reached only through the shared outbound policy: https
|
|
496
|
+
* only, public addresses only (looked up, pinned, every redirect hop checked),
|
|
497
|
+
* one deadline and a capped reply. A failure says what kind it was; the
|
|
498
|
+
* endpoint's own reply is never copied into it.
|
|
492
499
|
*/
|
|
493
500
|
export async function postWebhookCore(opts: {
|
|
494
501
|
url: string;
|
|
@@ -501,11 +508,13 @@ export async function postWebhookCore(opts: {
|
|
|
501
508
|
createdAt: string;
|
|
502
509
|
payload: Record<string, string>;
|
|
503
510
|
timeoutMs?: number;
|
|
511
|
+
/** Test seams: a fake resolver and transport instead of the network. */
|
|
512
|
+
network?: { resolver?: Resolver; transport?: Transport };
|
|
504
513
|
}): Promise<DeliveryResult & { status?: number }> {
|
|
505
514
|
const url = (opts.url ?? "").trim();
|
|
506
515
|
if (!url) return { ok: false, error: "No webhook URL configured." };
|
|
507
|
-
|
|
508
|
-
|
|
516
|
+
const checked = checkWebhookUrl(url);
|
|
517
|
+
if (!checked.ok) return { ok: false, error: checked.message };
|
|
509
518
|
|
|
510
519
|
const body = JSON.stringify({
|
|
511
520
|
event: "form.submission",
|
|
@@ -530,28 +539,26 @@ export async function postWebhookCore(opts: {
|
|
|
530
539
|
const { createHmac } = await import("crypto");
|
|
531
540
|
headers["X-Signature"] = createHmac("sha256", secret).update(body).digest("hex");
|
|
532
541
|
} catch {
|
|
533
|
-
|
|
542
|
+
// A receiver that was promised a signature must never get an unsigned
|
|
543
|
+
// request it cannot tell from a forgery.
|
|
544
|
+
return { ok: false, error: "Webhook not sent: it could not be signed." };
|
|
534
545
|
}
|
|
535
546
|
}
|
|
536
547
|
|
|
537
|
-
const
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
return { ok:
|
|
551
|
-
} catch (e) {
|
|
552
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
553
|
-
return { ok: false, error: msg === "The operation was aborted." ? "Webhook timed out." : msg };
|
|
554
|
-
} finally {
|
|
555
|
-
clearTimeout(timer);
|
|
548
|
+
const res = await safeRequest({
|
|
549
|
+
url: checked.url.toString(),
|
|
550
|
+
method: "POST",
|
|
551
|
+
headers,
|
|
552
|
+
body,
|
|
553
|
+
policy: { ...WEBHOOK_POLICY, timeoutMs: opts.timeoutMs ?? WEBHOOK_POLICY.timeoutMs },
|
|
554
|
+
resolver: opts.network?.resolver,
|
|
555
|
+
transport: opts.network?.transport,
|
|
556
|
+
});
|
|
557
|
+
if (!res.ok) {
|
|
558
|
+
return { ok: false, error: res.failure === "timeout" ? "Webhook timed out." : `Webhook not delivered: ${res.message}` };
|
|
559
|
+
}
|
|
560
|
+
if (res.status < 200 || res.status >= 300) {
|
|
561
|
+
return { ok: false, status: res.status, error: `Webhook answered ${res.status}.` };
|
|
556
562
|
}
|
|
563
|
+
return { ok: true, status: res.status };
|
|
557
564
|
}
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
type FormsDb,
|
|
35
35
|
} from "./forms-core";
|
|
36
36
|
import { auditAs } from "./audit-core";
|
|
37
|
+
import { redactUrl } from "./outbound-http";
|
|
37
38
|
import {
|
|
38
39
|
evaluateRule,
|
|
39
40
|
isFieldVisible,
|
|
@@ -358,7 +359,7 @@ export async function submitFormCore(admin: FormsDb, data: SubmitInput, meta: Re
|
|
|
358
359
|
formKey: form.key,
|
|
359
360
|
channel: "webhook",
|
|
360
361
|
ok: res.ok,
|
|
361
|
-
target: eff.webhookUrl,
|
|
362
|
+
target: redactUrl(eff.webhookUrl),
|
|
362
363
|
error: res.error,
|
|
363
364
|
durationMs: Date.now() - startedAt,
|
|
364
365
|
meta: { status: res.status ?? null },
|
|
@@ -1170,10 +1171,12 @@ export async function deliverChannelCore(
|
|
|
1170
1171
|
})
|
|
1171
1172
|
.eq("id", sub.id);
|
|
1172
1173
|
} else if (channel === "webhook") {
|
|
1173
|
-
|
|
1174
|
-
if (!
|
|
1174
|
+
const webhookUrl = toOverride?.trim() || eff.webhookUrl;
|
|
1175
|
+
if (!webhookUrl) throw new Error(`No webhook URL configured for "${form.key}".`);
|
|
1176
|
+
// Recorded without its query string, which may carry the receiver's key.
|
|
1177
|
+
target = redactUrl(webhookUrl);
|
|
1175
1178
|
result = await postWebhookCore({
|
|
1176
|
-
url:
|
|
1179
|
+
url: webhookUrl,
|
|
1177
1180
|
secret: eff.webhookSecret,
|
|
1178
1181
|
formKey: form.key,
|
|
1179
1182
|
formName: form.name,
|
|
@@ -1385,7 +1388,7 @@ export async function testWebhookCore(
|
|
|
1385
1388
|
formKey: form.key,
|
|
1386
1389
|
channel: "webhook",
|
|
1387
1390
|
ok: res.ok,
|
|
1388
|
-
target: url,
|
|
1391
|
+
target: redactUrl(url),
|
|
1389
1392
|
error: res.error,
|
|
1390
1393
|
durationMs: Date.now() - startedAt,
|
|
1391
1394
|
meta: { test: true, status: res.status ?? null },
|
|
@@ -10,6 +10,15 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { variantSiblings } from "../media/srcset";
|
|
12
12
|
import { cleanFileName, splitExt, uniqueFileName } from "../media/filename";
|
|
13
|
+
import {
|
|
14
|
+
MEDIA_IMPORT_POLICY,
|
|
15
|
+
redactUrl,
|
|
16
|
+
safeRequest,
|
|
17
|
+
sniffMedia,
|
|
18
|
+
type MediaKind,
|
|
19
|
+
type Resolver,
|
|
20
|
+
type Transport,
|
|
21
|
+
} from "./outbound-http";
|
|
13
22
|
import type { MediaImportResult, MediaUsage, MediaUsageReport } from "../media/usage-types";
|
|
14
23
|
|
|
15
24
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
@@ -42,6 +51,8 @@ export type MediaImportHost = MediaUsageHost & {
|
|
|
42
51
|
admin: any;
|
|
43
52
|
/** User-Agent sent when fetching remote media. */
|
|
44
53
|
userAgent?: string;
|
|
54
|
+
/** Test seams: a fake resolver and transport instead of the network. */
|
|
55
|
+
network?: { resolver?: Resolver; transport?: Transport };
|
|
45
56
|
};
|
|
46
57
|
|
|
47
58
|
const URL_RE = /https?:\/\/[^\s"'<>)]+/gi;
|
|
@@ -154,28 +165,55 @@ export function remoteUrlCandidates(rawUrl: string): string[] {
|
|
|
154
165
|
return Array.from(candidates);
|
|
155
166
|
}
|
|
156
167
|
|
|
157
|
-
|
|
168
|
+
/**
|
|
169
|
+
* Download one remote media file, under the shared outbound policy: public
|
|
170
|
+
* addresses only (looked up, pinned, every redirect hop checked), one
|
|
171
|
+
* deadline, a capped size. What it is comes from its first bytes — never from
|
|
172
|
+
* the server's content type or the address's extension.
|
|
173
|
+
*/
|
|
174
|
+
async function fetchRemoteMedia(
|
|
175
|
+
rawUrl: string,
|
|
176
|
+
userAgent: string,
|
|
177
|
+
network: MediaImportHost["network"],
|
|
178
|
+
): Promise<{ bytes: Uint8Array; kind: MediaKind; resolvedUrl: string } | { error: string }> {
|
|
158
179
|
let lastStatus: number | null = null;
|
|
159
180
|
let lastError: string | null = null;
|
|
160
181
|
|
|
161
182
|
const headers = {
|
|
162
183
|
"User-Agent": userAgent,
|
|
163
|
-
Accept:
|
|
164
|
-
"image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,application/pdf,*/*;q=0.8",
|
|
184
|
+
Accept: "image/avif,image/webp,image/apng,image/*,video/*,audio/*,application/pdf;q=0.9",
|
|
165
185
|
};
|
|
166
186
|
|
|
167
187
|
for (const url of remoteUrlCandidates(rawUrl)) {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
188
|
+
const res = await safeRequest({
|
|
189
|
+
url,
|
|
190
|
+
method: "GET",
|
|
191
|
+
headers,
|
|
192
|
+
policy: MEDIA_IMPORT_POLICY,
|
|
193
|
+
resolver: network?.resolver,
|
|
194
|
+
transport: network?.transport,
|
|
195
|
+
});
|
|
196
|
+
if (!res.ok) {
|
|
197
|
+
// Every candidate is a spelling of the same address: a refused one is refused for all.
|
|
198
|
+
if (res.failure === "blocked-url" || res.failure === "blocked-address") {
|
|
199
|
+
return { error: `${res.message} (${redactUrl(rawUrl)})` };
|
|
200
|
+
}
|
|
201
|
+
lastError = res.message;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (res.status < 200 || res.status >= 300) {
|
|
205
|
+
lastStatus = res.status;
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const kind = sniffMedia(res.body);
|
|
209
|
+
if (!kind) {
|
|
210
|
+
return { error: `Not an image, video, audio or PDF file (${redactUrl(rawUrl)}).` };
|
|
174
211
|
}
|
|
212
|
+
return { bytes: res.body, kind, resolvedUrl: url };
|
|
175
213
|
}
|
|
176
214
|
|
|
177
215
|
const reason = lastStatus ? `Fetch failed: ${lastStatus}` : (lastError ?? "Fetch failed");
|
|
178
|
-
return { error: `${reason}
|
|
216
|
+
return { error: `${reason} (${redactUrl(rawUrl)})` };
|
|
179
217
|
}
|
|
180
218
|
|
|
181
219
|
/** Scan every CMS content column and report which library files are used where. */
|
|
@@ -417,19 +455,21 @@ export async function importRemoteUrlCore(
|
|
|
417
455
|
const remote = await fetchRemoteMedia(
|
|
418
456
|
data.url,
|
|
419
457
|
host.userAgent ?? "Mozilla/5.0 (compatible; CmsMediaImporter/1.0)",
|
|
458
|
+
host.network,
|
|
420
459
|
);
|
|
421
460
|
if ("error" in remote) {
|
|
422
461
|
return { path: "", url: "", skipped: true, error: remote.error ?? "Fetch failed" };
|
|
423
462
|
}
|
|
424
463
|
|
|
425
|
-
|
|
426
|
-
const
|
|
464
|
+
// Stored as what the bytes are: their sniffed type and its extension.
|
|
465
|
+
const contentType = remote.kind.mime;
|
|
466
|
+
const buffer = remote.bytes;
|
|
427
467
|
const urlObj = new URL(remote.resolvedUrl);
|
|
428
468
|
const rawName =
|
|
429
469
|
data.filename ||
|
|
430
470
|
urlObj.pathname.split("/").filter(Boolean).map(safeDecodeURIComponent).pop() ||
|
|
431
471
|
"imported-file";
|
|
432
|
-
const clean = cleanFileName(rawName, "imported-file")
|
|
472
|
+
const clean = `${splitExt(cleanFileName(rawName.replace(/[\\/]/g, "-"), "imported-file")).base}${remote.kind.ext}`;
|
|
433
473
|
|
|
434
474
|
const all = await host.storage.list();
|
|
435
475
|
const existing = all
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which network addresses a server-side request may connect to.
|
|
3
|
+
*
|
|
4
|
+
* Only globally routable unicast addresses. Everything that names this
|
|
5
|
+
* machine, its private network, a cloud metadata service, a multicast group
|
|
6
|
+
* or a reserved block is refused — in IPv4 and IPv6, and in every IPv6 form
|
|
7
|
+
* that carries an IPv4 address inside it (mapped, compatible, NAT64, 6to4).
|
|
8
|
+
*
|
|
9
|
+
* Pure and runtime-independent: it only reads address text.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/** IPv4 blocks that are not public destinations: [network, prefix length]. */
|
|
13
|
+
const BLOCKED_V4: ReadonlyArray<readonly [string, number]> = [
|
|
14
|
+
["0.0.0.0", 8], // "this network"
|
|
15
|
+
["10.0.0.0", 8], // private
|
|
16
|
+
["100.64.0.0", 10], // carrier-grade NAT
|
|
17
|
+
["127.0.0.0", 8], // loopback
|
|
18
|
+
["169.254.0.0", 16], // link-local, incl. cloud metadata 169.254.169.254
|
|
19
|
+
["172.16.0.0", 12], // private
|
|
20
|
+
["192.0.0.0", 24], // IETF protocol assignments
|
|
21
|
+
["192.0.2.0", 24], // documentation
|
|
22
|
+
["192.88.99.0", 24], // 6to4 relay anycast
|
|
23
|
+
["192.168.0.0", 16], // private
|
|
24
|
+
["198.18.0.0", 15], // benchmarking
|
|
25
|
+
["198.51.100.0", 24], // documentation
|
|
26
|
+
["203.0.113.0", 24], // documentation
|
|
27
|
+
["224.0.0.0", 4], // multicast
|
|
28
|
+
["240.0.0.0", 4], // reserved, incl. broadcast
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
/** IPv6 blocks that are not public destinations. */
|
|
32
|
+
const BLOCKED_V6: ReadonlyArray<readonly [string, number]> = [
|
|
33
|
+
["::", 128], // unspecified
|
|
34
|
+
["::1", 128], // loopback
|
|
35
|
+
["::ffff:0:0", 96], // IPv4-mapped
|
|
36
|
+
["::", 96], // IPv4-compatible (deprecated)
|
|
37
|
+
["64:ff9b::", 96], // NAT64
|
|
38
|
+
["64:ff9b:1::", 48], // local-use NAT64
|
|
39
|
+
["100::", 64], // discard
|
|
40
|
+
["2001::", 23], // IETF protocol assignments, incl. Teredo
|
|
41
|
+
["2001:db8::", 32], // documentation
|
|
42
|
+
["2002::", 16], // 6to4
|
|
43
|
+
["3fff::", 20], // documentation
|
|
44
|
+
["5f00::", 16], // segment routing
|
|
45
|
+
["fc00::", 7], // unique local, incl. fd00:ec2::254 (cloud metadata)
|
|
46
|
+
["fe80::", 10], // link-local
|
|
47
|
+
["fec0::", 10], // site-local (deprecated)
|
|
48
|
+
["ff00::", 8], // multicast
|
|
49
|
+
];
|
|
50
|
+
|
|
51
|
+
/** An IPv4 address as a 32-bit number, or null. Dotted decimal only (as DNS and URL produce). */
|
|
52
|
+
export function parseIPv4(text: string): number | null {
|
|
53
|
+
const parts = text.split(".");
|
|
54
|
+
if (parts.length !== 4) return null;
|
|
55
|
+
let value = 0;
|
|
56
|
+
for (const part of parts) {
|
|
57
|
+
if (!/^(0|[1-9][0-9]{0,2})$/.test(part)) return null;
|
|
58
|
+
const octet = Number(part);
|
|
59
|
+
if (octet > 255) return null;
|
|
60
|
+
value = value * 256 + octet;
|
|
61
|
+
}
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** An IPv6 address as a 128-bit number, or null. Zone ids are refused. */
|
|
66
|
+
export function parseIPv6(text: string): bigint | null {
|
|
67
|
+
let address = text.trim().toLowerCase();
|
|
68
|
+
if (address.startsWith("[") && address.endsWith("]")) address = address.slice(1, -1);
|
|
69
|
+
if (address === "" || address.includes("%")) return null;
|
|
70
|
+
|
|
71
|
+
// An IPv4 tail ("::ffff:1.2.3.4") becomes two hex groups.
|
|
72
|
+
const lastColon = address.lastIndexOf(":");
|
|
73
|
+
const tail = address.slice(lastColon + 1);
|
|
74
|
+
if (tail.includes(".")) {
|
|
75
|
+
const v4 = parseIPv4(tail);
|
|
76
|
+
if (v4 === null) return null;
|
|
77
|
+
address = `${address.slice(0, lastColon + 1)}${(v4 >>> 16).toString(16)}:${(v4 & 0xffff).toString(16)}`;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const halves = address.split("::");
|
|
81
|
+
if (halves.length > 2) return null;
|
|
82
|
+
const parse = (half: string) => (half === "" ? [] : half.split(":"));
|
|
83
|
+
const head = parse(halves[0] ?? "");
|
|
84
|
+
const rest = halves.length === 2 ? parse(halves[1] ?? "") : [];
|
|
85
|
+
const zeros = 8 - head.length - rest.length;
|
|
86
|
+
// "::" stands for at least one group of zeros; without it there are exactly eight groups.
|
|
87
|
+
if (halves.length === 2 ? zeros < 1 : zeros !== 0) return null;
|
|
88
|
+
const groups = [...head, ...Array<string>(halves.length === 2 ? zeros : 0).fill("0"), ...rest];
|
|
89
|
+
|
|
90
|
+
let value = 0n;
|
|
91
|
+
for (const group of groups) {
|
|
92
|
+
if (!/^[0-9a-f]{1,4}$/.test(group)) return null;
|
|
93
|
+
value = (value << 16n) | BigInt(Number.parseInt(group, 16));
|
|
94
|
+
}
|
|
95
|
+
return value;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function inV4Block(address: number, [network, prefix]: readonly [string, number]): boolean {
|
|
99
|
+
const base = parseIPv4(network)!;
|
|
100
|
+
if (prefix === 0) return true;
|
|
101
|
+
const shift = 32 - prefix;
|
|
102
|
+
return Math.floor(address / 2 ** shift) === Math.floor(base / 2 ** shift);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function inV6Block(address: bigint, [network, prefix]: readonly [string, number]): boolean {
|
|
106
|
+
const base = parseIPv6(network)!;
|
|
107
|
+
const shift = BigInt(128 - prefix);
|
|
108
|
+
return address >> shift === base >> shift;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export type AddressFamily = 4 | 6;
|
|
112
|
+
|
|
113
|
+
/** The family of an address literal, or null when it is not one. */
|
|
114
|
+
export function addressFamily(text: string): AddressFamily | null {
|
|
115
|
+
if (parseIPv4(text) !== null) return 4;
|
|
116
|
+
if (parseIPv6(text) !== null) return 6;
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** True only for a globally routable unicast address. Anything unparseable is refused. */
|
|
121
|
+
export function isPublicAddress(text: string): boolean {
|
|
122
|
+
const v4 = parseIPv4(text);
|
|
123
|
+
if (v4 !== null) return !BLOCKED_V4.some((block) => inV4Block(v4, block));
|
|
124
|
+
const v6 = parseIPv6(text);
|
|
125
|
+
if (v6 !== null) {
|
|
126
|
+
if (BLOCKED_V6.some((block) => inV6Block(v6, block))) return false;
|
|
127
|
+
// Only the global unicast range 2000::/3 is public.
|
|
128
|
+
return v6 >> 125n === 1n;
|
|
129
|
+
}
|
|
130
|
+
return false;
|
|
131
|
+
}
|