@tideworthy/tidysites 0.1.13 → 0.1.15

package/README.md

@@ -63,7 +63,7 @@ shows in `status`.
 Styling: a row's `.css` file is scoped to that row (every rule is prefixed with the row's own selector), so it cannot style the rest of the page; rules for the whole page, the sections and the theme's classes go in `business/business.css`.
 
 Images: an image field, a hero's image or a Columns image slot takes
-`{ "$media": { "url": "https://...", "alt": "what it shows" } }` and the
+`{ "$media": { "url": "https://...", "alt": "what it shows" } }` (or `{ "$media": { "file": "assets/x.png", "alt": "..." } }` for a picture that exists only as a file in the folder, say an illustration drawn from the old page's inline SVG; the push carries its bytes; a `file` field, such as a row's `media` list or the Business font files, takes a video (mp4, webm, up to 40 MB), a font (woff2) or a pdf the same way) and the
 push fetches the file into the website's library; a file the library already
 holds (the same bytes) is reused, not added again. An image already in the
 library keeps the value the pull gave it.

package/bin/tidysites.mjs

@@ -322,6 +322,9 @@ async function push(ctx) {
 	}
 	const doc = pushDocument(s.local, s.snap, { keys, removeKeys });
 	if (flags.force) doc.force = true;
+	// A picture that exists only as a file here (an illustration drawn from the old page's inline SVG, a rasterised
+	// icon) goes up with the push as its bytes; the website stores it in the library like a fetched one.
+	inlineMediaFiles(doc, ctx.dir);
 	if (flags.dry) {
 		say(`Would push ${keys.length} entr${keys.length === 1 ? 'y' : 'ies'}${removeKeys.length ? `, remove ${removeKeys.length}` : ''}${menus.length ? `, rewrite menu${menus.length === 1 ? '' : 's'} ${menus.join(', ')}` : ''}:`);
 		printStatus(s);
@@ -366,6 +369,32 @@ async function push(ctx) {
 	else if (menus.length) await pullMenus(ctx);
 }
 
+const MEDIA_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml', ico: 'image/x-icon', mp4: 'video/mp4', m4v: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', woff2: 'font/woff2', woff: 'font/woff', ttf: 'font/ttf', otf: 'font/otf', pdf: 'application/pdf' };
+/** Pictures travel up to 15 MB; a video (a row's media list) up to 40 MB. */
+const mediaCap = (type) => (type.startsWith('video/') ? 40 : 15);
+/** Every { "$media": { "file": ... } } in the document becomes { "$media": { "data", "type", "filename", "alt" } }, read from the folder. */
+function inlineMediaFiles(doc, dir) {
+	const walk = (v) => {
+		if (Array.isArray(v)) return v.map(walk);
+		if (v && typeof v === 'object') {
+			const m = v.$media;
+			if (m && typeof m === 'object' && typeof m.file === 'string') {
+				const path = resolve(dir, m.file);
+				if (!path.startsWith(resolve(dir))) fail(`$media.file ${m.file}: outside the folder`);
+				if (!existsSync(path)) fail(`$media.file ${m.file}: no such file`);
+				const bytes = readFileSync(path);
+				const ext = m.file.split('.').pop().toLowerCase();
+				const type = MEDIA_TYPES[ext] ?? 'application/octet-stream';
+				if (bytes.length > mediaCap(type) * 1024 * 1024) fail(`$media.file ${m.file}: ${(bytes.length / 1048576).toFixed(1)} MB; ${type.startsWith('video/') ? 'videos up to 40 MB' : 'pictures up to 15 MB'}`);
+				return { $media: { data: bytes.toString('base64'), type, filename: m.file.split('/').pop(), alt: typeof m.alt === 'string' ? m.alt : '' } };
+			}
+			return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
+		}
+		return v;
+	};
+	if (doc.content) doc.content = walk(doc.content);
+}
+
 async function shot(ctx, target = positional[0] ?? '/', opts = {}) {
 	const isUrl = /^https?:\/\//.test(target);
 	// --desktop, --mobile, --tablet, --wide, any of them together; --all for every width; none = desktop and mobile.
@@ -633,7 +662,7 @@ async function publish(ctx) {
 
 function help() {
 	say(readFileSync(new URL(import.meta.url), 'utf8').split('\n').slice(2, 20).map((l) => l.replace(/^ \* ?/, '')).join('\n'));
-	say(`\nFiles:\n  <collection>/<slug>.json      { "id", "slug", "status", "updated_at", "data": {...} }; a new entry is a new file with "data"\n  <collection>/<slug>.css       data.custom_css as a file (rows: data.css); rows/<slug>.html is data.html\n  menus/<name>.json             a menu as a tree: { label, items: [{ label, url, children: [...] }] }\n  ${META}/                   config, key, the last pull, the content model, screenshots\n\nA field taken out of a file is cleared on the website at the next push; a file you changed here is never overwritten by a pull.\nImages: an image field or a Columns image slot takes { "$media": { "url": "https://...", "alt": "..." } }; the push fetches it into the library.\nReferences: a record names another by its id, or "$ref:<slug-or-id>" for one in the same push.`);
+	say(`\nFiles:\n  <collection>/<slug>.json      { "id", "slug", "status", "updated_at", "data": {...} }; a new entry is a new file with "data"\n  <collection>/<slug>.css       data.custom_css as a file (rows: data.css); rows/<slug>.html is data.html\n  menus/<name>.json             a menu as a tree: { label, items: [{ label, url, children: [...] }] }\n  ${META}/                   config, key, the last pull, the content model, screenshots\n\nA field taken out of a file is cleared on the website at the next push; a file you changed here is never overwritten by a pull.\nImages: an image field or a Columns image slot takes { "$media": { "url": "https://...", "alt": "..." } } (the push fetches it into the library) or { "$media": { "file": "assets/x.png", "alt": "..." } } for a picture that exists only as a file in this folder (the push carries its bytes).\nReferences: a record names another by its id, or "$ref:<slug-or-id>" for one in the same push.`);
 }
 
 async function main() {

package/lib/fetch.mjs

@@ -32,7 +32,9 @@ const text = (s) => decode(s.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim
 export function cleanHtml(html) {
 	return html
 		.replace(/<!--[\s\S]*?-->/g, '')
-		.replace(/<(script|style|noscript|template|iframe)\b[\s\S]*?<\/\1>/gi, '')
+		.replace(/<(script|style|noscript|template)\b[\s\S]*?<\/\1>/gi, '')
+		// A frame is a band of the page (a map, a report, a video): keep where it sits and what it shows, drop the rest of the tag.
+		.replace(/<iframe\b([^>]*)>[\s\S]*?<\/iframe>/gi, (m, attrs) => { const src = attrs.match(/\s(?:data-src|src)=["']([^"']+)["']/i)?.[1] ?? ''; const title = attrs.match(/\stitle=["']([^"']*)["']/i)?.[1] ?? ''; return `<iframe src="${src}"${title ? ` title="${title}"` : ''}></iframe>`; })
 		.replace(/<svg\b[\s\S]*?<\/svg>/gi, '<svg></svg>')
 		.replace(/\son[a-z]+="[^"]*"/gi, '')
 		.replace(/\sstyle="[^"]*"/gi, '')
@@ -51,7 +53,7 @@ function abs(base, href) {
 
 /** What the page is made of, in order. */
 export function outline(html, base) {
-	const out = { url: base, title: null, description: null, headings: [], nav: [], images: [], phones: [], emails: [], stylesheets: [], fonts: [], colors: [] };
+	const out = { url: base, title: null, description: null, headings: [], nav: [], images: [], videos: [], phones: [], emails: [], stylesheets: [], fonts: [], colors: [] };
 	const t = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
 	if (t) out.title = text(t[1]);
 	const d = html.match(/<meta[^>]+name=["']description["'][^>]*content=["']([^"']*)["']/i) || html.match(/<meta[^>]+content=["']([^"']*)["'][^>]*name=["']description["']/i);
@@ -93,6 +95,11 @@ export function outline(html, base) {
 		const height = tag.match(/\sheight=["']?(\d+)/i)?.[1];
 		out.images.push({ src: abs(base, src), alt, ...(width ? { width: Number(width) } : {}), ...(height ? { height: Number(height) } : {}) });
 	}
+	// A video on the page (a hero background, a <source> inside a <video>): a file to bring over, never to play from the old address.
+	for (const m of html.matchAll(/<(?:video|source)\b[^>]*\ssrc=["']([^"']+\.(?:mp4|webm|m4v|mov)(?:\?[^"']*)?)["'][^>]*>/gi)) {
+		const src = abs(base, m[1]);
+		if (!out.videos.some((v) => v.src === src)) out.videos.push({ src, poster: m[0].match(/\sposter=["']([^"']+)["']/i)?.[1] ? abs(base, m[0].match(/\sposter=["']([^"']+)["']/i)[1]) : null });
+	}
 	const body = text(html.replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ''));
 	out.phones = [...new Set([...body.matchAll(/(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/g)].map((m) => m[0].trim()))].slice(0, 10);
 	out.emails = [...new Set([...body.matchAll(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi)].map((m) => m[0]))].slice(0, 10);
@@ -107,6 +114,7 @@ export function outline(html, base) {
 	out.fonts = [...new Set(out.fonts)];
 	// Adobe Fonts (Typekit) kits belong to the old site or its builder, so a port cannot load them; the families are named from the kit's CSS by the fetch command.
 	out.adobe_kits = [...new Set([...html.matchAll(/use\.typekit\.net\/([a-z0-9]+)\.css/gi)].map((m) => m[1].toLowerCase()))];
+	out.frames = [...html.matchAll(/<iframe\b([^>]*)>/gi)].map((m) => ({ src: abs(base, m[1].match(/\s(?:data-src|src)=["']([^"']+)["']/i)?.[1] ?? ''), title: decode(m[1].match(/\stitle=["']([^"']*)["']/i)?.[1] ?? '') })).filter((f) => f.src && !/^about:/.test(f.src)).slice(0, 20);
 	out.adobe_families = [];
 	return out;
 }
@@ -162,6 +170,14 @@ export function outlineText(o) {
 		lines.push('', `Images (${o.images.length}):`);
 		for (const i of o.images.slice(0, 60)) lines.push(`  ${i.src}${i.width ? ` (${i.width}x${i.height ?? '?'})` : ''}${i.alt ? `  alt: ${i.alt}` : ''}`);
 	}
+	if (o.videos?.length) {
+		lines.push('', `Videos (${o.videos.length}): download each (curl -L -o assets/<name>.mp4 <src>), list it in the row's "media" with a name, and write <video src="{{ media.<name>.url }}" autoplay muted loop playsinline>; a video is never played from the old site's address`);
+		for (const v of o.videos) lines.push(`  ${v.src}${v.poster ? `  poster: ${v.poster}` : ''}`);
+	}
+	if (o.frames?.length) {
+		lines.push('', `Embedded frames (${o.frames.length}): a band of the page each; rebuild as the frame section or a row's <iframe>`);
+		for (const f of o.frames) lines.push(`  ${f.src}${f.title ? `  title: ${f.title}` : ''}`);
+	}
 	if (o.phones.length) lines.push('', `Phones: ${o.phones.join(', ')}`);
 	if (o.emails.length) lines.push(`Emails: ${o.emails.join(', ')}`);
 	if (o.fonts.length) lines.push(`Google fonts: ${o.fonts.join(', ')}`);

package/lib/folder.mjs

@@ -144,7 +144,7 @@ export function pushDocument(local, snapshot, { keys, removeKeys = [], settings
 		// A field taken out of the file is cleared on the site: the site keeps what a push does not name,
 		// so the names the snapshot had and the file no longer has go as null.
 		const data = { ...e.data };
-		if (was?.data) for (const k of Object.keys(was.data)) if (!(k in data) && was.data[k] !== null && was.data[k] !== '') data[k] = null;
+		if (was?.data) for (const k of Object.keys(was.data)) if (!(k in data) && !emptyValue(was.data[k])) data[k] = null;
 		(content[collection] ??= []).push({ id: e.id, slug, status: e.status, data });
 		if (was?.updated_at) expect[key] = was.updated_at;
 	}
@@ -163,6 +163,9 @@ export function pushDocument(local, snapshot, { keys, removeKeys = [], settings
 	return doc;
 }
 
+/** Nothing to clear: null, an empty string, an empty list or an empty object (the website refuses null for a list it stores as []). */
+const emptyValue = (v) => v === null || v === undefined || v === '' || (Array.isArray(v) && v.length === 0) || (typeof v === 'object' && !Array.isArray(v) && Object.keys(v).length === 0);
+
 export function splitKey(key) {
 	const i = key.indexOf('/');
 	return [key.slice(0, i), key.slice(i + 1)];

package/lib/validate.mjs

@@ -52,8 +52,12 @@ function checkScalar(type, value, at, errors) {
 		case 'image':
 		case 'media':
 		case 'file':
-			if (!isMedia(value)) errors.push(`${at}: an image is the stored value (from the pull or add_website_image) or { "$media": { "url": "https://...", "alt": "..." } }`);
-			else if (value.$media && (typeof value.$media.url !== 'string' || !/^https?:\/\//.test(value.$media.url))) errors.push(`${at}: $media.url must be an http(s) address`);
+			if (!isMedia(value)) errors.push(`${at}: an image is the stored value (from the pull or add_website_image) or { "$media": { "url": "https://..." | "file": "assets/x.png", "alt": "..." } }`);
+			else if (value.$media && typeof value.$media.file === 'string') {
+				const ok = field.type === 'file' ? /\.(png|jpe?g|gif|webp|avif|svg|ico|mp4|webm|m4v|mov|woff2?|ttf|otf|pdf)$/i : /\.(png|jpe?g|gif|webp|avif|svg|ico)$/i;
+				if (!ok.test(value.$media.file) || value.$media.file.startsWith('/') || value.$media.file.includes('..')) errors.push(`${at}: $media.file must be a file inside the folder (${field.type === 'file' ? 'a picture, a video (mp4, webm), a font (woff2) or a pdf' : 'a picture: png, jpg, gif, webp, avif, svg, ico'})`);
+			}
+			else if (value.$media && (typeof value.$media.url !== 'string' || !/^https?:\/\//.test(value.$media.url))) errors.push(`${at}: $media.url must be an http(s) address (or $media.file for a picture in the folder)`);
 			else if (value.$media && typeof value.$media.alt !== 'string') errors.push(`${at}: $media.alt is required (what the image shows; "" for a decorative one)`);
 			return;
 		case 'json':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {