@tideworthy/tidysites 0.1.12 → 0.1.14

package/README.md

@@ -30,7 +30,7 @@ nothing else; it lives in `.tidysites/key`, which is kept out of git.
 | `tidysites page <path> [--html]` | the rendered page, section by section, with the key each section carries |
 | `tidysites schema [type or collection]` | the section types and layouts, or one type's or collection's fields |
 | `tidysites gallery [type]` | a screenshot of every section type and layout in this website's theme, or the first screen of one type |
-| `tidysites fetch <url> [--out dir]` | a page of the site being ported: cleaned HTML, its outline (headings, navigation, images, phones, fonts, the stylesheet's own variables and colours) and its stylesheets, inline style blocks included |
+| `tidysites fetch <url> [--out dir]` | a page of the site being ported: cleaned HTML, its outline (headings, navigation, images, phones, fonts, an Adobe Fonts kit and the families it serves, the stylesheet's own variables and colours) and its stylesheets, inline style blocks included |
 | `tidysites rm <collection>/<slug> ...` | delete entry files with their sidecars (`services/*` for every entry of a collection); `push --prune` then removes them from the website |
 | `tidysites publish --yes` | the live site becomes a copy of the preview site. A website that is live on the client's own domain is never published from the command line: the website refuses and says so; the person publishes it (Publish now on the Website tab), or an assistant with them does through the connector after they confirm |
 
@@ -63,7 +63,7 @@ shows in `status`.
 Styling: a row's `.css` file is scoped to that row (every rule is prefixed with the row's own selector), so it cannot style the rest of the page; rules for the whole page, the sections and the theme's classes go in `business/business.css`.
 
 Images: an image field, a hero's image or a Columns image slot takes
-`{ "$media": { "url": "https://...", "alt": "what it shows" } }` and the
+`{ "$media": { "url": "https://...", "alt": "what it shows" } }` (or `{ "$media": { "file": "assets/x.png", "alt": "..." } }` for a picture that exists only as a file in the folder, say an illustration drawn from the old page's inline SVG; the push carries its bytes) and the
 push fetches the file into the website's library; a file the library already
 holds (the same bytes) is reused, not added again. An image already in the
 library keeps the value the pull gave it.

package/bin/tidysites.mjs

@@ -322,6 +322,9 @@ async function push(ctx) {
 	}
 	const doc = pushDocument(s.local, s.snap, { keys, removeKeys });
 	if (flags.force) doc.force = true;
+	// A picture that exists only as a file here (an illustration drawn from the old page's inline SVG, a rasterised
+	// icon) goes up with the push as its bytes; the website stores it in the library like a fetched one.
+	inlineMediaFiles(doc, ctx.dir);
 	if (flags.dry) {
 		say(`Would push ${keys.length} entr${keys.length === 1 ? 'y' : 'ies'}${removeKeys.length ? `, remove ${removeKeys.length}` : ''}${menus.length ? `, rewrite menu${menus.length === 1 ? '' : 's'} ${menus.join(', ')}` : ''}:`);
 		printStatus(s);
@@ -366,6 +369,29 @@ async function push(ctx) {
 	else if (menus.length) await pullMenus(ctx);
 }
 
+const MEDIA_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml', ico: 'image/x-icon' };
+/** Every { "$media": { "file": ... } } in the document becomes { "$media": { "data", "type", "filename", "alt" } }, read from the folder. */
+function inlineMediaFiles(doc, dir) {
+	const walk = (v) => {
+		if (Array.isArray(v)) return v.map(walk);
+		if (v && typeof v === 'object') {
+			const m = v.$media;
+			if (m && typeof m === 'object' && typeof m.file === 'string') {
+				const path = resolve(dir, m.file);
+				if (!path.startsWith(resolve(dir))) fail(`$media.file ${m.file}: outside the folder`);
+				if (!existsSync(path)) fail(`$media.file ${m.file}: no such file`);
+				const bytes = readFileSync(path);
+				if (bytes.length > 15 * 1024 * 1024) fail(`$media.file ${m.file}: ${(bytes.length / 1048576).toFixed(1)} MB; pictures up to 15 MB`);
+				const ext = m.file.split('.').pop().toLowerCase();
+				return { $media: { data: bytes.toString('base64'), type: MEDIA_TYPES[ext] ?? 'application/octet-stream', filename: m.file.split('/').pop(), alt: typeof m.alt === 'string' ? m.alt : '' } };
+			}
+			return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
+		}
+		return v;
+	};
+	if (doc.content) doc.content = walk(doc.content);
+}
+
 async function shot(ctx, target = positional[0] ?? '/', opts = {}) {
 	const isUrl = /^https?:\/\//.test(target);
 	// --desktop, --mobile, --tablet, --wide, any of them together; --all for every width; none = desktop and mobile.
@@ -569,6 +595,8 @@ async function fetchCmd(ctx) {
 				const css = await r.text();
 				cssTexts.push(css);
 				writeFileSync(join(outDir, `${base}-${cssTexts.length}.css`), css);
+				// The kit's CSS names the families it serves, which the page's HTML does not.
+				if (/use\.typekit\.net/i.test(href)) o.adobe_families = [...new Set([...o.adobe_families ?? [], ...[...css.matchAll(/font-family\s*:\s*["']([^"';]+)["']/gi)].map((m) => m[1])])];
 			}
 		} catch {
 			/* a stylesheet that will not download is not the port's problem */
@@ -631,7 +659,7 @@ async function publish(ctx) {
 
 function help() {
 	say(readFileSync(new URL(import.meta.url), 'utf8').split('\n').slice(2, 20).map((l) => l.replace(/^ \* ?/, '')).join('\n'));
-	say(`\nFiles:\n  <collection>/<slug>.json      { "id", "slug", "status", "updated_at", "data": {...} }; a new entry is a new file with "data"\n  <collection>/<slug>.css       data.custom_css as a file (rows: data.css); rows/<slug>.html is data.html\n  menus/<name>.json             a menu as a tree: { label, items: [{ label, url, children: [...] }] }\n  ${META}/                   config, key, the last pull, the content model, screenshots\n\nA field taken out of a file is cleared on the website at the next push; a file you changed here is never overwritten by a pull.\nImages: an image field or a Columns image slot takes { "$media": { "url": "https://...", "alt": "..." } }; the push fetches it into the library.\nReferences: a record names another by its id, or "$ref:<slug-or-id>" for one in the same push.`);
+	say(`\nFiles:\n  <collection>/<slug>.json      { "id", "slug", "status", "updated_at", "data": {...} }; a new entry is a new file with "data"\n  <collection>/<slug>.css       data.custom_css as a file (rows: data.css); rows/<slug>.html is data.html\n  menus/<name>.json             a menu as a tree: { label, items: [{ label, url, children: [...] }] }\n  ${META}/                   config, key, the last pull, the content model, screenshots\n\nA field taken out of a file is cleared on the website at the next push; a file you changed here is never overwritten by a pull.\nImages: an image field or a Columns image slot takes { "$media": { "url": "https://...", "alt": "..." } } (the push fetches it into the library) or { "$media": { "file": "assets/x.png", "alt": "..." } } for a picture that exists only as a file in this folder (the push carries its bytes).\nReferences: a record names another by its id, or "$ref:<slug-or-id>" for one in the same push.`);
 }
 
 async function main() {

package/lib/fetch.mjs

@@ -32,7 +32,9 @@ const text = (s) => decode(s.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim
 export function cleanHtml(html) {
 	return html
 		.replace(/<!--[\s\S]*?-->/g, '')
-		.replace(/<(script|style|noscript|template|iframe)\b[\s\S]*?<\/\1>/gi, '')
+		.replace(/<(script|style|noscript|template)\b[\s\S]*?<\/\1>/gi, '')
+		// A frame is a band of the page (a map, a report, a video): keep where it sits and what it shows, drop the rest of the tag.
+		.replace(/<iframe\b([^>]*)>[\s\S]*?<\/iframe>/gi, (m, attrs) => { const src = attrs.match(/\s(?:data-src|src)=["']([^"']+)["']/i)?.[1] ?? ''; const title = attrs.match(/\stitle=["']([^"']*)["']/i)?.[1] ?? ''; return `<iframe src="${src}"${title ? ` title="${title}"` : ''}></iframe>`; })
 		.replace(/<svg\b[\s\S]*?<\/svg>/gi, '<svg></svg>')
 		.replace(/\son[a-z]+="[^"]*"/gi, '')
 		.replace(/\sstyle="[^"]*"/gi, '')
@@ -105,6 +107,10 @@ export function outline(html, base) {
 		out.fonts.push(...fams);
 	}
 	out.fonts = [...new Set(out.fonts)];
+	// Adobe Fonts (Typekit) kits belong to the old site or its builder, so a port cannot load them; the families are named from the kit's CSS by the fetch command.
+	out.adobe_kits = [...new Set([...html.matchAll(/use\.typekit\.net\/([a-z0-9]+)\.css/gi)].map((m) => m[1].toLowerCase()))];
+	out.frames = [...html.matchAll(/<iframe\b([^>]*)>/gi)].map((m) => ({ src: abs(base, m[1].match(/\s(?:data-src|src)=["']([^"']+)["']/i)?.[1] ?? ''), title: decode(m[1].match(/\stitle=["']([^"']*)["']/i)?.[1] ?? '') })).filter((f) => f.src && !/^about:/.test(f.src)).slice(0, 20);
+	out.adobe_families = [];
 	return out;
 }
 
@@ -159,9 +165,17 @@ export function outlineText(o) {
 		lines.push('', `Images (${o.images.length}):`);
 		for (const i of o.images.slice(0, 60)) lines.push(`  ${i.src}${i.width ? ` (${i.width}x${i.height ?? '?'})` : ''}${i.alt ? `  alt: ${i.alt}` : ''}`);
 	}
+	if (o.frames?.length) {
+		lines.push('', `Embedded frames (${o.frames.length}): a band of the page each; rebuild as the frame section or a row's <iframe>`);
+		for (const f of o.frames) lines.push(`  ${f.src}${f.title ? `  title: ${f.title}` : ''}`);
+	}
 	if (o.phones.length) lines.push('', `Phones: ${o.phones.join(', ')}`);
 	if (o.emails.length) lines.push(`Emails: ${o.emails.join(', ')}`);
 	if (o.fonts.length) lines.push(`Google fonts: ${o.fonts.join(', ')}`);
+	if (o.adobe_kits?.length) {
+		lines.push(`Adobe Fonts kit: ${o.adobe_kits.join(', ')} (the old site's own kit; this site cannot load it)${o.adobe_families?.length ? ` serving ${o.adobe_families.join(', ')}` : ''}.`);
+		lines.push('  Each family from it needs a Google Fonts stand-in on the Business record that matches its shape and weights as closely as possible, and the report must name the original and the stand-in. A font is never left unset.');
+	}
 	if (o.css) {
 		const vars = Object.entries(o.css.variables ?? {});
 		if (vars.length) { lines.push('', 'Stylesheet variables (the site\'s own names; trust these over the counts below):'); for (const [k, v] of vars.slice(0, 40)) lines.push(`  --${k}: ${v}`); }

package/lib/folder.mjs

@@ -144,7 +144,7 @@ export function pushDocument(local, snapshot, { keys, removeKeys = [], settings
 		// A field taken out of the file is cleared on the site: the site keeps what a push does not name,
 		// so the names the snapshot had and the file no longer has go as null.
 		const data = { ...e.data };
-		if (was?.data) for (const k of Object.keys(was.data)) if (!(k in data) && was.data[k] !== null && was.data[k] !== '') data[k] = null;
+		if (was?.data) for (const k of Object.keys(was.data)) if (!(k in data) && !emptyValue(was.data[k])) data[k] = null;
 		(content[collection] ??= []).push({ id: e.id, slug, status: e.status, data });
 		if (was?.updated_at) expect[key] = was.updated_at;
 	}
@@ -163,6 +163,9 @@ export function pushDocument(local, snapshot, { keys, removeKeys = [], settings
 	return doc;
 }
 
+/** Nothing to clear: null, an empty string, an empty list or an empty object (the website refuses null for a list it stores as []). */
+const emptyValue = (v) => v === null || v === undefined || v === '' || (Array.isArray(v) && v.length === 0) || (typeof v === 'object' && !Array.isArray(v) && Object.keys(v).length === 0);
+
 export function splitKey(key) {
 	const i = key.indexOf('/');
 	return [key.slice(0, i), key.slice(i + 1)];

package/lib/validate.mjs

@@ -52,8 +52,9 @@ function checkScalar(type, value, at, errors) {
 		case 'image':
 		case 'media':
 		case 'file':
-			if (!isMedia(value)) errors.push(`${at}: an image is the stored value (from the pull or add_website_image) or { "$media": { "url": "https://...", "alt": "..." } }`);
-			else if (value.$media && (typeof value.$media.url !== 'string' || !/^https?:\/\//.test(value.$media.url))) errors.push(`${at}: $media.url must be an http(s) address`);
+			if (!isMedia(value)) errors.push(`${at}: an image is the stored value (from the pull or add_website_image) or { "$media": { "url": "https://..." | "file": "assets/x.png", "alt": "..." } }`);
+			else if (value.$media && typeof value.$media.file === 'string') { if (!/\.(png|jpe?g|gif|webp|avif|svg|ico)$/i.test(value.$media.file) || value.$media.file.startsWith('/') || value.$media.file.includes('..')) errors.push(`${at}: $media.file must be a picture file inside the folder (png, jpg, gif, webp, avif, svg, ico)`); }
+			else if (value.$media && (typeof value.$media.url !== 'string' || !/^https?:\/\//.test(value.$media.url))) errors.push(`${at}: $media.url must be an http(s) address (or $media.file for a picture in the folder)`);
 			else if (value.$media && typeof value.$media.alt !== 'string') errors.push(`${at}: $media.alt is required (what the image shows; "" for a decorative one)`);
 			return;
 		case 'json':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.12",
+  "version": "0.1.14",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {