@tideworthy/tidysites 0.1.19 → 0.1.21

package/README.md

@@ -63,7 +63,7 @@ shows in `status`.
 Styling: a row's `.css` file is scoped to that row (every rule is prefixed with the row's own selector), so it cannot style the rest of the page; rules for the whole page, the sections and the theme's classes go in `business/business.css`.
 
 Images: an image field, a hero's image or a Columns image slot takes
-`{ "$media": { "url": "https://...", "alt": "what it shows" } }` (or `{ "$media": { "file": "assets/x.png", "alt": "..." } }` for a picture that exists only as a file in the folder, say an illustration drawn from the old page's inline SVG; the push carries its bytes; a `file` field, such as a row's `media` list or the Business font files, takes a video (mp4, webm, up to 40 MB), a font (woff2) or a pdf the same way) and the
+`{ "$media": { "url": "https://...", "alt": "what it shows" } }` (or `{ "$media": { "file": "assets/x.png", "alt": "..." } }` for a picture that exists only as a file in the folder, say an illustration drawn from the old page's inline SVG; the push carries its bytes; a `file` field, such as a row's `media` list or the Business font files, takes a video (mp4, webm, up to 25 MB, the most one file of a published site may weigh), a font (woff2) or a pdf the same way) and the
 push fetches the file into the website's library; a file the library already
 holds (the same bytes) is reused, not added again. An image already in the
 library keeps the value the pull gave it.

package/bin/tidysites.mjs

@@ -26,8 +26,8 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { dirname, join, relative, resolve } from 'node:path';
 import { callTool, ApiError } from '../lib/api.mjs';
-import { diffEntries, entryToFiles, filesToEntry, FILE_NAME, menuItems, menuTree, pushDocument, sameEntry, snapshotOf, SIDECARS } from '../lib/folder.mjs';
-import { validateAll, indexSchema } from '../lib/validate.mjs';
+import { diffEntries, entryToFiles, filesToEntry, FILE_NAME, menuItems, menuTree, pushDocument, sameEntry, snapshotOf, SIDECARS, splitDocument, documentWeight, mergePushResults } from '../lib/folder.mjs';
+import { validateAll, indexSchema, validateMenuNames } from '../lib/validate.mjs';
 import { cleanHtml, cssSummary, fetchPage, outline, outlineText } from '../lib/fetch.mjs';
 
 const META = '.tidysites';
@@ -297,7 +297,7 @@ function validate({ dir }, local = null) {
 	const schema = readJson(join(dir, META, 'schema.json'));
 	if (!schema) fail('No content model here yet. Run `tidysites pull`.');
 	const read = local ? { local, problems: [] } : readLocal(dir);
-	const errors = [...read.problems, ...validateAll(read.local, schema)];
+	const errors = [...read.problems, ...validateAll(read.local, schema), ...validateMenuNames(Object.keys(readLocalMenus(dir)), Object.keys(readJson(join(dir, META, 'snapshot.json'))?.menus ?? {}))];
 	// The website keeps a section's type with its key: a key that changes type is refused with a bare "internal error", so say it here.
 	const snap = readJson(join(dir, META, 'snapshot.json'));
 	for (const [key, entry] of Object.entries(read.local)) {
@@ -341,7 +341,14 @@ async function push(ctx) {
 	if (keys.length || removeKeys.length) {
 		let r;
 		try {
-			({ json: r } = await callTool(ctx.cfg, 'push_website', { site_id: ctx.cfg.site, ...doc }));
+			// A push heavy with folder pictures goes in parts: the website takes each part well inside its time, where one request of tens of megabytes timed out part way.
+			const parts = splitDocument(doc);
+			if (parts.length > 1) say(`${parts.length} parts: the folder's pictures and videos weigh ${(documentWeight(doc) / 1048576).toFixed(0)} MB.`);
+			for (const [i, part] of parts.entries()) {
+				const { json } = await callTool(ctx.cfg, 'push_website', { site_id: ctx.cfg.site, ...part });
+				if (!json || typeof json !== 'object' || !json.content) throw new ApiError(`The website did not answer the push${parts.length > 1 ? ` (part ${i + 1} of ${parts.length})` : ''} in time or in a readable way`, { status: 504 });
+				r = r ? mergePushResults(r, json) : json;
+			}
 		} catch (e) {
 			if (!(e instanceof ApiError)) throw e;
 			say(describeError(e));
@@ -392,8 +399,8 @@ async function push(ctx) {
 }
 
 const MEDIA_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml', ico: 'image/x-icon', mp4: 'video/mp4', m4v: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', woff2: 'font/woff2', woff: 'font/woff', ttf: 'font/ttf', otf: 'font/otf', pdf: 'application/pdf' };
-/** Pictures travel up to 15 MB; a video (a row's media list) up to 40 MB. */
-const mediaCap = (type) => (type.startsWith('video/') ? 40 : 15);
+/** Pictures travel up to 15 MB; a video up to 25 MB, the most one file of a published site may weigh. */
+const mediaCap = (type) => (type.startsWith('video/') ? 25 : 15);
 /** Every { "$media": { "file": ... } } in the document becomes { "$media": { "data", "type", "filename", "alt" } }, read from the folder. */
 function inlineMediaFiles(doc, dir) {
 	const walk = (v) => {
@@ -407,7 +414,7 @@ function inlineMediaFiles(doc, dir) {
 				const bytes = readFileSync(path);
 				const ext = m.file.split('.').pop().toLowerCase();
 				const type = MEDIA_TYPES[ext] ?? 'application/octet-stream';
-				if (bytes.length > mediaCap(type) * 1024 * 1024) fail(`$media.file ${m.file}: ${(bytes.length / 1048576).toFixed(1)} MB; ${type.startsWith('video/') ? 'videos up to 40 MB' : 'pictures up to 15 MB'}`);
+				if (bytes.length > mediaCap(type) * 1024 * 1024) fail(`$media.file ${m.file}: ${(bytes.length / 1048576).toFixed(1)} MB; ${type.startsWith('video/') ? 'videos up to 25 MB (the most one file of a published site may weigh; re-encode it smaller)' : 'pictures up to 15 MB'}`);
 				return { $media: { data: bytes.toString('base64'), type, filename: m.file.split('/').pop(), alt: typeof m.alt === 'string' ? m.alt : '' } };
 			}
 			return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));

package/lib/folder.mjs

@@ -232,3 +232,35 @@ export function menuItems(tree) {
 	walk(tree, null);
 	return out;
 }
+
+/** How many bytes of inlined pictures and video one push carries at most; a heavier document goes in parts. */
+export const PUSH_PART_BYTES = 12 * 1024 * 1024;
+/** The inlined media weight of a document, in bytes of base64 text. */
+export function documentWeight(doc) {
+	let n = 0;
+	const walk = (v) => { if (Array.isArray(v)) v.forEach(walk); else if (v && typeof v === 'object') { if (v.$media && typeof v.$media.data === 'string') n += v.$media.data.length; else Object.values(v).forEach(walk); } };
+	walk(doc.content ?? {});
+	return n;
+}
+/** The document as a list of pushes, each under the weight cap (an entry heavier than the cap travels alone); settings, removals and the expectations ride with the last part. */
+export function splitDocument(doc) {
+	if (!doc.content || documentWeight(doc) <= PUSH_PART_BYTES) return [doc];
+	const parts = [];
+	let current = { content: {} }; let weight = 0;
+	for (const [collection, entries] of Object.entries(doc.content)) {
+		for (const e of entries) {
+			const w = documentWeight({ content: { x: [e] } });
+			if (weight > 0 && weight + w > PUSH_PART_BYTES) { parts.push(current); current = { content: {} }; weight = 0; }
+			(current.content[collection] ??= []).push(e); weight += w;
+		}
+	}
+	parts.push(current);
+	const last = parts[parts.length - 1];
+	for (const k of ['settings', 'remove', 'expect']) if (doc[k] !== undefined) last[k] = doc[k];
+	// Every part expects the same website state it was read against.
+	if (doc.expect) for (const p of parts) p.expect = Object.fromEntries(Object.entries(doc.expect).filter(([k]) => Object.entries(p.content).some(([c, es]) => es.some((e) => `${c}/${e.slug}` === k))));
+	return parts;
+}
+export function mergePushResults(a, b) {
+	return { ...a, ...b, content: { created: (a.content?.created ?? 0) + (b.content?.created ?? 0), updated: (a.content?.updated ?? 0) + (b.content?.updated ?? 0) }, images: (a.images ?? 0) + (b.images ?? 0), images_reused: (a.images_reused ?? 0) + (b.images_reused ?? 0), missing: [...(a.missing ?? []), ...(b.missing ?? [])] };
+}

package/lib/validate.mjs

@@ -188,6 +188,11 @@ export function validateEntry(collection, entry, model, file = `${collection}/${
 }
 
 /** Every entry (by "collection/slug") checked; the errors in file order. */
+/** A menu the website lacks is created at push time under its file name: lowercase letters, digits and underscores. */
+export function validateMenuNames(menuNames, knownNames = []) {
+	return menuNames.filter((n) => !knownNames.includes(n) && !/^[a-z][a-z0-9_]*$/.test(n)).map((n) => `menus/${n}.json: a new menu's name is lowercase letters, digits and underscores (rename the file)`);
+}
+
 export function validateAll(local, schema) {
 	const model = indexSchema(schema);
 	const errors = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.19",
+  "version": "0.1.21",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {