@tideworthy/tidysites 0.1.18 → 0.1.20

package/bin/tidysites.mjs

@@ -26,8 +26,8 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { dirname, join, relative, resolve } from 'node:path';
 import { callTool, ApiError } from '../lib/api.mjs';
-import { diffEntries, entryToFiles, filesToEntry, FILE_NAME, menuItems, menuTree, pushDocument, sameEntry, snapshotOf, SIDECARS } from '../lib/folder.mjs';
-import { validateAll, indexSchema } from '../lib/validate.mjs';
+import { diffEntries, entryToFiles, filesToEntry, FILE_NAME, menuItems, menuTree, pushDocument, sameEntry, snapshotOf, SIDECARS, splitDocument, documentWeight, mergePushResults } from '../lib/folder.mjs';
+import { validateAll, indexSchema, validateMenuNames } from '../lib/validate.mjs';
 import { cleanHtml, cssSummary, fetchPage, outline, outlineText } from '../lib/fetch.mjs';
 
 const META = '.tidysites';
@@ -133,11 +133,12 @@ function describeError(e) {
 	return msg;
 }
 
-async function tool(cfg, name, args) {
+async function tool(cfg, name, args, { throws = false } = {}) {
 	try {
 		return await callTool(cfg, name, args);
 	} catch (e) {
-		if (e instanceof ApiError) fail(describeError(e));
+		// A caller that can recover (a menu to create first) takes the error; every other call ends the command with it.
+		if (e instanceof ApiError && !throws) fail(describeError(e));
 		throw e;
 	}
 }
@@ -296,7 +297,7 @@ function validate({ dir }, local = null) {
 	const schema = readJson(join(dir, META, 'schema.json'));
 	if (!schema) fail('No content model here yet. Run `tidysites pull`.');
 	const read = local ? { local, problems: [] } : readLocal(dir);
-	const errors = [...read.problems, ...validateAll(read.local, schema)];
+	const errors = [...read.problems, ...validateAll(read.local, schema), ...validateMenuNames(Object.keys(readLocalMenus(dir)), Object.keys(readJson(join(dir, META, 'snapshot.json'))?.menus ?? {}))];
 	// The website keeps a section's type with its key: a key that changes type is refused with a bare "internal error", so say it here.
 	const snap = readJson(join(dir, META, 'snapshot.json'));
 	for (const [key, entry] of Object.entries(read.local)) {
@@ -340,7 +341,14 @@ async function push(ctx) {
 	if (keys.length || removeKeys.length) {
 		let r;
 		try {
-			({ json: r } = await callTool(ctx.cfg, 'push_website', { site_id: ctx.cfg.site, ...doc }));
+			// A push heavy with folder pictures goes in parts: the website takes each part well inside its time, where one request of tens of megabytes timed out part way.
+			const parts = splitDocument(doc);
+			if (parts.length > 1) say(`${parts.length} parts: the folder's pictures and videos weigh ${(documentWeight(doc) / 1048576).toFixed(0)} MB.`);
+			for (const [i, part] of parts.entries()) {
+				const { json } = await callTool(ctx.cfg, 'push_website', { site_id: ctx.cfg.site, ...part });
+				if (!json || typeof json !== 'object' || !json.content) throw new ApiError(`The website did not answer the push${parts.length > 1 ? ` (part ${i + 1} of ${parts.length})` : ''} in time or in a readable way`, { status: 504 });
+				r = r ? mergePushResults(r, json) : json;
+			}
 		} catch (e) {
 			if (!(e instanceof ApiError)) throw e;
 			say(describeError(e));
@@ -367,24 +375,27 @@ async function push(ctx) {
 		}
 		try {
 			try {
-				await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_set_items', arguments: { name, items } });
+				await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_set_items', arguments: { name, items } }, { throws: true });
 			} catch (e) {
 				// A menu the website does not have yet (a phone drawer, a footer column) is created from the file, then filled.
 				if (!/not.?found/i.test(e.message ?? '')) throw e;
-				await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_create', arguments: { name, label: m.label || name } });
-				await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_set_items', arguments: { name, items } });
+				if (!/^[a-z][a-z0-9_]*$/.test(name)) throw new Error(`a new menu's name is lowercase letters, digits and underscores (${name} has something else)`);
+				await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_create', arguments: { name, label: m.label || name } }, { throws: true });
+				await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_set_items', arguments: { name, items } }, { throws: true });
 				say(`Menu ${name}: created.`);
 			}
 			say(`Menu ${name}: ${items.length} item${items.length === 1 ? '' : 's'} written.`);
 		} catch (e) {
-			menuProblems.push(`menus/${name}.json: ${e.message}`);
+			menuProblems.push(`menus/${name}.json: ${e instanceof ApiError ? describeError(e) : e.message}`);
 		}
 	}
+	// Refresh what went (the entries, and the menus that were written) before any menu failure is reported, so the snapshot
+	// knows the website's new versions and the next push is not refused as "changed on the website".
+	const written = menus.filter((n) => !menuProblems.some((p) => p.startsWith(`menus/${n}.json`)));
+	const pushed = new Set([...keys, ...written.map((m) => `menus/${m}`)]);
+	if (touched.length) await pull(ctx, touched, { overwrite: pushed, menus: written.length > 0 });
+	else if (written.length) await pullMenus(ctx);
 	if (menuProblems.length) fail(menuProblems.join('\n'));
-	// Refresh only what went, and take the website's version of exactly those entries; every other file stays as it is.
-	const pushed = new Set([...keys, ...menus.map((m) => `menus/${m}`)]);
-	if (touched.length) await pull(ctx, touched, { overwrite: pushed, menus: menus.length > 0 });
-	else if (menus.length) await pullMenus(ctx);
 }
 
 const MEDIA_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml', ico: 'image/x-icon', mp4: 'video/mp4', m4v: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', woff2: 'font/woff2', woff: 'font/woff', ttf: 'font/ttf', otf: 'font/otf', pdf: 'application/pdf' };

package/lib/folder.mjs

@@ -232,3 +232,35 @@ export function menuItems(tree) {
 	walk(tree, null);
 	return out;
 }
+
+/** How many bytes of inlined pictures and video one push carries at most; a heavier document goes in parts. */
+export const PUSH_PART_BYTES = 12 * 1024 * 1024;
+/** The inlined media weight of a document, in bytes of base64 text. */
+export function documentWeight(doc) {
+	let n = 0;
+	const walk = (v) => { if (Array.isArray(v)) v.forEach(walk); else if (v && typeof v === 'object') { if (v.$media && typeof v.$media.data === 'string') n += v.$media.data.length; else Object.values(v).forEach(walk); } };
+	walk(doc.content ?? {});
+	return n;
+}
+/** The document as a list of pushes, each under the weight cap (an entry heavier than the cap travels alone); settings, removals and the expectations ride with the last part. */
+export function splitDocument(doc) {
+	if (!doc.content || documentWeight(doc) <= PUSH_PART_BYTES) return [doc];
+	const parts = [];
+	let current = { content: {} }; let weight = 0;
+	for (const [collection, entries] of Object.entries(doc.content)) {
+		for (const e of entries) {
+			const w = documentWeight({ content: { x: [e] } });
+			if (weight > 0 && weight + w > PUSH_PART_BYTES) { parts.push(current); current = { content: {} }; weight = 0; }
+			(current.content[collection] ??= []).push(e); weight += w;
+		}
+	}
+	parts.push(current);
+	const last = parts[parts.length - 1];
+	for (const k of ['settings', 'remove', 'expect']) if (doc[k] !== undefined) last[k] = doc[k];
+	// Every part expects the same website state it was read against.
+	if (doc.expect) for (const p of parts) p.expect = Object.fromEntries(Object.entries(doc.expect).filter(([k]) => Object.entries(p.content).some(([c, es]) => es.some((e) => `${c}/${e.slug}` === k))));
+	return parts;
+}
+export function mergePushResults(a, b) {
+	return { ...a, ...b, content: { created: (a.content?.created ?? 0) + (b.content?.created ?? 0), updated: (a.content?.updated ?? 0) + (b.content?.updated ?? 0) }, images: (a.images ?? 0) + (b.images ?? 0), images_reused: (a.images_reused ?? 0) + (b.images_reused ?? 0), missing: [...(a.missing ?? []), ...(b.missing ?? [])] };
+}

package/lib/validate.mjs

@@ -188,6 +188,11 @@ export function validateEntry(collection, entry, model, file = `${collection}/${
 }
 
 /** Every entry (by "collection/slug") checked; the errors in file order. */
+/** A menu the website lacks is created at push time under its file name: lowercase letters, digits and underscores. */
+export function validateMenuNames(menuNames, knownNames = []) {
+	return menuNames.filter((n) => !knownNames.includes(n) && !/^[a-z][a-z0-9_]*$/.test(n)).map((n) => `menus/${n}.json: a new menu's name is lowercase letters, digits and underscores (rename the file)`);
+}
+
 export function validateAll(local, schema) {
 	const model = indexSchema(schema);
 	const errors = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.18",
+  "version": "0.1.20",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {