@tideworthy/tidysites 0.1.19 → 0.1.20

package/bin/tidysites.mjs

@@ -26,8 +26,8 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { dirname, join, relative, resolve } from 'node:path';
 import { callTool, ApiError } from '../lib/api.mjs';
-import { diffEntries, entryToFiles, filesToEntry, FILE_NAME, menuItems, menuTree, pushDocument, sameEntry, snapshotOf, SIDECARS } from '../lib/folder.mjs';
-import { validateAll, indexSchema } from '../lib/validate.mjs';
+import { diffEntries, entryToFiles, filesToEntry, FILE_NAME, menuItems, menuTree, pushDocument, sameEntry, snapshotOf, SIDECARS, splitDocument, documentWeight, mergePushResults } from '../lib/folder.mjs';
+import { validateAll, indexSchema, validateMenuNames } from '../lib/validate.mjs';
 import { cleanHtml, cssSummary, fetchPage, outline, outlineText } from '../lib/fetch.mjs';
 
 const META = '.tidysites';
@@ -297,7 +297,7 @@ function validate({ dir }, local = null) {
 	const schema = readJson(join(dir, META, 'schema.json'));
 	if (!schema) fail('No content model here yet. Run `tidysites pull`.');
 	const read = local ? { local, problems: [] } : readLocal(dir);
-	const errors = [...read.problems, ...validateAll(read.local, schema)];
+	const errors = [...read.problems, ...validateAll(read.local, schema), ...validateMenuNames(Object.keys(readLocalMenus(dir)), Object.keys(readJson(join(dir, META, 'snapshot.json'))?.menus ?? {}))];
 	// The website keeps a section's type with its key: a key that changes type is refused with a bare "internal error", so say it here.
 	const snap = readJson(join(dir, META, 'snapshot.json'));
 	for (const [key, entry] of Object.entries(read.local)) {
@@ -341,7 +341,14 @@ async function push(ctx) {
 	if (keys.length || removeKeys.length) {
 		let r;
 		try {
-			({ json: r } = await callTool(ctx.cfg, 'push_website', { site_id: ctx.cfg.site, ...doc }));
+			// A push heavy with folder pictures goes in parts: the website takes each part well inside its time, where one request of tens of megabytes timed out part way.
+			const parts = splitDocument(doc);
+			if (parts.length > 1) say(`${parts.length} parts: the folder's pictures and videos weigh ${(documentWeight(doc) / 1048576).toFixed(0)} MB.`);
+			for (const [i, part] of parts.entries()) {
+				const { json } = await callTool(ctx.cfg, 'push_website', { site_id: ctx.cfg.site, ...part });
+				if (!json || typeof json !== 'object' || !json.content) throw new ApiError(`The website did not answer the push${parts.length > 1 ? ` (part ${i + 1} of ${parts.length})` : ''} in time or in a readable way`, { status: 504 });
+				r = r ? mergePushResults(r, json) : json;
+			}
 		} catch (e) {
 			if (!(e instanceof ApiError)) throw e;
 			say(describeError(e));

package/lib/folder.mjs

@@ -232,3 +232,35 @@ export function menuItems(tree) {
 	walk(tree, null);
 	return out;
 }
+
+/** How many bytes of inlined pictures and video one push carries at most; a heavier document goes in parts. */
+export const PUSH_PART_BYTES = 12 * 1024 * 1024;
+/** The inlined media weight of a document, in bytes of base64 text. */
+export function documentWeight(doc) {
+	let n = 0;
+	const walk = (v) => { if (Array.isArray(v)) v.forEach(walk); else if (v && typeof v === 'object') { if (v.$media && typeof v.$media.data === 'string') n += v.$media.data.length; else Object.values(v).forEach(walk); } };
+	walk(doc.content ?? {});
+	return n;
+}
+/** The document as a list of pushes, each under the weight cap (an entry heavier than the cap travels alone); settings, removals and the expectations ride with the last part. */
+export function splitDocument(doc) {
+	if (!doc.content || documentWeight(doc) <= PUSH_PART_BYTES) return [doc];
+	const parts = [];
+	let current = { content: {} }; let weight = 0;
+	for (const [collection, entries] of Object.entries(doc.content)) {
+		for (const e of entries) {
+			const w = documentWeight({ content: { x: [e] } });
+			if (weight > 0 && weight + w > PUSH_PART_BYTES) { parts.push(current); current = { content: {} }; weight = 0; }
+			(current.content[collection] ??= []).push(e); weight += w;
+		}
+	}
+	parts.push(current);
+	const last = parts[parts.length - 1];
+	for (const k of ['settings', 'remove', 'expect']) if (doc[k] !== undefined) last[k] = doc[k];
+	// Every part expects the same website state it was read against.
+	if (doc.expect) for (const p of parts) p.expect = Object.fromEntries(Object.entries(doc.expect).filter(([k]) => Object.entries(p.content).some(([c, es]) => es.some((e) => `${c}/${e.slug}` === k))));
+	return parts;
+}
+export function mergePushResults(a, b) {
+	return { ...a, ...b, content: { created: (a.content?.created ?? 0) + (b.content?.created ?? 0), updated: (a.content?.updated ?? 0) + (b.content?.updated ?? 0) }, images: (a.images ?? 0) + (b.images ?? 0), images_reused: (a.images_reused ?? 0) + (b.images_reused ?? 0), missing: [...(a.missing ?? []), ...(b.missing ?? [])] };
+}

package/lib/validate.mjs

@@ -188,6 +188,11 @@ export function validateEntry(collection, entry, model, file = `${collection}/${
 }
 
 /** Every entry (by "collection/slug") checked; the errors in file order. */
+/** A menu the website lacks is created at push time under its file name: lowercase letters, digits and underscores. */
+export function validateMenuNames(menuNames, knownNames = []) {
+	return menuNames.filter((n) => !knownNames.includes(n) && !/^[a-z][a-z0-9_]*$/.test(n)).map((n) => `menus/${n}.json: a new menu's name is lowercase letters, digits and underscores (rename the file)`);
+}
+
 export function validateAll(local, schema) {
 	const model = indexSchema(schema);
 	const errors = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.19",
+  "version": "0.1.20",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {