@tideworthy/tidysites 0.1.0 → 0.1.2

package/README.md

@@ -18,7 +18,7 @@ nothing else; it lives in `.tidysites/key`, which is kept out of git.
 | | |
 |---|---|
 | `tidysites init <site_id> --host <url> --key <key>` | connect this folder to a website and pull it |
-| `tidysites pull [--collections pages,services]` | the website's content to files; files for entries the website no longer has are removed |
+| `tidysites pull [--collections pages,services,menus]` | the website's content to files; files for entries the website no longer has are removed |
 | `tidysites status` | what changed here since the pull |
 | `tidysites validate` | every file against the content model, with the file and the field in each message |
 | `tidysites push [--prune] [--force] [--dry] [--only pages]` | validate, then write the created and updated entries back (with `--prune`, delete the entries whose files are gone), then pull |
@@ -30,6 +30,10 @@ nothing else; it lives in `.tidysites/key`, which is kept out of git.
 | `tidysites fetch <url> [--out dir]` | a page of the site being ported: cleaned HTML, its outline (headings, navigation, images, phones, fonts, colours) and its stylesheets |
 | `tidysites publish --yes` | the live site becomes a copy of the preview site |
 
+A section keeps its type with its key: to replace an intro with a row, give the row its own `_key`
+(`validate` says so; the website would only say "internal error"). After a push the pushed files take the
+website's version (a `$media` becomes the stored image, `_version` appears); every other file stays as it is.
+
 A push that would overwrite an entry changed on the website since the pull
 is refused with the list; pull again (your files are overwritten, so copy
 what you need first) or push `--force`.
@@ -62,6 +66,15 @@ References: a record names another by its id (what the pull shows), or by
 
 Rich text (a section's `copy`, a post's `content`) is markdown.
 
+Custom HTML (a row's `.html` file, a Columns HTML slot) reads the records:
+`{{ business.phone }}`, `{{ page.title }}`, `{{#each services as s}}…{{ s.url }}…{{/each}}`,
+`{{#if page.collection == "services"}}…{{/if}}`, and the drop-ins `{{> menu "primary"}}`,
+`{{> logo}}`, `{{> phone}}`, `{{> address}}`, `{{> hours}}`, `{{> social}}`, `{{> year}}`,
+`{{> button "Label" "/url" style=outline}}`, `{{> form "Name"}}`. A link to `#form:<Name>`
+opens that form in a dialog. So a custom header is written once against `page.*` and
+`business.*` and fits every page; `frame/frame.json` names it as `header_row`, or a page
+names its own in `header_row`.
+
 ## Building a site from these files
 
 1. `tidysites schema` for the section types, `tidysites gallery` to see them in this theme.

package/bin/tidysites.mjs

@@ -165,8 +165,15 @@ function readLocalMenus(dir) {
 	return out;
 }
 
-async function pull({ dir, cfg }, collections = null) {
-	const want = collections ?? (flags.collections ? String(flags.collections).split(',').map((s) => s.trim()).filter(Boolean) : null);
+async function pull({ dir, cfg }, collections = null, { overwrite = new Set(), menus: wantMenus = null } = {}) {
+	let want = collections ?? (flags.collections ? String(flags.collections).split(',').map((s) => s.trim()).filter(Boolean) : null);
+	// `menus` is not a collection on the website; it is pulled through the site's own menu tools.
+	let refreshMenus = wantMenus ?? (want === null || want.includes('menus'));
+	if (want) want = want.filter((c) => c !== 'menus');
+	if (want && !want.length) {
+		await pullMenus({ dir, cfg });
+		return;
+	}
 	const [{ json: doc }, { json: schema }] = await Promise.all([
 		tool(cfg, 'pull_website', { site_id: cfg.site, ...(want ? { collections: want.join(',') } : {}) }),
 		tool(cfg, 'get_website_schema', { site_id: cfg.site })
@@ -184,7 +191,9 @@ async function pull({ dir, cfg }, collections = null) {
 			const key = `${collection}/${e.slug ?? e.id}`;
 			const mine = local[key];
 			const was = prev?.entries?.[key];
-			if (mine && was && !sameEntry(mine, was) && !sameEntry(mine, { status: e.status, data: e.data })) {
+			// An entry this push just wrote is the website's now: its file takes the website's version ($media
+			// becomes the stored image, _version appears), whatever the file said a moment ago.
+			if (mine && was && !overwrite.has(key) && !sameEntry(mine, was) && !sameEntry(mine, { status: e.status, data: e.data })) {
 				kept.push(key);
 				continue;
 			}
@@ -205,14 +214,14 @@ async function pull({ dir, cfg }, collections = null) {
 	}
 	writeFiles(dir, files);
 	const snap = snapshotOf(doc);
-	if (!want) {
+	if (refreshMenus) {
 		const menus = await readMenus(cfg);
 		snap.menus = menus;
 		const localMenus = readLocalMenus(dir);
 		for (const [name, menu] of Object.entries(menus)) {
 			const mine = localMenus[name];
 			const wasMenu = prev?.menus?.[name];
-			if (mine && !mine.error && wasMenu && JSON.stringify(mine.items) !== JSON.stringify(wasMenu.items) && JSON.stringify(mine.items) !== JSON.stringify(menu.items)) {
+			if (mine && !mine.error && wasMenu && !overwrite.has(`menus/${name}`) && JSON.stringify(mine.items) !== JSON.stringify(wasMenu.items) && JSON.stringify(mine.items) !== JSON.stringify(menu.items)) {
 				kept.push(`menus/${name}`);
 				continue;
 			}
@@ -234,6 +243,17 @@ async function pull({ dir, cfg }, collections = null) {
 	for (const k of kept) say(`  kept your unpushed changes in ${k}`);
 }
 
+/** The menus alone: files and the snapshot's menus, the entries untouched. */
+async function pullMenus({ dir, cfg }) {
+	const snap = readJson(join(dir, META, 'snapshot.json'));
+	if (!snap) fail('Nothing pulled yet. Run `tidysites pull`.');
+	const menus = await readMenus(cfg);
+	for (const [name, menu] of Object.entries(menus)) writeFiles(dir, { [`menus/${name}.json`]: `${JSON.stringify({ label: menu.label, items: menu.items }, null, '\t')}\n` });
+	snap.menus = menus;
+	writeFileSync(join(dir, META, 'snapshot.json'), JSON.stringify(snap, null, '\t'));
+	say(`Pulled ${Object.keys(menus).length} menu${Object.keys(menus).length === 1 ? '' : 's'}.`);
+}
+
 function status({ dir }) {
 	const snap = readJson(join(dir, META, 'snapshot.json'));
 	if (!snap) fail('Nothing pulled yet. Run `tidysites pull`.');
@@ -264,6 +284,15 @@ function validate({ dir }, local = null) {
 	if (!schema) fail('No content model here yet. Run `tidysites pull`.');
 	const read = local ? { local, problems: [] } : readLocal(dir);
 	const errors = [...read.problems, ...validateAll(read.local, schema)];
+	// The website keeps a section's type with its key: a key that changes type is refused with a bare "internal error", so say it here.
+	const snap = readJson(join(dir, META, 'snapshot.json'));
+	for (const [key, entry] of Object.entries(read.local)) {
+		const before = snap?.entries?.[key]?.data?.sections;
+		const after = entry.data?.sections;
+		if (!Array.isArray(before) || !Array.isArray(after)) continue;
+		const types = new Map(before.filter((b) => b && b._key).map((b) => [b._key, b._type]));
+		for (const b of after) if (b && b._key && types.has(b._key) && types.get(b._key) !== b._type) errors.push(`${key}.json: section "${b._key}" was a ${types.get(b._key)} and is now a ${b._type}; a section keeps its type, so give the new one its own _key`);
+	}
 	return errors;
 }
 
@@ -320,8 +349,10 @@ async function push(ctx) {
 		await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_set_items', arguments: { name, items } });
 		say(`Menu ${name}: ${items.length} item${items.length === 1 ? '' : 's'} written.`);
 	}
-	// Refresh only what went: a partial push leaves the other collections' files exactly as they are.
-	await pull(ctx, menus.length && !touched.length ? null : touched.length ? touched : null);
+	// Refresh only what went, and take the website's version of exactly those entries; every other file stays as it is.
+	const pushed = new Set([...keys, ...menus.map((m) => `menus/${m}`)]);
+	if (touched.length) await pull(ctx, touched, { overwrite: pushed, menus: menus.length > 0 });
+	else if (menus.length) await pullMenus(ctx);
 }
 
 async function shot(ctx, target = positional[0] ?? '/', opts = {}) {

package/lib/fetch.mjs

@@ -108,8 +108,14 @@ export function cssSummary(css) {
 		const f = m[1].split(',')[0].replace(/["']/g, '').trim();
 		if (f && !/^(inherit|initial|unset|var\()/.test(f)) fonts.set(f, (fonts.get(f) ?? 0) + 1);
 	}
+	// The site's own names for its colours and fonts, when its stylesheet defines them; a framework's defaults sit below these.
+	const variables = {};
+	for (const m of css.matchAll(/--([a-zA-Z][\w-]*)\s*:\s*([^;}]+)/g)) {
+		const v = m[2].trim();
+		if (/#[0-9a-f]{3,8}\b|rgba?\(|hsla?\(|font|["'][A-Z][\w ]+["']/i.test(v) && !/^var\(/.test(v)) variables[m[1]] = v;
+	}
 	const top = (map, n) => [...map.entries()].sort((a, b) => b[1] - a[1]).slice(0, n).map(([value, count]) => ({ value, count }));
-	return { colors: top(colors, 12), fonts: top(fonts, 6) };
+	return { colors: top(colors, 12), fonts: top(fonts, 6), variables };
 }
 
 export function outlineText(o) {
@@ -133,6 +139,8 @@ export function outlineText(o) {
 	if (o.emails.length) lines.push(`Emails: ${o.emails.join(', ')}`);
 	if (o.fonts.length) lines.push(`Google fonts: ${o.fonts.join(', ')}`);
 	if (o.css) {
+		const vars = Object.entries(o.css.variables ?? {});
+		if (vars.length) { lines.push('', 'Stylesheet variables (the site\'s own names; trust these over the counts below):'); for (const [k, v] of vars.slice(0, 40)) lines.push(`  --${k}: ${v}`); }
 		lines.push('', 'Stylesheet colours (most used first):', `  ${o.css.colors.map((c) => `${c.value} ×${c.count}`).join('  ')}`);
 		lines.push('Stylesheet fonts:', `  ${o.css.fonts.map((c) => `${c.value} ×${c.count}`).join('  ')}`);
 	}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {