@tideworthy/tidysites 0.1.1 → 0.1.3

package/README.md

@@ -18,11 +18,11 @@ nothing else; it lives in `.tidysites/key`, which is kept out of git.
 | | |
 |---|---|
 | `tidysites init <site_id> --host <url> --key <key>` | connect this folder to a website and pull it |
-| `tidysites pull [--collections pages,services]` | the website's content to files; files for entries the website no longer has are removed |
+| `tidysites pull [--collections pages,services,menus]` | the website's content to files; files for entries the website no longer has are removed |
 | `tidysites status` | what changed here since the pull |
 | `tidysites validate` | every file against the content model, with the file and the field in each message |
 | `tidysites push [--prune] [--force] [--dry] [--only pages]` | validate, then write the created and updated entries back (with `--prune`, delete the entries whose files are gone), then pull |
-| `tidysites shot <path or url> [--mobile] [--desktop] [--first-screen] [--live]` | a screenshot of a page of the preview site, the live site, or any address, saved under `.tidysites/shots/` |
+| `tidysites shot <path or url> [--mobile] [--tablet] [--desktop] [--wide] [--all] [--first-screen] [--live]` | a screenshot of a page of the preview site, the live site, or any address, saved under `.tidysites/shots/`; none of the width flags means desktop and mobile; a port compares all four |
 | `tidysites pages` | every address the website renders |
 | `tidysites page <path> [--html]` | the rendered page, section by section, with the key each section carries |
 | `tidysites schema [type or collection]` | the section types and layouts, or one type's or collection's fields |
@@ -30,6 +30,10 @@ nothing else; it lives in `.tidysites/key`, which is kept out of git.
 | `tidysites fetch <url> [--out dir]` | a page of the site being ported: cleaned HTML, its outline (headings, navigation, images, phones, fonts, colours) and its stylesheets |
 | `tidysites publish --yes` | the live site becomes a copy of the preview site |
 
+A section keeps its type with its key: to replace an intro with a row, give the row its own `_key`
+(`validate` says so; the website would only say "internal error"). After a push the pushed files take the
+website's version (a `$media` becomes the stored image, `_version` appears); every other file stays as it is.
+
 A push that would overwrite an entry changed on the website since the pull
 is refused with the list; pull again (your files are overwritten, so copy
 what you need first) or push `--force`.
@@ -78,4 +82,4 @@ names its own in `header_row`.
 3. `frame/frame.json` for the top bar, the call bar, the footer headings, the wording; `menus/primary.json` and `menus/footer.json` for the menus (an item with `children` is a dropdown).
 4. Pages as sections: `pages/<slug>.json`, each a list in `data.sections`. Columns is the section for a band with no record behind it.
 5. The look: brand colours and fonts on the Business record, `business/business.css` for what the tokens cannot say. The theme's own rules are written as `.ts.ts-accent` and `.ts.ts-ink .ts-prose a`, so a rule of yours needs the same weight (`.ts.ts-accent …`) to win.
-6. `tidysites validate`, `tidysites push`, `tidysites shot /` and `tidysites shot / --mobile`, and look.
+6. `tidysites validate`, `tidysites push`, `tidysites shot / --all`, and look at every width: a design often shows a different header above 1440px.

package/bin/tidysites.mjs

@@ -7,7 +7,7 @@
  *   status                                    what changed here since the pull
  *   validate                                  check the files against the content model
  *   push [--prune] [--force] [--dry]          write the changes back, then pull
- *   shot <path|url> [--mobile|--desktop] [--first-screen]   screenshot to .tidysites/shots
+ *   shot <path|url> [--mobile] [--tablet] [--desktop] [--wide] [--all] [--first-screen]   screenshot to .tidysites/shots (none = desktop and mobile)
  *   pages                                     every address the website renders
  *   page <path> [--html]                      the rendered page, section by section
  *   schema [name]                             section types and layouts, or one type's or collection's fields
@@ -165,8 +165,15 @@ function readLocalMenus(dir) {
 	return out;
 }
 
-async function pull({ dir, cfg }, collections = null) {
-	const want = collections ?? (flags.collections ? String(flags.collections).split(',').map((s) => s.trim()).filter(Boolean) : null);
+async function pull({ dir, cfg }, collections = null, { overwrite = new Set(), menus: wantMenus = null } = {}) {
+	let want = collections ?? (flags.collections ? String(flags.collections).split(',').map((s) => s.trim()).filter(Boolean) : null);
+	// `menus` is not a collection on the website; it is pulled through the site's own menu tools.
+	let refreshMenus = wantMenus ?? (want === null || want.includes('menus'));
+	if (want) want = want.filter((c) => c !== 'menus');
+	if (want && !want.length) {
+		await pullMenus({ dir, cfg });
+		return;
+	}
 	const [{ json: doc }, { json: schema }] = await Promise.all([
 		tool(cfg, 'pull_website', { site_id: cfg.site, ...(want ? { collections: want.join(',') } : {}) }),
 		tool(cfg, 'get_website_schema', { site_id: cfg.site })
@@ -184,7 +191,9 @@ async function pull({ dir, cfg }, collections = null) {
 			const key = `${collection}/${e.slug ?? e.id}`;
 			const mine = local[key];
 			const was = prev?.entries?.[key];
-			if (mine && was && !sameEntry(mine, was) && !sameEntry(mine, { status: e.status, data: e.data })) {
+			// An entry this push just wrote is the website's now: its file takes the website's version ($media
+			// becomes the stored image, _version appears), whatever the file said a moment ago.
+			if (mine && was && !overwrite.has(key) && !sameEntry(mine, was) && !sameEntry(mine, { status: e.status, data: e.data })) {
 				kept.push(key);
 				continue;
 			}
@@ -205,14 +214,14 @@ async function pull({ dir, cfg }, collections = null) {
 	}
 	writeFiles(dir, files);
 	const snap = snapshotOf(doc);
-	if (!want) {
+	if (refreshMenus) {
 		const menus = await readMenus(cfg);
 		snap.menus = menus;
 		const localMenus = readLocalMenus(dir);
 		for (const [name, menu] of Object.entries(menus)) {
 			const mine = localMenus[name];
 			const wasMenu = prev?.menus?.[name];
-			if (mine && !mine.error && wasMenu && JSON.stringify(mine.items) !== JSON.stringify(wasMenu.items) && JSON.stringify(mine.items) !== JSON.stringify(menu.items)) {
+			if (mine && !mine.error && wasMenu && !overwrite.has(`menus/${name}`) && JSON.stringify(mine.items) !== JSON.stringify(wasMenu.items) && JSON.stringify(mine.items) !== JSON.stringify(menu.items)) {
 				kept.push(`menus/${name}`);
 				continue;
 			}
@@ -234,6 +243,17 @@ async function pull({ dir, cfg }, collections = null) {
 	for (const k of kept) say(`  kept your unpushed changes in ${k}`);
 }
 
+/** The menus alone: files and the snapshot's menus, the entries untouched. */
+async function pullMenus({ dir, cfg }) {
+	const snap = readJson(join(dir, META, 'snapshot.json'));
+	if (!snap) fail('Nothing pulled yet. Run `tidysites pull`.');
+	const menus = await readMenus(cfg);
+	for (const [name, menu] of Object.entries(menus)) writeFiles(dir, { [`menus/${name}.json`]: `${JSON.stringify({ label: menu.label, items: menu.items }, null, '\t')}\n` });
+	snap.menus = menus;
+	writeFileSync(join(dir, META, 'snapshot.json'), JSON.stringify(snap, null, '\t'));
+	say(`Pulled ${Object.keys(menus).length} menu${Object.keys(menus).length === 1 ? '' : 's'}.`);
+}
+
 function status({ dir }) {
 	const snap = readJson(join(dir, META, 'snapshot.json'));
 	if (!snap) fail('Nothing pulled yet. Run `tidysites pull`.');
@@ -264,6 +284,15 @@ function validate({ dir }, local = null) {
 	if (!schema) fail('No content model here yet. Run `tidysites pull`.');
 	const read = local ? { local, problems: [] } : readLocal(dir);
 	const errors = [...read.problems, ...validateAll(read.local, schema)];
+	// The website keeps a section's type with its key: a key that changes type is refused with a bare "internal error", so say it here.
+	const snap = readJson(join(dir, META, 'snapshot.json'));
+	for (const [key, entry] of Object.entries(read.local)) {
+		const before = snap?.entries?.[key]?.data?.sections;
+		const after = entry.data?.sections;
+		if (!Array.isArray(before) || !Array.isArray(after)) continue;
+		const types = new Map(before.filter((b) => b && b._key).map((b) => [b._key, b._type]));
+		for (const b of after) if (b && b._key && types.has(b._key) && types.get(b._key) !== b._type) errors.push(`${key}.json: section "${b._key}" was a ${types.get(b._key)} and is now a ${b._type}; a section keeps its type, so give the new one its own _key`);
+	}
 	return errors;
 }
 
@@ -320,13 +349,17 @@ async function push(ctx) {
 		await tool(ctx.cfg, 'call_website_site_tool', { site_id: ctx.cfg.site, tool: 'menu_set_items', arguments: { name, items } });
 		say(`Menu ${name}: ${items.length} item${items.length === 1 ? '' : 's'} written.`);
 	}
-	// Refresh only what went: a partial push leaves the other collections' files exactly as they are.
-	await pull(ctx, menus.length && !touched.length ? null : touched.length ? touched : null);
+	// Refresh only what went, and take the website's version of exactly those entries; every other file stays as it is.
+	const pushed = new Set([...keys, ...menus.map((m) => `menus/${m}`)]);
+	if (touched.length) await pull(ctx, touched, { overwrite: pushed, menus: menus.length > 0 });
+	else if (menus.length) await pullMenus(ctx);
 }
 
 async function shot(ctx, target = positional[0] ?? '/', opts = {}) {
 	const isUrl = /^https?:\/\//.test(target);
-	const view = flags.mobile ? 'mobile' : flags.desktop ? 'desktop' : (opts.view ?? 'both');
+	// --desktop, --mobile, --tablet, --wide, any of them together; --all for every width; none = desktop and mobile.
+	const picked = ['desktop', 'mobile', 'tablet', 'wide'].filter((v) => flags[v]);
+	const view = flags.all ? 'all' : picked.length ? picked.join(',') : (opts.view ?? 'both');
 	const args = { site_id: ctx.cfg.site, view, full_page: !(flags['first-screen'] || opts.firstScreen), ...(isUrl ? { url: target } : { path: target, site: flags.live ? 'live' : 'preview' }) };
 	const { json, images } = await tool(ctx.cfg, 'screenshot_website', args);
 	const outDir = join(ctx.dir, META, 'shots');
@@ -335,7 +368,7 @@ async function shot(ctx, target = positional[0] ?? '/', opts = {}) {
 	const base = (opts.name ?? (isUrl ? new URL(target).hostname + new URL(target).pathname : target)).replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '') || 'home';
 	const named = Array.isArray(json?.captures) ? json.captures : [];
 	images.forEach((img, i) => {
-		const v = named[i]?.view ?? (view === 'both' ? ['desktop', 'mobile'][i] : view);
+		const v = named[i]?.view ?? (view === 'both' ? ['desktop', 'mobile'][i] : view === 'all' ? ['desktop', 'mobile', 'tablet', 'wide'][i] : view.split(',')[i] ?? view);
 		const p = join(outDir, `${base}-${v}-${stamp}.${img.mimeType.includes('png') ? 'png' : 'jpg'}`);
 		writeFileSync(p, Buffer.from(img.base64, 'base64'));
 		say(relative(process.cwd(), p));

package/lib/fetch.mjs

@@ -108,8 +108,14 @@ export function cssSummary(css) {
 		const f = m[1].split(',')[0].replace(/["']/g, '').trim();
 		if (f && !/^(inherit|initial|unset|var\()/.test(f)) fonts.set(f, (fonts.get(f) ?? 0) + 1);
 	}
+	// The site's own names for its colours and fonts, when its stylesheet defines them; a framework's defaults sit below these.
+	const variables = {};
+	for (const m of css.matchAll(/--([a-zA-Z][\w-]*)\s*:\s*([^;}]+)/g)) {
+		const v = m[2].trim();
+		if (/#[0-9a-f]{3,8}\b|rgba?\(|hsla?\(|font|["'][A-Z][\w ]+["']/i.test(v) && !/^var\(/.test(v)) variables[m[1]] = v;
+	}
 	const top = (map, n) => [...map.entries()].sort((a, b) => b[1] - a[1]).slice(0, n).map(([value, count]) => ({ value, count }));
-	return { colors: top(colors, 12), fonts: top(fonts, 6) };
+	return { colors: top(colors, 12), fonts: top(fonts, 6), variables };
 }
 
 export function outlineText(o) {
@@ -133,6 +139,8 @@ export function outlineText(o) {
 	if (o.emails.length) lines.push(`Emails: ${o.emails.join(', ')}`);
 	if (o.fonts.length) lines.push(`Google fonts: ${o.fonts.join(', ')}`);
 	if (o.css) {
+		const vars = Object.entries(o.css.variables ?? {});
+		if (vars.length) { lines.push('', 'Stylesheet variables (the site\'s own names; trust these over the counts below):'); for (const [k, v] of vars.slice(0, 40)) lines.push(`  --${k}: ${v}`); }
 		lines.push('', 'Stylesheet colours (most used first):', `  ${o.css.colors.map((c) => `${c.value} ×${c.count}`).join('  ')}`);
 		lines.push('Stylesheet fonts:', `  ${o.css.fonts.map((c) => `${c.value} ×${c.count}`).join('  ')}`);
 	}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {