@tideworthy/tidysites 0.1.14 → 0.1.16

package/README.md

@@ -63,7 +63,7 @@ shows in `status`.
 Styling: a row's `.css` file is scoped to that row (every rule is prefixed with the row's own selector), so it cannot style the rest of the page; rules for the whole page, the sections and the theme's classes go in `business/business.css`.
 
 Images: an image field, a hero's image or a Columns image slot takes
-`{ "$media": { "url": "https://...", "alt": "what it shows" } }` (or `{ "$media": { "file": "assets/x.png", "alt": "..." } }` for a picture that exists only as a file in the folder, say an illustration drawn from the old page's inline SVG; the push carries its bytes) and the
+`{ "$media": { "url": "https://...", "alt": "what it shows" } }` (or `{ "$media": { "file": "assets/x.png", "alt": "..." } }` for a picture that exists only as a file in the folder, say an illustration drawn from the old page's inline SVG; the push carries its bytes; a `file` field, such as a row's `media` list or the Business font files, takes a video (mp4, webm, up to 40 MB), a font (woff2) or a pdf the same way) and the
 push fetches the file into the website's library; a file the library already
 holds (the same bytes) is reused, not added again. An image already in the
 library keeps the value the pull gave it.

package/bin/tidysites.mjs

@@ -87,11 +87,15 @@ function writeFiles(dir, files) {
 function readLocal(dir, only = null) {
 	const local = {};
 	const problems = [];
+	// Only the website's own collections are entries; a folder of the person's own (notes, assets, source) is left alone.
+	const schema = readJson(join(dir, META, 'schema.json'));
+	const known = schema ? new Set((schema.collections?.items ?? schema.collections ?? []).map((c) => c.slug).filter(Boolean)) : null;
 	for (const name of readdirSync(dir)) {
 		if (name.startsWith('.') || name === 'node_modules') continue;
 		const colDir = join(dir, name);
 		if (!statSync(colDir).isDirectory()) continue;
 		if (!/^[a-z][a-z0-9_]*$/.test(name) || name === 'menus') continue;
+		if (known && known.size && !known.has(name)) continue;
 		if (only && !only.includes(name)) continue;
 		const sidecarExts = Object.keys(SIDECARS[name] ?? SIDECARS.default);
 		for (const file of readdirSync(colDir)) {
@@ -369,7 +373,9 @@ async function push(ctx) {
 	else if (menus.length) await pullMenus(ctx);
 }
 
-const MEDIA_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml', ico: 'image/x-icon' };
+const MEDIA_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml', ico: 'image/x-icon', mp4: 'video/mp4', m4v: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', woff2: 'font/woff2', woff: 'font/woff', ttf: 'font/ttf', otf: 'font/otf', pdf: 'application/pdf' };
+/** Pictures travel up to 15 MB; a video (a row's media list) up to 40 MB. */
+const mediaCap = (type) => (type.startsWith('video/') ? 40 : 15);
 /** Every { "$media": { "file": ... } } in the document becomes { "$media": { "data", "type", "filename", "alt" } }, read from the folder. */
 function inlineMediaFiles(doc, dir) {
 	const walk = (v) => {
@@ -381,9 +387,10 @@ function inlineMediaFiles(doc, dir) {
 				if (!path.startsWith(resolve(dir))) fail(`$media.file ${m.file}: outside the folder`);
 				if (!existsSync(path)) fail(`$media.file ${m.file}: no such file`);
 				const bytes = readFileSync(path);
-				if (bytes.length > 15 * 1024 * 1024) fail(`$media.file ${m.file}: ${(bytes.length / 1048576).toFixed(1)} MB; pictures up to 15 MB`);
 				const ext = m.file.split('.').pop().toLowerCase();
-				return { $media: { data: bytes.toString('base64'), type: MEDIA_TYPES[ext] ?? 'application/octet-stream', filename: m.file.split('/').pop(), alt: typeof m.alt === 'string' ? m.alt : '' } };
+				const type = MEDIA_TYPES[ext] ?? 'application/octet-stream';
+				if (bytes.length > mediaCap(type) * 1024 * 1024) fail(`$media.file ${m.file}: ${(bytes.length / 1048576).toFixed(1)} MB; ${type.startsWith('video/') ? 'videos up to 40 MB' : 'pictures up to 15 MB'}`);
+				return { $media: { data: bytes.toString('base64'), type, filename: m.file.split('/').pop(), alt: typeof m.alt === 'string' ? m.alt : '' } };
 			}
 			return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
 		}
@@ -675,7 +682,15 @@ async function main() {
 			writeFileSync(join(dir, META, 'key'), `${key}\n`, { mode: 0o600 });
 			writeFileSync(join(dir, META, '.gitignore'), 'key\nshots/\nsource/\npages/\n');
 			say(`Connected ${relative(process.cwd(), dir) || '.'} to ${site} on ${host}.`);
-			await pull(config());
+			// A website that is still being set up answers 404 on its content model for a few minutes; the first pull waits for it.
+			for (let attempt = 1; ; attempt++) {
+				try { await pull(config()); break; }
+				catch (e) {
+					if (e?.code !== 'website_unreachable' || attempt >= 20) throw e;
+					say(`The website is still being set up (${e.message.slice(0, 120)}); trying again in 30 seconds (${attempt}/20).`);
+					await new Promise((r) => setTimeout(r, 30_000));
+				}
+			}
 			return;
 		}
 		case 'pull':

package/lib/fetch.mjs

@@ -53,7 +53,7 @@ function abs(base, href) {
 
 /** What the page is made of, in order. */
 export function outline(html, base) {
-	const out = { url: base, title: null, description: null, headings: [], nav: [], images: [], phones: [], emails: [], stylesheets: [], fonts: [], colors: [] };
+	const out = { url: base, title: null, description: null, headings: [], nav: [], images: [], videos: [], phones: [], emails: [], stylesheets: [], fonts: [], colors: [] };
 	const t = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
 	if (t) out.title = text(t[1]);
 	const d = html.match(/<meta[^>]+name=["']description["'][^>]*content=["']([^"']*)["']/i) || html.match(/<meta[^>]+content=["']([^"']*)["'][^>]*name=["']description["']/i);
@@ -95,6 +95,11 @@ export function outline(html, base) {
 		const height = tag.match(/\sheight=["']?(\d+)/i)?.[1];
 		out.images.push({ src: abs(base, src), alt, ...(width ? { width: Number(width) } : {}), ...(height ? { height: Number(height) } : {}) });
 	}
+	// A video on the page (a hero background, a <source> inside a <video>): a file to bring over, never to play from the old address.
+	for (const m of html.matchAll(/<(?:video|source)\b[^>]*\ssrc=["']([^"']+\.(?:mp4|webm|m4v|mov)(?:\?[^"']*)?)["'][^>]*>/gi)) {
+		const src = abs(base, m[1]);
+		if (!out.videos.some((v) => v.src === src)) out.videos.push({ src, poster: m[0].match(/\sposter=["']([^"']+)["']/i)?.[1] ? abs(base, m[0].match(/\sposter=["']([^"']+)["']/i)[1]) : null });
+	}
 	const body = text(html.replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ''));
 	out.phones = [...new Set([...body.matchAll(/(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/g)].map((m) => m[0].trim()))].slice(0, 10);
 	out.emails = [...new Set([...body.matchAll(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi)].map((m) => m[0]))].slice(0, 10);
@@ -165,6 +170,10 @@ export function outlineText(o) {
 		lines.push('', `Images (${o.images.length}):`);
 		for (const i of o.images.slice(0, 60)) lines.push(`  ${i.src}${i.width ? ` (${i.width}x${i.height ?? '?'})` : ''}${i.alt ? `  alt: ${i.alt}` : ''}`);
 	}
+	if (o.videos?.length) {
+		lines.push('', `Videos (${o.videos.length}): download each (curl -L -o assets/<name>.mp4 <src>), list it in the row's "media" with a name, and write <video src="{{ media.<name>.url }}" autoplay muted loop playsinline>; a video is never played from the old site's address`);
+		for (const v of o.videos) lines.push(`  ${v.src}${v.poster ? `  poster: ${v.poster}` : ''}`);
+	}
 	if (o.frames?.length) {
 		lines.push('', `Embedded frames (${o.frames.length}): a band of the page each; rebuild as the frame section or a row's <iframe>`);
 		for (const f of o.frames) lines.push(`  ${f.src}${f.title ? `  title: ${f.title}` : ''}`);

package/lib/validate.mjs

@@ -53,7 +53,10 @@ function checkScalar(type, value, at, errors) {
 		case 'media':
 		case 'file':
 			if (!isMedia(value)) errors.push(`${at}: an image is the stored value (from the pull or add_website_image) or { "$media": { "url": "https://..." | "file": "assets/x.png", "alt": "..." } }`);
-			else if (value.$media && typeof value.$media.file === 'string') { if (!/\.(png|jpe?g|gif|webp|avif|svg|ico)$/i.test(value.$media.file) || value.$media.file.startsWith('/') || value.$media.file.includes('..')) errors.push(`${at}: $media.file must be a picture file inside the folder (png, jpg, gif, webp, avif, svg, ico)`); }
+			else if (value.$media && typeof value.$media.file === 'string') {
+				const ok = type === 'file' ? /\.(png|jpe?g|gif|webp|avif|svg|ico|mp4|webm|m4v|mov|woff2?|ttf|otf|pdf)$/i : /\.(png|jpe?g|gif|webp|avif|svg|ico)$/i;
+				if (!ok.test(value.$media.file) || value.$media.file.startsWith('/') || value.$media.file.includes('..')) errors.push(`${at}: $media.file must be a file inside the folder (${type === 'file' ? 'a picture, a video (mp4, webm), a font (woff2) or a pdf' : 'a picture: png, jpg, gif, webp, avif, svg, ico'})`);
+			}
 			else if (value.$media && (typeof value.$media.url !== 'string' || !/^https?:\/\//.test(value.$media.url))) errors.push(`${at}: $media.url must be an http(s) address (or $media.file for a picture in the folder)`);
 			else if (value.$media && typeof value.$media.alt !== 'string') errors.push(`${at}: $media.alt is required (what the image shows; "" for a decorative one)`);
 			return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tideworthy/tidysites",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "Pull a Tidysites website to files, edit, validate against its content model, push it back, screenshot the result.",
   "type": "module",
   "bin": {