@pollium/vyva-convert 0.0.0-stage → 0.2.0

package/README.md

@@ -1,3 +1,101 @@
-# Temporary Holding Version
+# @pollium/vyva-convert
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Server-side tools for Vyva content. It cleans what clients send, imports Word, PowerPoint, PDF,
+Markdown and text files, and exports to PDF, Word, PowerPoint and HTML. It runs in Node.
+
+```sh
+npm install @pollium/vyva-convert
+```
+
+## Cleaning what clients send
+
+```ts
+import { cleanContent } from '@pollium/vyva-convert/clean-content';
+
+const content = cleanContent(body.content);
+```
+
+The cleaner keeps the editor's formatting. It removes scripts, event handlers, unsafe links and any
+image that does not point at a file id. Ids that repeat or are not valid get new ones. Each text box is
+saved as tall as its text. For a single HTML fragment, use `cleanHtml` from
+`@pollium/vyva-convert/clean-html`.
+
+## Exporting
+
+```ts
+import { exportContent, isExportFormat } from '@pollium/vyva-convert/export';
+import { gotenberg } from '@pollium/vyva-convert/gotenberg';
+import { contentDisposition } from '@pollium/vyva-convert/attachment';
+
+if(!isExportFormat(request.params.format)) return reply.code(400).send();
+
+const file = await exportContent(content, request.params.format, {
+    title: 'Subnetting guide',
+    language: 'en',
+    readImage: async (fileId) => {
+        const stored = await files.find(fileId);
+        return stored === null ? null : { data: stored.bytes, contentType: stored.contentType };
+    },
+    pdfEngine: gotenberg({ url: 'http://gotenberg:3000' })
+});
+
+reply.header('content-disposition', contentDisposition('Subnetting guide', file.extension)).type(file.contentType).send(file.data);
+```
+
+| Content | Formats |
+|---|---|
+| Document | `pdf`, `docx`, `html` |
+| Deck | `pptx`, `pdf` |
+
+- `readImage` is called once for every image the content uses. Return `null` and the export goes
+  on without that image.
+- The PDF fonts and images travel with the page, so the PDF engine needs no network access.
+- `pdfEngine` is only needed for `pdf`. It is a function that takes a `PdfJob` (the HTML, the files
+  it refers to by name, a footer with Chromium's `pageNumber` and `totalPages` classes, and the
+  page size in inches) and returns the PDF. Use `gotenberg` or write your own with Puppeteer or
+  Playwright.
+- The `html` export is one self-contained file, with its images and fonts inline.
+
+## Importing
+
+```ts
+import { importFile } from '@pollium/vyva-convert/import';
+
+const { title, content } = await importFile(upload.bytes, upload.filename, {
+    saveImage: async ({ name, data, contentType }) => (await files.store(name, data, contentType)).id
+});
+```
+
+- The file is recognized by its content, not its name. Word and PowerPoint files are told apart
+  by what their zip holds, and PDF by its signature. Only Markdown (`.md`) and text (`.txt`) are
+  recognized by their extension, and only when they are valid UTF-8.
+- Word, PDF, Markdown and text files become documents, and PowerPoint files become decks. A deck
+  keeps its theme colors and the font most of its text uses. What its slide master draws comes in
+  locked, behind the slide's own elements, and freeform shapes come in as polygons.
+- With `pdfSlides`, a PDF whose pages are landscape becomes a deck instead: each page a slide, with
+  its pictures and shapes where they were, and the loose lines a PDF keeps joined back into
+  paragraphs and lists. A portrait PDF stays a document. `pdfSlides` takes the PDF and returns it as
+  a PowerPoint file; `libreOffice()` from `@pollium/vyva-convert/libreoffice` does that with the
+  `soffice` on the machine (`command` changes the path), one conversion per call:
+
+  ```ts
+  import { libreOffice } from '@pollium/vyva-convert/libreoffice';
+
+  await importFile(upload.bytes, upload.filename, { saveImage, pdfSlides: libreOffice() });
+  ```
+- `saveImage` stores each image and returns its file id, which must be a UUID. Return `null` to
+  leave the image out. If it throws, the import fails with your error.
+- Files over 25 MB (`maxBytes` changes the limit) and files it cannot read are refused.
+
+## Errors
+
+Expected failures are a `ConvertError` from `@pollium/vyva-convert/errors`, with a `code`. Anything else is
+a bug or a broken file, so log it.
+
+| Code | Meaning |
+|---|---|
+| `unsupported` | The file is not one Vyva imports, or the content cannot be exported in that format |
+| `too-large` | The file, or what it unzips to, is over the limit |
+| `empty` | The file has nothing to import |
+| `pdf-failed` | The PDF engine could not be reached or answered with an error |
+| `slides-failed` | `pdfSlides` could not turn a PDF into slides |

package/dist/attachment.d.ts

@@ -0,0 +1,3 @@
+//#region src/attachment.d.ts
+export declare const contentDisposition: (title: string, extension: string) => string;
+//#endregion

package/dist/attachment.js

@@ -0,0 +1,8 @@
+//#region src/attachment.ts
+const ASCII_FALLBACK = "download";
+const NAME_LIMIT = 80;
+const asciiSlug = (title) => title.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, NAME_LIMIT) || ASCII_FALLBACK;
+const readable = (title) => title.replace(/[\\/:*?"<>|\r\n]+/g, " ").replace(/\s+/g, " ").trim().slice(0, NAME_LIMIT) || ASCII_FALLBACK;
+const contentDisposition = (title, extension) => `attachment; filename="${asciiSlug(title)}.${extension}"; filename*=UTF-8''${encodeURIComponent(`${readable(title)}.${extension}`)}`;
+//#endregion
+export { contentDisposition };

package/dist/clean-content.d.ts

@@ -0,0 +1,6 @@
+import { Content, DeckContent, DocumentContent } from "@pollium/vyva-core/content";
+//#region src/clean-content.d.ts
+export declare const cleanDocument: (content: DocumentContent) => DocumentContent;
+export declare const cleanDeck: (content: DeckContent) => DeckContent;
+export declare const cleanContent: (content: Content) => Content;
+//#endregion

package/dist/clean-content.js

@@ -0,0 +1,63 @@
+import { cleanHtml } from "./clean-html.js";
+import { textHeight } from "./text-measure.js";
+import { TEXT_SHAPES } from "@pollium/vyva-core/content";
+import { newId } from "@pollium/vyva-core/ids";
+import { withTextHeight } from "@pollium/vyva-core/render";
+//#region src/clean-content.ts
+const ID = /^[A-Za-z0-9_-]{1,40}$/;
+const uniqueIds = (items) => {
+	const seen = /* @__PURE__ */ new Set();
+	return items.map((item) => {
+		const id = ID.test(item.id) && !seen.has(item.id) ? item.id : newId();
+		seen.add(id);
+		return id === item.id ? item : {
+			...item,
+			id
+		};
+	});
+};
+const cleanElement = (element, font) => {
+	if (element.type === "text") {
+		const html = cleanHtml(element.html);
+		const height = textHeight(html, element.style, font, element.w);
+		return Number.isFinite(height) && height > 0 ? withTextHeight({
+			...element,
+			html
+		}, height) : {
+			...element,
+			html
+		};
+	}
+	if (element.type !== "shape") return element;
+	if (element.points !== void 0 && element.shape !== "polygon") {
+		const { points: _points, ...plain } = element;
+		return cleanElement(plain, font);
+	}
+	if (element.text === void 0) return element;
+	const { text, ...bare } = element;
+	const html = TEXT_SHAPES.includes(element.shape) ? cleanHtml(text.html) : "";
+	return html === "" ? bare : {
+		...bare,
+		text: {
+			...text,
+			html
+		}
+	};
+};
+const cleanDocument = (content) => ({
+	...content,
+	blocks: uniqueIds(content.blocks).map((block) => ({
+		id: block.id,
+		html: cleanHtml(block.html)
+	})).filter((block) => block.html !== "")
+});
+const cleanDeck = (content) => ({
+	...content,
+	slides: uniqueIds(content.slides).map((slide) => ({
+		...slide,
+		elements: uniqueIds(slide.elements).map((element) => cleanElement(element, content.theme.font))
+	}))
+});
+const cleanContent = (content) => content.kind === "deck" ? cleanDeck(content) : cleanDocument(content);
+//#endregion
+export { cleanContent, cleanDeck, cleanDocument };

package/dist/clean-html.d.ts

@@ -0,0 +1,3 @@
+//#region src/clean-html.d.ts
+export declare const cleanHtml: (html: string) => string;
+//#endregion

package/dist/clean-html.js

@@ -0,0 +1,161 @@
+import sanitize from "sanitize-html";
+//#region src/clean-html.ts
+const COLOR = /^(#[0-9a-fA-F]{3,8}|rgba?\(\s*\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}\s*(,\s*(0|1|0?\.\d+)\s*)?\))$/;
+const FONT_SIZE = /^(\d{1,3}(\.\d{1,2})?)(px|pt|em|rem|%)$/;
+const ALIGN = /^(left|right|center|justify)$/;
+const LINE_HEIGHT = /^(\d(\.\d{1,2})?|\d{2,3}%)$/;
+const WIDTH = /^(\d{1,4}(\.\d{1,2})?)(px|%)$/;
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+const TEXT_STYLES = {
+	color: [COLOR],
+	"background-color": [COLOR],
+	"font-size": [FONT_SIZE],
+	"text-align": [ALIGN],
+	"line-height": [LINE_HEIGHT]
+};
+const OPTIONS = {
+	allowedTags: [
+		"p",
+		"h1",
+		"h2",
+		"h3",
+		"h4",
+		"strong",
+		"b",
+		"em",
+		"i",
+		"u",
+		"s",
+		"mark",
+		"span",
+		"a",
+		"br",
+		"hr",
+		"ul",
+		"ol",
+		"li",
+		"blockquote",
+		"pre",
+		"code",
+		"sup",
+		"sub",
+		"table",
+		"colgroup",
+		"col",
+		"thead",
+		"tbody",
+		"tr",
+		"th",
+		"td",
+		"img",
+		"div",
+		"label",
+		"input"
+	],
+	allowedAttributes: {
+		a: [
+			"href",
+			"target",
+			"rel"
+		],
+		img: [
+			"data-file-id",
+			"alt",
+			"width",
+			"style",
+			{
+				name: "data-align",
+				multiple: false,
+				values: [
+					"left",
+					"center",
+					"right"
+				]
+			}
+		],
+		ol: ["start"],
+		ul: ["data-type"],
+		li: ["data-type", "data-checked"],
+		code: ["class"],
+		pre: ["class"],
+		th: [
+			"colspan",
+			"rowspan",
+			"colwidth",
+			"style"
+		],
+		td: [
+			"colspan",
+			"rowspan",
+			"colwidth",
+			"style"
+		],
+		col: ["style"],
+		div: ["data-type", "data-tone"],
+		input: [
+			"type",
+			"checked",
+			"disabled"
+		],
+		"*": ["style"]
+	},
+	allowedClasses: {
+		code: [/^language-[a-z0-9+#-]{1,30}$/],
+		pre: [/^language-[a-z0-9+#-]{1,30}$/]
+	},
+	allowedStyles: {
+		"*": TEXT_STYLES,
+		img: { width: [WIDTH] },
+		col: { width: [WIDTH] },
+		td: {
+			...TEXT_STYLES,
+			width: [WIDTH]
+		},
+		th: {
+			...TEXT_STYLES,
+			width: [WIDTH]
+		}
+	},
+	allowedSchemes: [
+		"http",
+		"https",
+		"mailto"
+	],
+	allowedSchemesAppliedToAttributes: ["href"],
+	allowProtocolRelative: false,
+	disallowedTagsMode: "discard",
+	nonTextTags: [
+		"script",
+		"style",
+		"textarea",
+		"option",
+		"noscript",
+		"iframe",
+		"object",
+		"embed",
+		"svg",
+		"math"
+	],
+	exclusiveFilter: (frame) => frame.tag === "img" && !UUID.test(frame.attribs["data-file-id"] ?? "") || frame.tag === "input" && frame.attribs.type !== "checkbox" || frame.tag === "div" && frame.attribs["data-type"] !== void 0 && !["page-break", "callout"].includes(frame.attribs["data-type"]),
+	transformTags: {
+		a: (tagName, attribs) => ({
+			tagName,
+			attribs: {
+				...attribs,
+				target: "_blank",
+				rel: "noopener noreferrer"
+			}
+		}),
+		input: (tagName, attribs) => ({
+			tagName,
+			attribs: {
+				type: attribs.type ?? "",
+				...attribs.checked === void 0 ? {} : { checked: "checked" },
+				disabled: "disabled"
+			}
+		})
+	}
+};
+const cleanHtml = (html) => sanitize(html, OPTIONS).trim();
+//#endregion
+export { cleanHtml };

package/dist/errors.d.ts

@@ -0,0 +1,7 @@
+//#region src/errors.d.ts
+export type ConvertErrorCode = 'too-large' | 'unsupported' | 'empty' | 'pdf-failed' | 'slides-failed';
+export declare class ConvertError extends Error {
+  readonly code: ConvertErrorCode;
+  constructor(code: ConvertErrorCode, message: string, options?: ErrorOptions);
+}
+//#endregion

package/dist/errors.js

@@ -0,0 +1,11 @@
+//#region src/errors.ts
+var ConvertError = class extends Error {
+	code;
+	constructor(code, message, options) {
+		super(message, options);
+		this.name = "ConvertError";
+		this.code = code;
+	}
+};
+//#endregion
+export { ConvertError };

package/dist/export/bundle.d.ts

@@ -0,0 +1,34 @@
+import { ImageSize } from "./image-size.js";
+import { Content, FontFamily } from "@pollium/vyva-core/content";
+//#region src/export/bundle.d.ts
+export interface StoredImage {
+  data: Uint8Array;
+  contentType: string;
+}
+export type ReadImage = (fileId: string) => Promise<StoredImage | null>;
+export interface BundleImage {
+  id: string;
+  name: string;
+  data: Buffer;
+  contentType: string;
+  extension: string;
+  size: ImageSize;
+}
+export interface ExportBundle {
+  title: string;
+  language: string;
+  images: Map<string, BundleImage>;
+  fonts: FontFamily[];
+}
+export interface ExportedFile {
+  data: Uint8Array;
+  contentType: string;
+  extension: string;
+}
+export interface BundleOptions {
+  title: string;
+  language: string;
+  readImage: ReadImage;
+}
+export declare const bundleOf: (content: Content, { title, language, readImage }: BundleOptions) => Promise<ExportBundle>;
+//#endregion

package/dist/export/bundle.js

@@ -0,0 +1,41 @@
+import { imageSize } from "./image-size.js";
+import { CODE_FONT } from "@pollium/vyva-core/code";
+import { fileIdsOf } from "@pollium/vyva-core/render";
+//#region src/export/bundle.ts
+const EXTENSIONS = {
+	"image/png": "png",
+	"image/jpeg": "jpg",
+	"image/gif": "gif",
+	"image/webp": "webp"
+};
+const fontsOf = (content) => {
+	if (content.kind === "document") return [content.design.font, CODE_FONT];
+	return content.slides.some((slide) => slide.elements.some((element) => element.type === "code")) ? [content.theme.font, CODE_FONT] : [content.theme.font];
+};
+const bundleImage = (id, stored) => {
+	const data = Buffer.from(stored.data);
+	const extension = EXTENSIONS[stored.contentType] ?? "bin";
+	return {
+		id,
+		name: `${id}.${extension}`,
+		data,
+		contentType: stored.contentType,
+		extension,
+		size: imageSize(data, stored.contentType)
+	};
+};
+const bundleOf = async (content, { title, language, readImage }) => {
+	const ids = fileIdsOf(content);
+	const stored = await Promise.all(ids.map((id) => readImage(id)));
+	return {
+		title,
+		language,
+		images: new Map(ids.flatMap((id, index) => {
+			const image = stored[index];
+			return image === null ? [] : [[id, bundleImage(id, image)]];
+		})),
+		fonts: fontsOf(content)
+	};
+};
+//#endregion
+export { bundleOf };

package/dist/export/docx.d.ts

@@ -0,0 +1,5 @@
+import { ExportBundle, ExportedFile } from "./bundle.js";
+import { DocumentContent } from "@pollium/vyva-core/content";
+//#region src/export/docx.d.ts
+export declare const buildDocx: (content: DocumentContent, bundle: ExportBundle) => Promise<ExportedFile>;
+//#endregion