hecks 3.9.1 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hecks/deploy/bluebook/deploy.bluebook +106 -0
- data/lib/hecks/deploy/bluebook/deploy.hecksagon +8 -0
- data/lib/hecks/deploy/oidc.json +16 -3
- data/lib/hecks/hecks/adapters/deploy_toolchain/clearing.rb +51 -0
- data/lib/hecks/hecks/adapters/deploy_toolchain.rb +2 -0
- data/lib/hecks/hecks/adapters/site_toolchain/role_prober.rb +78 -0
- data/lib/hecks/hecks/adapters/site_toolchain.rb +29 -4
- data/lib/hecks/hecks/adapters/world_clearing.rb +65 -0
- data/lib/hecks/hecks/hecks.hecksagon +8 -0
- data/lib/hecks/hecks/hecks.world +2 -1
- data/lib/hecks/projections/deploy/fargate/settings/tuning.rb +15 -0
- data/lib/hecks/projections/deploy/fargate/settings.rb +3 -1
- data/lib/hecks/projections/deploy/fargate/stack/readers.rb +1 -0
- data/lib/hecks/projections/deploy/templates/fargate/Makefile.tmpl +7 -4
- data/lib/hecks/projections/site/cms_editor/attributes.rb +75 -0
- data/lib/hecks/projections/site/cms_editor/checks.rb +64 -0
- data/lib/hecks/projections/site/cms_editor/media.rb +85 -0
- data/lib/hecks/projections/site/cms_editor/pretty.rb +40 -0
- data/lib/hecks/projections/site/cms_editor/schema.rb +84 -0
- data/lib/hecks/projections/site/cms_editor/setting.rb +87 -0
- data/lib/hecks/projections/site/cms_editor/templates/package.json.tmpl +19 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/app.ts.tmpl +176 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/auth/membership.ts.tmpl +79 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/auth/session.ts.tmpl +36 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/auth/sso.ts.tmpl +20 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/commands.ts.tmpl +79 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/config.ts.tmpl +23 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/host.ts.tmpl +26 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/media/handler.ts.tmpl +154 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/media/multipart.ts.tmpl +53 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/media/sniff.ts.tmpl +94 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/media/storage.ts.tmpl +50 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/schema.ts.tmpl +85 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/server.ts.tmpl +65 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/body_model.js.tmpl +120 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/body_parse.js.tmpl +320 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/body_widget.js.tmpl +282 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/fields.ts.tmpl +80 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/html.ts.tmpl +102 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/input.ts.tmpl +75 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/media_picker.js.tmpl +109 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/outline.ts.tmpl +27 -0
- data/lib/hecks/projections/site/cms_editor/templates/src/ui/pages.ts.tmpl +123 -0
- data/lib/hecks/projections/site/cms_editor/templates/tsconfig.json.tmpl +16 -0
- data/lib/hecks/projections/site/cms_editor.rb +107 -0
- data/lib/hecks/projections/site/role_probe.rb +72 -0
- data/lib/hecks/site/bluebook/site.bluebook +28 -1
- data/lib/hecks/site/bluebook/site.ports.hecksagon +6 -0
- data/lib/hecks/tools/site_routes/cli.rb +1 -1
- data/lib/hecks/tools/site_routes/editor_files.rb +38 -0
- data/lib/hecks/tools/site_routes.rb +14 -7
- data/lib/hecks/version.rb +1 -1
- data/prebuilt/{verdicts-91aaf93ce8981bd9a754948f075d95650dc2ec6435015376f5c60992b36a24b1.json → verdicts-d6d584ad43b5f3cebf88aafbbdbf024c38574acf18e06adffbc555f7786b1c2f.json} +1 -1
- data/rust/Cargo.toml +3 -1
- data/rust/codegen/src/bridging.rs +1 -1
- data/rust/codegen/src/domain_generator.rs +9 -9
- data/rust/codegen/src/main.rs +1 -1
- data/rust/codegen/src/mutations.rs +46 -1
- data/rust/codegen/src/naming.rs +30 -0
- data/rust/host/HECKS_RELEASE +1 -1
- data/rust/src/kernel/expr.rs +7 -1
- data/rust/src/kernel/expression_operators/presence.rs +41 -3
- metadata +41 -5
- /data/prebuilt/{6e4a6bcf0aa880e2e6b056e1846b70e5a768090b9dc998bbadeaf5d6bcfd3db8.marshal → c28d6d9db14eea3e17161e9d9c262efee30b256cf97693a7cfd552bce24b2f7f.marshal} +0 -0
- /data/prebuilt/{0ee7f83cba59ce7d1c45c65ba166b071779f70143449f584f33ff7989f64eccb.marshal → cb375cba61ca0dbe8ddb9739c306510fe05e5f4b7a9b6955caff8224c63ea0e0.marshal} +0 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
__BANNER__
|
|
2
|
+
|
|
3
|
+
// The editor's HTTP server: node's own `http` module turning each request into a web `Request` for
|
|
4
|
+
// the handler in app.ts and writing its `Response` back. No framework. Start it with `npm start`;
|
|
5
|
+
// it listens on `PORT` (default 4400) on `HOST` (default 127.0.0.1), behind the site's edge, which
|
|
6
|
+
// routes the editor's base path here.
|
|
7
|
+
import { createServer } from "node:http";
|
|
8
|
+
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
9
|
+
|
|
10
|
+
import { createApp } from "./app.ts";
|
|
11
|
+
import { EDITOR } from "./config.ts";
|
|
12
|
+
|
|
13
|
+
const MAX_BODY_BYTES = 1_000_000;
|
|
14
|
+
|
|
15
|
+
/** The room a picture upload's form framing takes beyond the picture's own size cap. */
|
|
16
|
+
const UPLOAD_OVERHEAD_BYTES = 65_536;
|
|
17
|
+
|
|
18
|
+
/** How much of a request's body is read: a picture upload may be as large as the row's size cap. */
|
|
19
|
+
function limitOf(req: IncomingMessage): number {
|
|
20
|
+
const path = (req.url ?? "").split("?")[0];
|
|
21
|
+
return req.method === "POST" && path === `${EDITOR.basePath}/media` ? EDITOR.mediaMaxBytes + UPLOAD_OVERHEAD_BYTES : MAX_BODY_BYTES;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async function bodyOf(req: IncomingMessage): Promise<Buffer | undefined> {
|
|
25
|
+
if (req.method === "GET" || req.method === "HEAD") return undefined;
|
|
26
|
+
const limit = limitOf(req);
|
|
27
|
+
const chunks: Buffer[] = [];
|
|
28
|
+
let size = 0;
|
|
29
|
+
for await (const chunk of req) {
|
|
30
|
+
size += chunk.length;
|
|
31
|
+
if (size > limit) throw new RangeError("request body too large");
|
|
32
|
+
chunks.push(chunk);
|
|
33
|
+
}
|
|
34
|
+
return Buffer.concat(chunks);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function requestOf(req: IncomingMessage): Promise<Request> {
|
|
38
|
+
const proto = String(req.headers["x-forwarded-proto"] ?? "http").split(",")[0].trim();
|
|
39
|
+
const headers = new Headers();
|
|
40
|
+
for (const [name, value] of Object.entries(req.headers)) if (value !== undefined) headers.set(name, String(value));
|
|
41
|
+
return new Request(`${proto}://${req.headers.host ?? "localhost"}${req.url ?? "/"}`, { method: req.method, headers, body: (await bodyOf(req)) as BodyInit | undefined });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function send(res: ServerResponse, response: Response): Promise<void> {
|
|
45
|
+
const headers: Record<string, string | string[]> = {};
|
|
46
|
+
response.headers.forEach((value, name) => {
|
|
47
|
+
if (name !== "set-cookie") headers[name] = value;
|
|
48
|
+
});
|
|
49
|
+
const cookies = response.headers.getSetCookie();
|
|
50
|
+
if (cookies.length > 0) headers["set-cookie"] = cookies;
|
|
51
|
+
res.writeHead(response.status, headers);
|
|
52
|
+
res.end(Buffer.from(await response.arrayBuffer()));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const app = createApp();
|
|
56
|
+
|
|
57
|
+
createServer(async (req, res) => {
|
|
58
|
+
try {
|
|
59
|
+
await send(res, await app(await requestOf(req)));
|
|
60
|
+
} catch (err) {
|
|
61
|
+
console.error(err);
|
|
62
|
+
res.writeHead(err instanceof RangeError ? 413 : 500, { "Content-Type": "text/plain" });
|
|
63
|
+
res.end(err instanceof RangeError ? "Request too large." : "The editor failed.");
|
|
64
|
+
}
|
|
65
|
+
}).listen(Number(process.env.PORT || 4400), process.env.HOST || "127.0.0.1");
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
__BANNER__
|
|
2
|
+
|
|
3
|
+
// The structured document body (`blocks` of `spans` and `items`) as read-only HTML and as form
|
|
4
|
+
// fields. Plain JavaScript with no dependencies, so the same file runs in node (the pages) and in
|
|
5
|
+
// the browser (the widget). The domain's own tree is the only format: nothing here is an editor
|
|
6
|
+
// format, and every text, address and attribute value is escaped on its way into markup.
|
|
7
|
+
|
|
8
|
+
export const KINDS = ["paragraph", "heading", "quote", "bullet_list", "numbered_list", "image", "divider"];
|
|
9
|
+
export const ALIGNS = ["left", "center", "right", "justify"];
|
|
10
|
+
export const MARKS = ["bold", "italic", "underline", "strike", "code"];
|
|
11
|
+
export const TAGS = { bold: "strong", italic: "em", underline: "u", strike: "s", code: "code" };
|
|
12
|
+
export const MAX_INDENT = 8;
|
|
13
|
+
export const MAX_DEPTH = 5;
|
|
14
|
+
|
|
15
|
+
/** The body of a document with nothing in it. */
|
|
16
|
+
export const emptyBody = () => ({ blocks: [] });
|
|
17
|
+
|
|
18
|
+
/** Escapes text for an HTML body or a double-quoted attribute. */
|
|
19
|
+
export function esc(value) {
|
|
20
|
+
return String(value ?? "").replace(/[&<>"']/g, (char) => `&#${char.charCodeAt(0)};`);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The address when it is one a link may carry: a path (not `//`), http://, https://, mailto: or
|
|
25
|
+
* tel:, with no whitespace or control characters; otherwise null.
|
|
26
|
+
*/
|
|
27
|
+
export function safeHref(value) {
|
|
28
|
+
const text = String(value ?? "");
|
|
29
|
+
if (text === "" || /[\u0000- \u007f-\u009f\\]/.test(text)) return null;
|
|
30
|
+
return /^(\/(?!\/)|https?:\/\/|mailto:|tel:)/i.test(text) ? text : null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const markName = (mark) => (typeof mark === "string" ? mark : mark?.name);
|
|
34
|
+
const whole = (value, low, high) => (Number.isInteger(Number(value)) && value !== "" && value !== null ? Math.min(high, Math.max(low, Number(value))) : null);
|
|
35
|
+
const list = (value) => (Array.isArray(value) ? value : []);
|
|
36
|
+
|
|
37
|
+
function wrap(span, inner) {
|
|
38
|
+
const names = list(span.marks).map(markName).filter((name) => MARKS.includes(name));
|
|
39
|
+
const marked = [...names].reverse().reduce((html, name) => `<${TAGS[name]}>${html}</${TAGS[name]}>`, inner);
|
|
40
|
+
const href = safeHref(span.href);
|
|
41
|
+
return href ? `<a href="${esc(href)}" rel="noopener noreferrer">${marked}</a>` : marked;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The spans as inline HTML; a span of "\n" is a line break. */
|
|
45
|
+
export function spansToHtml(spans) {
|
|
46
|
+
return list(spans)
|
|
47
|
+
.map((span) => String(span?.text ?? "").split("\n").map((part, at) => wrap(span, (at > 0 ? "<br>" : "") + esc(part))).join(""))
|
|
48
|
+
.join("");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const attrs = (pairs) => pairs.filter(([, value]) => value !== null && value !== undefined).map(([name, value]) => ` ${name}="${esc(value)}"`).join("");
|
|
52
|
+
|
|
53
|
+
function layout(block) {
|
|
54
|
+
const align = ALIGNS.includes(block.align) ? block.align : null;
|
|
55
|
+
return attrs([["data-align", align], ["data-indent", whole(block.indent, 0, MAX_INDENT)]]);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function itemHtml(item) {
|
|
59
|
+
const kind = KINDS.slice(3, 5).includes(item?.list_kind) ? item.list_kind : null;
|
|
60
|
+
const depth = whole(item?.depth, 0, MAX_DEPTH);
|
|
61
|
+
return `<li${attrs([["data-depth", depth], ["data-list-kind", kind]])}>${spansToHtml(item?.spans)}</li>`;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function imageHtml(block) {
|
|
65
|
+
const note = `<span class="body-media">image <code>${esc(block.media_ref ?? "")}</code>${block.alt ? ` ${esc(block.alt)}` : ""}</span>`;
|
|
66
|
+
const caption = block.caption ? `<figcaption>${esc(block.caption)}</figcaption>` : "";
|
|
67
|
+
return `<figure${attrs([["data-media-ref", block.media_ref ?? ""], ["data-alt", block.alt], ["data-caption", block.caption]])}>${note}${caption}</figure>`;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** One block as HTML. An unknown kind is shown as a paragraph. */
|
|
71
|
+
export function blockToHtml(block) {
|
|
72
|
+
const kind = block?.kind;
|
|
73
|
+
if (kind === "divider") return "<hr>";
|
|
74
|
+
if (kind === "image") return imageHtml(block);
|
|
75
|
+
if (kind === "bullet_list" || kind === "numbered_list") {
|
|
76
|
+
const tag = kind === "bullet_list" ? "ul" : "ol";
|
|
77
|
+
return `<${tag}${layout(block)}>${list(block.items).map(itemHtml).join("")}</${tag}>`;
|
|
78
|
+
}
|
|
79
|
+
const level = whole(block?.level, 1, 4) ?? 2;
|
|
80
|
+
const tag = kind === "heading" ? `h${level}` : kind === "quote" ? "blockquote" : "p";
|
|
81
|
+
return `<${tag}${layout(block ?? {})}>${spansToHtml(block?.spans)}</${tag}>`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The body as HTML, one block after another; the same body gives the same text. */
|
|
85
|
+
export function bodyToHtml(body) {
|
|
86
|
+
return list(body?.blocks).map(blockToHtml).join("\n");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const present = (value) => value !== undefined && value !== null && value !== "";
|
|
90
|
+
|
|
91
|
+
function spanFields(span, name) {
|
|
92
|
+
const marks = list(span.marks).map(markName).filter(present).map((mark, at) => [`${name}.marks.${at}.name`, mark]);
|
|
93
|
+
return [[`${name}.text`, span.text ?? ""], ...marks, ...(present(span.href) ? [[`${name}.href`, span.href]] : [])];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function itemFields(item, name) {
|
|
97
|
+
const spans = list(item.spans).flatMap((span, at) => spanFields(span, `${name}.spans.${at}`));
|
|
98
|
+
return [...spans, ...(present(item.depth) ? [[`${name}.depth`, item.depth]] : []), ...(present(item.list_kind) ? [[`${name}.list_kind`, item.list_kind]] : [])];
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function blockFields(block, name) {
|
|
102
|
+
const spans = list(block.spans).flatMap((span, at) => spanFields(span, `${name}.spans.${at}`));
|
|
103
|
+
const items = list(block.items).flatMap((item, at) => itemFields(item, `${name}.items.${at}`));
|
|
104
|
+
const scalars = ["level", "align", "indent", "media_ref", "alt", "caption"].filter((key) => present(block[key])).map((key) => [`${name}.${key}`, block[key]]);
|
|
105
|
+
return [[`${name}.kind`, block.kind], ...scalars, ...spans, ...items];
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The body as the form's own fields: `[name, value]` pairs named by dotted path under `name`
|
|
110
|
+
* (`body.blocks.0.spans.1.marks.0.name`), the encoding the editor's other value objects and lists
|
|
111
|
+
* use, which the server reads back into the tree the command takes.
|
|
112
|
+
*/
|
|
113
|
+
export function bodyToFields(body, name) {
|
|
114
|
+
return list(body?.blocks).flatMap((block, at) => blockFields(block, `${name}.blocks.${at}`));
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The fields as hidden inputs. */
|
|
118
|
+
export function fieldsToInputs(pairs) {
|
|
119
|
+
return pairs.map(([name, value]) => `<input type="hidden" name="${esc(name)}" value="${esc(value)}">`).join("\n");
|
|
120
|
+
}
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
__BANNER__
|
|
2
|
+
|
|
3
|
+
// HTML read back into the domain's body tree, with no DOM: a small tokenizer and tree, so the same
|
|
4
|
+
// function runs in node (tests, the server) and in the browser (the widget, a paste). What the body
|
|
5
|
+
// cannot hold is reduced to the nearest block or mark it can, and every reduction adds a note, so a
|
|
6
|
+
// paste never loses anything without saying so. Output is deterministic: the same HTML gives the
|
|
7
|
+
// same tree, with adjacent spans of the same marks merged and empty ones dropped.
|
|
8
|
+
import { ALIGNS, MARKS, MAX_DEPTH, MAX_INDENT, safeHref } from "./body_model.js";
|
|
9
|
+
|
|
10
|
+
const VOID = new Set(["br", "hr", "img", "input", "meta", "link", "wbr", "col", "area", "base", "source"]);
|
|
11
|
+
const SKIPPED = new Set(["script", "style", "head", "title", "template", "noscript", "iframe", "object", "embed", "svg", "math"]);
|
|
12
|
+
const TRANSPARENT = new Set(["html", "body", "span", "font", "main", "article", "section", "header", "footer", "nav", "aside", "tbody", "thead", "tfoot", "center", "label", "abbr", "cite", "q", "small", "big", "time", "wbr"]);
|
|
13
|
+
const MARK_TAGS = { strong: "bold", b: "bold", em: "italic", i: "italic", u: "underline", ins: "underline", s: "strike", strike: "strike", del: "strike", code: "code", tt: "code", kbd: "code", samp: "code" };
|
|
14
|
+
const BLOCKISH = new Set(["p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "blockquote", "hr", "figure", "pre", "table", "tr", "td", "th", "dl", "dt", "dd", "details", "summary", "form", "fieldset"]);
|
|
15
|
+
const ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", hellip: "…", mdash: "—", ndash: "–", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", copy: "©", middot: "·", bull: "•" };
|
|
16
|
+
const LIMIT = 100;
|
|
17
|
+
|
|
18
|
+
function decode(text) {
|
|
19
|
+
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name) => {
|
|
20
|
+
if (name[0] !== "#") return ENTITIES[name.toLowerCase()] ?? whole;
|
|
21
|
+
const code = name[1].toLowerCase() === "x" ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10);
|
|
22
|
+
if (!(code > 0 && code <= 0x10ffff) || (code >= 0xd800 && code < 0xe000)) return "�";
|
|
23
|
+
return code === 160 ? " " : String.fromCodePoint(code);
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function attributesOf(source) {
|
|
28
|
+
const found = {};
|
|
29
|
+
for (const m of source.matchAll(/([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g)) {
|
|
30
|
+
const name = m[1].toLowerCase();
|
|
31
|
+
if (!(name in found)) found[name] = decode(m[2] ?? m[3] ?? m[4] ?? "");
|
|
32
|
+
}
|
|
33
|
+
return found;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** The tokens of `html`: text, open tags (with attributes) and close tags. Comments are dropped. */
|
|
37
|
+
function tokenize(html) {
|
|
38
|
+
const tokens = [];
|
|
39
|
+
const pattern = /<!--[\s\S]*?(?:-->|$)|<![^>]*>|<\?[^>]*>|<\/([a-z][a-z0-9-]*)[^>]*>|<([a-z][a-z0-9-]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>/gi;
|
|
40
|
+
let at = 0;
|
|
41
|
+
let m;
|
|
42
|
+
while ((m = pattern.exec(html)) !== null) {
|
|
43
|
+
if (m.index > at) tokens.push({ text: html.slice(at, m.index) });
|
|
44
|
+
at = m.index + m[0].length;
|
|
45
|
+
if (m[1]) tokens.push({ close: m[1].toLowerCase() });
|
|
46
|
+
else if (m[2]) tokens.push({ open: m[2].toLowerCase(), attrs: attributesOf(m[3]) });
|
|
47
|
+
if (m[2] && /^(script|style|textarea)$/i.test(m[2])) {
|
|
48
|
+
at = skipRaw(html, at, m[2].toLowerCase(), tokens);
|
|
49
|
+
pattern.lastIndex = at;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (at < html.length) tokens.push({ text: html.slice(at) });
|
|
53
|
+
return tokens;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function skipRaw(html, from, tag, tokens) {
|
|
57
|
+
const end = new RegExp(`</${tag}[^>]*>`, "i").exec(html.slice(from));
|
|
58
|
+
tokens.push({ close: tag });
|
|
59
|
+
return end ? from + end.index + end[0].length : html.length;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const closesLi = (stack) => {
|
|
63
|
+
for (let i = stack.length - 1; i >= 0; i -= 1) {
|
|
64
|
+
if (stack[i].tag === "li") return i;
|
|
65
|
+
if (stack[i].tag === "ul" || stack[i].tag === "ol") return -1;
|
|
66
|
+
}
|
|
67
|
+
return -1;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/** The nodes of `html` as a tree of `{ tag, attrs, kids }` and `{ text }`. */
|
|
71
|
+
function parseTree(html) {
|
|
72
|
+
const root = { tag: "#root", attrs: {}, kids: [] };
|
|
73
|
+
const stack = [root];
|
|
74
|
+
const ignored = {};
|
|
75
|
+
const top = () => stack[stack.length - 1];
|
|
76
|
+
for (const token of tokenize(html)) {
|
|
77
|
+
if ("text" in token) top().kids.push({ text: token.text });
|
|
78
|
+
else if (token.open) open(token, stack, ignored, top);
|
|
79
|
+
else close(token.close, stack, ignored);
|
|
80
|
+
}
|
|
81
|
+
return root.kids;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function open(token, stack, ignored, top) {
|
|
85
|
+
const tag = token.open;
|
|
86
|
+
if (tag === "li" && closesLi(stack) > 0) stack.length = closesLi(stack);
|
|
87
|
+
if (BLOCKISH.has(tag) && top().tag === "p") stack.pop();
|
|
88
|
+
const node = { tag, attrs: token.attrs, kids: [] };
|
|
89
|
+
if (stack.length > LIMIT) {
|
|
90
|
+
if (!VOID.has(tag)) ignored[tag] = (ignored[tag] ?? 0) + 1;
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
top().kids.push(node);
|
|
94
|
+
if (!VOID.has(tag)) stack.push(node);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function close(tag, stack, ignored) {
|
|
98
|
+
if (ignored[tag] > 0) {
|
|
99
|
+
ignored[tag] -= 1;
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
for (let i = stack.length - 1; i > 0; i -= 1) {
|
|
103
|
+
if (stack[i].tag === tag) {
|
|
104
|
+
stack.length = i;
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function styleOf(attrs) {
|
|
111
|
+
const style = {};
|
|
112
|
+
for (const part of String(attrs.style ?? "").split(";")) {
|
|
113
|
+
const [name, ...value] = part.split(":");
|
|
114
|
+
if (name && value.length) style[name.trim().toLowerCase()] = value.join(":").trim().toLowerCase();
|
|
115
|
+
}
|
|
116
|
+
return style;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const STYLE_MARKS = (style) => {
|
|
120
|
+
const found = [];
|
|
121
|
+
const weight = style["font-weight"];
|
|
122
|
+
if (weight === "bold" || weight === "bolder" || Number(weight) >= 600) found.push("bold");
|
|
123
|
+
if (style["font-style"] === "italic") found.push("italic");
|
|
124
|
+
const line = `${style["text-decoration"] ?? ""} ${style["text-decoration-line"] ?? ""}`;
|
|
125
|
+
if (line.includes("underline")) found.push("underline");
|
|
126
|
+
if (line.includes("line-through")) found.push("strike");
|
|
127
|
+
return found;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const STYLE_QUIET = new Set(["font-weight", "font-style", "text-decoration", "text-decoration-line", "text-align", "white-space", "margin-left", "padding-left"]);
|
|
131
|
+
|
|
132
|
+
function noteStyle(style, ctx) {
|
|
133
|
+
if (Object.keys(style).some((name) => !STYLE_QUIET.has(name))) ctx.note("inline styles other than bold, italic, underline, strike-through and alignment were dropped");
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function alignOf(attrs, style) {
|
|
137
|
+
const value = String(attrs["data-align"] ?? attrs.align ?? style["text-align"] ?? "").toLowerCase();
|
|
138
|
+
return ALIGNS.includes(value) ? value : undefined;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function indentOf(attrs) {
|
|
142
|
+
const value = Number(attrs["data-indent"]);
|
|
143
|
+
return attrs["data-indent"] !== undefined && Number.isInteger(value) ? Math.min(MAX_INDENT, Math.max(0, value)) : undefined;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function layoutOf(attrs) {
|
|
147
|
+
const style = styleOf(attrs);
|
|
148
|
+
const layout = {};
|
|
149
|
+
const align = alignOf(attrs, style);
|
|
150
|
+
const indent = indentOf(attrs);
|
|
151
|
+
if (align !== undefined) layout.align = align;
|
|
152
|
+
if (indent !== undefined) layout.indent = indent;
|
|
153
|
+
return layout;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const textOf = (nodes) => nodes.map((node) => ("text" in node ? decode(node.text) : textOf(node.kids))).join("");
|
|
157
|
+
|
|
158
|
+
/** Spans from inline nodes; `state` carries the marks and the address in force. */
|
|
159
|
+
function inline(nodes, state, ctx, out) {
|
|
160
|
+
for (const node of nodes) {
|
|
161
|
+
if ("text" in node) pushText(decode(node.text), state, out);
|
|
162
|
+
else inlineElement(node, state, ctx, out);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function pushText(raw, state, out) {
|
|
167
|
+
const text = raw.replace(/[ \t]*[\r\n\t][ \t\r\n]*/g, " ");
|
|
168
|
+
if (text !== "") out.push({ text, marks: state.marks.map((name) => ({ name })), ...(state.href ? { href: state.href } : {}) });
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function inlineElement(node, state, ctx, out) {
|
|
172
|
+
const tag = node.tag;
|
|
173
|
+
if (SKIPPED.has(tag)) return ctx.note(`<${tag}> content was removed`);
|
|
174
|
+
if (tag === "br") return out.push({ text: "\n", marks: state.marks.map((name) => ({ name })), ...(state.href ? { href: state.href } : {}) });
|
|
175
|
+
if (tag === "img") return ctx.note("an image with no media key was reduced to its alt text"), pushText(node.attrs.alt ?? "", state, out);
|
|
176
|
+
const style = styleOf(node.attrs);
|
|
177
|
+
noteStyle(style, ctx);
|
|
178
|
+
const added = [MARK_TAGS[tag], ...STYLE_MARKS(style)].filter(Boolean);
|
|
179
|
+
const href = tag === "a" ? linkOf(node.attrs.href, ctx) : undefined;
|
|
180
|
+
const next = { marks: [...state.marks, ...added.filter((name) => !state.marks.includes(name))], href: href ?? state.href };
|
|
181
|
+
if (!MARK_TAGS[tag] && tag !== "a" && !TRANSPARENT.has(tag)) ctx.note(`<${tag}> formatting was reduced to plain text`);
|
|
182
|
+
inline(node.kids, next, ctx, out);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function linkOf(value, ctx) {
|
|
186
|
+
if (value === undefined) return undefined;
|
|
187
|
+
const href = safeHref(value);
|
|
188
|
+
if (!href) ctx.note("a link to an address that is not a path, http(s), mailto or tel was reduced to text");
|
|
189
|
+
return href ?? undefined;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Adjacent spans with the same marks and address become one; empty text is dropped. */
|
|
193
|
+
function merged(spans) {
|
|
194
|
+
const out = [];
|
|
195
|
+
for (const span of spans) {
|
|
196
|
+
const last = out[out.length - 1];
|
|
197
|
+
const same = last && last.text !== "\n" && span.text !== "\n" && last.href === span.href && JSON.stringify(last.marks) === JSON.stringify(span.marks);
|
|
198
|
+
if (same) last.text += span.text;
|
|
199
|
+
else if (span.text !== "") out.push({ ...span, marks: span.marks.map((mark) => ({ ...mark })) });
|
|
200
|
+
}
|
|
201
|
+
return out;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function spansOf(nodes, ctx, state = { marks: [], href: undefined }) {
|
|
205
|
+
const out = [];
|
|
206
|
+
inline(nodes, state, ctx, out);
|
|
207
|
+
const only = out.length === 1 && out[0].text === "\n";
|
|
208
|
+
return only ? [] : merged(out);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function stateOf(node, ctx) {
|
|
212
|
+
const style = styleOf(node.attrs);
|
|
213
|
+
noteStyle(style, ctx);
|
|
214
|
+
return { marks: STYLE_MARKS(style), href: undefined };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const block = (kind, layout, spans, level) => ({ kind, ...(level ? { level } : {}), ...layout, spans });
|
|
218
|
+
|
|
219
|
+
function headingLevel(tag, ctx) {
|
|
220
|
+
const level = Number(tag[1]);
|
|
221
|
+
if (level > 4) ctx.note(`<${tag}> was reduced to a level 4 heading`);
|
|
222
|
+
return Math.min(4, level);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function figure(node, ctx) {
|
|
226
|
+
const attrs = node.attrs;
|
|
227
|
+
const kids = node.kids.filter((kid) => kid.tag === "figcaption");
|
|
228
|
+
const caption = attrs["data-caption"] ?? (kids.length ? textOf(kids[0].kids).trim() : "");
|
|
229
|
+
const made = { kind: "image" };
|
|
230
|
+
for (const [key, source] of [["media_ref", "data-media-ref"], ["alt", "data-alt"]]) if (attrs[source]) made[key] = attrs[source];
|
|
231
|
+
if (caption) made.caption = caption;
|
|
232
|
+
if (made.media_ref) return [made];
|
|
233
|
+
ctx.note("an image with no media key was reduced to its caption");
|
|
234
|
+
return caption ? [block("paragraph", {}, spansOf([{ text: caption }], ctx))] : [];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function listBlock(node, ctx) {
|
|
238
|
+
const kind = node.tag === "ol" ? "numbered_list" : "bullet_list";
|
|
239
|
+
const items = [];
|
|
240
|
+
collectItems(node, kind, 0, items, ctx);
|
|
241
|
+
return [{ kind, ...layoutOf(node.attrs), items }];
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function collectItems(list, blockKind, depth, items, ctx) {
|
|
245
|
+
const own = list.tag === "ol" ? "numbered_list" : "bullet_list";
|
|
246
|
+
for (const child of list.kids) {
|
|
247
|
+
if (child.tag === "li") itemOf(child, own, blockKind, depth, items, ctx);
|
|
248
|
+
else if (child.tag === "ul" || child.tag === "ol") collectItems(child, blockKind, Math.min(depth + 1, MAX_DEPTH), items, ctx);
|
|
249
|
+
else if (child.tag && !("text" in child)) collectItems({ ...child, tag: own }, blockKind, depth, items, ctx);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function itemOf(li, own, blockKind, depth, items, ctx) {
|
|
254
|
+
const nested = li.kids.filter((kid) => kid.tag === "ul" || kid.tag === "ol");
|
|
255
|
+
const here = li.kids.filter((kid) => !nested.includes(kid));
|
|
256
|
+
const marked = Number(li.attrs["data-depth"]);
|
|
257
|
+
const level = li.attrs["data-depth"] !== undefined && Number.isInteger(marked) ? Math.min(MAX_DEPTH, Math.max(0, marked)) : depth;
|
|
258
|
+
const given = li.attrs["data-list-kind"];
|
|
259
|
+
const kind = given === "bullet_list" || given === "numbered_list" ? given : own !== blockKind ? own : undefined;
|
|
260
|
+
if (depth > MAX_DEPTH) ctx.note(`a list nested deeper than ${MAX_DEPTH} levels was kept at depth ${MAX_DEPTH}`);
|
|
261
|
+
items.push({ spans: spansOf(here, ctx), depth: level, ...(kind ? { list_kind: kind } : {}) });
|
|
262
|
+
for (const list of nested) collectItems(list, blockKind, Math.min(level + 1, MAX_DEPTH), items, ctx);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** Blocks from block-level nodes; loose inline content between blocks becomes paragraphs. */
|
|
266
|
+
function blocks(nodes, ctx, quote = false) {
|
|
267
|
+
const out = [];
|
|
268
|
+
let run = [];
|
|
269
|
+
const flush = () => {
|
|
270
|
+
const spans = spansOf(run, ctx);
|
|
271
|
+
const blank = spans.every((span) => span.text.trim() === "" && span.text !== "\n");
|
|
272
|
+
if (!blank) out.push(block(quote ? "quote" : "paragraph", {}, spans));
|
|
273
|
+
run = [];
|
|
274
|
+
};
|
|
275
|
+
for (const node of nodes) {
|
|
276
|
+
if ("text" in node || !isBlock(node)) run.push(node);
|
|
277
|
+
else {
|
|
278
|
+
flush();
|
|
279
|
+
out.push(...blockOf(node, ctx, quote));
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
flush();
|
|
283
|
+
return out;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const isBlock = (node) => BLOCKISH.has(node.tag) || node.tag === "figcaption" ;
|
|
287
|
+
|
|
288
|
+
function blockOf(node, ctx, quote) {
|
|
289
|
+
const tag = node.tag;
|
|
290
|
+
if (tag === "hr") return [{ kind: "divider" }];
|
|
291
|
+
if (tag === "figure") return figure(node, ctx);
|
|
292
|
+
if (tag === "ul" || tag === "ol") return listBlock(node, ctx);
|
|
293
|
+
if (tag === "blockquote") return blocks(node.kids, ctx, true);
|
|
294
|
+
if (/^h[1-6]$/.test(tag)) return [block("heading", layoutOf(node.attrs), spansOf(node.kids, ctx, stateOf(node, ctx)), headingLevel(tag, ctx))];
|
|
295
|
+
if (tag === "pre") return preformatted(node, ctx);
|
|
296
|
+
if (tag === "p" || tag === "li" || tag === "dt" || tag === "dd") return [block(quote ? "quote" : "paragraph", layoutOf(node.attrs), spansOf(node.kids, ctx, stateOf(node, ctx)))];
|
|
297
|
+
if (tag === "div" && !node.kids.some((kid) => kid.tag && isBlock(kid))) return [block(quote ? "quote" : "paragraph", layoutOf(node.attrs), spansOf(node.kids, ctx, stateOf(node, ctx)))];
|
|
298
|
+
if (tag === "table" || tag === "tr" || tag === "td" || tag === "th") ctx.note("a table was reduced to paragraphs");
|
|
299
|
+
return blocks(node.kids, ctx, quote);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function preformatted(node, ctx) {
|
|
303
|
+
ctx.note("preformatted text was reduced to a paragraph in code");
|
|
304
|
+
const lines = textOf(node.kids).replace(/\r/g, "").split("\n");
|
|
305
|
+
const spans = lines.flatMap((line, at) => [...(at > 0 ? [{ text: "\n", marks: [] }] : []), ...(line ? [{ text: line, marks: [{ name: "code" }] }] : [])]);
|
|
306
|
+
return [block("paragraph", {}, spans)];
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** The body for `html`, with the notes naming whatever was reduced to fit. */
|
|
310
|
+
export function htmlToBodyWithNotes(html) {
|
|
311
|
+
const notes = [];
|
|
312
|
+
const ctx = { note: (message) => { if (!notes.includes(message)) notes.push(message); } };
|
|
313
|
+
const tree = parseTree(String(html ?? ""));
|
|
314
|
+
return { body: { blocks: blocks(tree, ctx) }, notes };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** The body for `html`; anything the body cannot hold is reduced to the nearest block or mark. */
|
|
318
|
+
export function htmlToBody(html) {
|
|
319
|
+
return htmlToBodyWithNotes(html).body;
|
|
320
|
+
}
|