@aglyn/plugins-ai 1.0.0-beta.288 → 1.0.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +17 -17
- package/src/lib/activity/ai-activity-actions.d.ts +3 -2
- package/src/lib/activity/ai-activity-actions.js +3 -0
- package/src/lib/activity/ai-activity-actions.js.map +1 -1
- package/src/lib/components/ai-job-requests.d.ts +2 -0
- package/src/lib/components/ai-job-requests.js +2 -0
- package/src/lib/components/ai-job-requests.js.map +1 -1
- package/src/lib/components/ai-site-build-page.component.js +4 -0
- package/src/lib/components/ai-site-build-page.component.js.map +1 -1
- package/src/lib/components/ai-site-import-job.component.d.ts +6 -1
- package/src/lib/components/ai-site-import-job.component.js +124 -17
- package/src/lib/components/ai-site-import-job.component.js.map +1 -1
- package/src/lib/components/ai-site-import-start.component.d.ts +6 -0
- package/src/lib/components/ai-site-import-start.component.js +18 -4
- package/src/lib/components/ai-site-import-start.component.js.map +1 -1
- package/src/lib/components/ai-site-start-card.component.js +2 -1
- package/src/lib/components/ai-site-start-card.component.js.map +1 -1
- package/src/lib/jobs/ai-image-reads.d.ts +20 -2
- package/src/lib/jobs/ai-image-reads.js +47 -0
- package/src/lib/jobs/ai-image-reads.js.map +1 -1
- package/src/lib/jobs/ai-job-import-step.d.ts +53 -3
- package/src/lib/jobs/ai-job-import-step.js +386 -64
- package/src/lib/jobs/ai-job-import-step.js.map +1 -1
- package/src/lib/jobs/ai-job-layout-language.d.ts +33 -0
- package/src/lib/jobs/ai-job-layout-language.js +75 -1
- package/src/lib/jobs/ai-job-layout-language.js.map +1 -1
- package/src/lib/jobs/ai-job-layout-step.js +8 -3
- package/src/lib/jobs/ai-job-layout-step.js.map +1 -1
- package/src/lib/jobs/ai-job-page-language.d.ts +29 -0
- package/src/lib/jobs/ai-job-page-language.js +178 -2
- package/src/lib/jobs/ai-job-page-language.js.map +1 -1
- package/src/lib/jobs/ai-job-page-step.js +3 -0
- package/src/lib/jobs/ai-job-page-step.js.map +1 -1
- package/src/lib/jobs/ai-site-import-document.d.ts +117 -0
- package/src/lib/jobs/ai-site-import-document.js +710 -0
- package/src/lib/jobs/ai-site-import-document.js.map +1 -0
- package/src/lib/jobs/ai-site-import-look.d.ts +112 -0
- package/src/lib/jobs/ai-site-import-look.js +402 -0
- package/src/lib/jobs/ai-site-import-look.js.map +1 -0
- package/src/lib/jobs/ai-site-import-publish.d.ts +41 -0
- package/src/lib/jobs/ai-site-import-publish.js +92 -0
- package/src/lib/jobs/ai-site-import-publish.js.map +1 -0
- package/src/lib/jobs/ai-theme-brand-inputs.d.ts +27 -2
- package/src/lib/jobs/ai-theme-brand-inputs.js +37 -8
- package/src/lib/jobs/ai-theme-brand-inputs.js.map +1 -1
- package/src/lib/layout-language/ai-layout-frame.d.ts +17 -0
- package/src/lib/layout-language/ai-layout-frame.js +73 -2
- package/src/lib/layout-language/ai-layout-frame.js.map +1 -1
- package/src/lib/layout-language/ai-layout-language.d.ts +43 -0
- package/src/lib/layout-language/ai-layout-language.js +199 -0
- package/src/lib/layout-language/ai-layout-language.js.map +1 -1
- package/src/lib/model/ai-credit-estimate.d.ts +15 -0
- package/src/lib/model/ai-credit-estimate.js +14 -1
- package/src/lib/model/ai-credit-estimate.js.map +1 -1
- package/src/lib/model/ai-jobs.types.d.ts +13 -1
- package/src/lib/model/ai-jobs.types.js.map +1 -1
- package/src/lib/model/ai-site-import-view.d.ts +47 -3
- package/src/lib/model/ai-site-import-view.js +122 -20
- package/src/lib/model/ai-site-import-view.js.map +1 -1
- package/src/lib/model/ai-site-import.d.ts +232 -7
- package/src/lib/model/ai-site-import.js +365 -35
- package/src/lib/model/ai-site-import.js.map +1 -1
- package/src/lib/runtime/ai-doctrine-validators.d.ts +18 -0
- package/src/lib/runtime/ai-doctrine-validators.js +16 -5
- package/src/lib/runtime/ai-doctrine-validators.js.map +1 -1
- package/src/lib/runtime/ai-palette.generated.js +31 -0
- package/src/lib/runtime/ai-palette.generated.js.map +1 -1
- package/src/lib/server/ai-image-reads-route.js +24 -1
- package/src/lib/server/ai-image-reads-route.js.map +1 -1
- package/src/lib/server/ai-jobs-resume.js +7 -1
- package/src/lib/server/ai-jobs-resume.js.map +1 -1
- package/src/lib/site-import/fixtures/aglyn-com-recordings.d.ts +57 -0
- package/src/lib/site-import/fixtures/aglyn-com-recordings.js +172 -0
- package/src/lib/site-import/fixtures/aglyn-com-recordings.js.map +1 -0
- package/src/lib/site-import/site-import-crawl.d.ts +32 -7
- package/src/lib/site-import/site-import-crawl.js +133 -44
- package/src/lib/site-import/site-import-crawl.js.map +1 -1
- package/src/lib/site-import/site-import-edge-server.d.ts +31 -0
- package/src/lib/site-import/site-import-edge-server.js +31 -0
- package/src/lib/site-import/site-import-edge-server.js.map +1 -0
- package/src/lib/site-import/site-import-edge.d.ts +44 -0
- package/src/lib/site-import/site-import-edge.js +108 -0
- package/src/lib/site-import/site-import-edge.js.map +1 -0
- package/src/lib/site-import/site-import-html.d.ts +73 -3
- package/src/lib/site-import/site-import-html.js +332 -45
- package/src/lib/site-import/site-import-html.js.map +1 -1
- package/src/lib/site-import/site-import-images.d.ts +22 -0
- package/src/lib/site-import/site-import-images.js +78 -2
- package/src/lib/site-import/site-import-images.js.map +1 -1
- package/src/lib/site-import/site-import-page.d.ts +44 -2
- package/src/lib/site-import/site-import-page.js +116 -16
- package/src/lib/site-import/site-import-page.js.map +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
1
2
|
/**
|
|
2
3
|
* @license
|
|
3
4
|
* Copyright 2026 Aglyn LLC
|
|
@@ -26,9 +27,17 @@
|
|
|
26
27
|
* and its photos' addresses. The text is kept as the page says it and is
|
|
27
28
|
* fenced as untrusted data only where a prompt shows it; the section names
|
|
28
29
|
* and the title travel outside the fence, so they are neutralized (no
|
|
29
|
-
* fence marker, no hidden character) and cut short.
|
|
30
|
+
* fence marker, no hidden character) and cut short. Every block of its
|
|
31
|
+
* main region rides along too (AGL-4083), so a page can be brought in word
|
|
32
|
+
* for word.
|
|
33
|
+
*
|
|
34
|
+
* And from the home page, what the site's header and footer are made of
|
|
35
|
+
* (AGL-4083): its brand as the source draws it (`siteImportHeaderBrand`) and
|
|
36
|
+
* its own facts (`siteImportSiteFacts`), for the header and footer the import
|
|
37
|
+
* builds.
|
|
30
38
|
*=========================================*/ import { neutralizeUntrustedText } from "@aglyn/shared-util-tools/untrusted-text";
|
|
31
|
-
import { AI_SITE_IMPORT_IMAGES_PER_PAGE, AI_SITE_IMPORT_LABEL_MAX_CHARS, AI_SITE_IMPORT_MAX_SECTIONS, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS } from "../model/ai-site-import.js";
|
|
39
|
+
import { AI_SITE_IMPORT_BLOCKED_STATUSES, AI_SITE_IMPORT_IMAGES_PER_PAGE, AI_SITE_IMPORT_LABEL_MAX_CHARS, AI_SITE_IMPORT_MAX_SECTIONS, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS, AI_SITE_IMPORT_READ_MAX_CHARS, aiSiteImportCapBlocks } from "../model/ai-site-import.js";
|
|
40
|
+
import { siteImportLogoCandidates } from "./site-import-images.js";
|
|
32
41
|
/** A line of untrusted text safe to show outside a fence: neutralized, one line, cut short. */ export function siteImportLabel(text, max = AI_SITE_IMPORT_LABEL_MAX_CHARS) {
|
|
33
42
|
const line = neutralizeUntrustedText(text).replace(/\s+/g, ' ').trim();
|
|
34
43
|
return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line;
|
|
@@ -37,8 +46,8 @@ const wordsIn = (text)=>{
|
|
|
37
46
|
var _text_match;
|
|
38
47
|
return ((_text_match = text.match(RegExp("[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}\\u0030-\\u0039\\u00B2-\\u00B3\\u00B9\\u00BC-\\u00BE\\u0660-\\u0669\\u06F0-\\u06F9\\u07C0-\\u07C9\\u0966-\\u096F\\u09E6-\\u09EF\\u09F4-\\u09F9\\u0A66-\\u0A6F\\u0AE6-\\u0AEF\\u0B66-\\u0B6F\\u0B72-\\u0B77\\u0BE6-\\u0BF2\\u0C66-\\u0C6F\\u0C78-\\u0C7E\\u0CE6-\\u0CEF\\u0D58-\\u0D5E\\u0D66-\\u0D78\\u0DE6-\\u0DEF\\u0E50-\\u0E59\\u0ED0-\\u0ED9\\u0F20-\\u0F33\\u1040-\\u1049\\u1090-\\u1099\\u1369-\\u137C\\u16EE-\\u16F0\\u17E0-\\u17E9\\u17F0-\\u17F9\\u1810-\\u1819\\u1946-\\u194F\\u19D0-\\u19DA\\u1A80-\\u1A89\\u1A90-\\u1A99\\u1B50-\\u1B59\\u1BB0-\\u1BB9\\u1C40-\\u1C49\\u1C50-\\u1C59\\u2070\\u2074-\\u2079\\u2080-\\u2089\\u2150-\\u2182\\u2185-\\u2189\\u2460-\\u249B\\u24EA-\\u24FF\\u2776-\\u2793\\u2CFD\\u3007\\u3021-\\u3029\\u3038-\\u303A\\u3192-\\u3195\\u3220-\\u3229\\u3248-\\u324F\\u3251-\\u325F\\u3280-\\u3289\\u32B1-\\u32BF\\uA620-\\uA629\\uA6E6-\\uA6EF\\uA830-\\uA835\\uA8D0-\\uA8D9\\uA900-\\uA909\\uA9D0-\\uA9D9\\uA9F0-\\uA9F9\\uAA50-\\uAA59\\uABF0-\\uABF9\\uFF10-\\uFF19\\u{10107}-\\u{10133}\\u{10140}-\\u{10178}\\u{1018A}-\\u{1018B}\\u{102E1}-\\u{102FB}\\u{10320}-\\u{10323}\\u{10341}\\u{1034A}\\u{103D1}-\\u{103D5}\\u{104A0}-\\u{104A9}\\u{10858}-\\u{1085F}\\u{10879}-\\u{1087F}\\u{108A7}-\\u{108AF}\\u{108FB}-\\u{108FF}\\u{10916}-\\u{1091B}\\u{109BC}-\\u{109BD}\\u{109C0}-\\u{109CF}\\u{109D2}-\\u{109FF}\\u{10A40}-\\u{10A48}\\u{10A7D}-\\u{10A7E}\\u{10A9D}-\\u{10A9F}\\u{10AEB}-\\u{10AEF}\\u{10B58}-\\u{10B5F}\\u{10B78}-\\u{10B7F}\\u{10BA9}-\\u{10BAF}\\u{10CFA}-\\u{10CFF}\\u{10D30}-\\u{10D39}\\u{10D40}-\\u{10D49}\\u{10E60}-\\u{10E7E}\\u{10F1D}-\\u{10F26}\\u{10F51}-\\u{10F54}\\u{10FC5}-\\u{10FCB}\\u{11052}-\\u{1106F}\\u{110F0}-\\u{110F9}\\u{11136}-\\u{1113F}\\u{111D0}-\\u{111D9}\\u{111E1}-\\u{111F4}\\u{112F0}-\\u{112F9}\\u{11450}-\\u{11459}\\u{114D0}-\\u{114D9}\\u{11650}-\\u{11659}\\u{116C0}-\\u{116C9}\\u{116D0}-\\u{116E3}\\u{11730}-\\u{1173B}\\u{118E0}-\\u{118F2}\\u{11950}-\\u{11959}\\u{11BF0}-\\u{11BF9}\\u{11C50}-\\u{11C6C}\\u{11D50}-\\u{11D59}\\u{11DA0}-\\u{11DA9}\\u{11DE0}-\\u{11DE9}\\u{11F50}-\\u{11F59}\\u{11FC0}-\\u{11FD4}\\u{12400}-\\u{1246E}\\u{16130}-\\u{16139}\\u{16A60}-\\u{16A69}\\u{16AC0}-\\u{16AC9}\\u{16B50}-\\u{16B59}\\u{16B5B}-\\u{16B61}\\u{16D70}-\\u{16D79}\\u{16E80}-\\u{16E96}\\u{16FF4}-\\u{16FF6}\\u{1CCF0}-\\u{1CCF9}\\u{1D2C0}-\\u{1D2D3}\\u{1D2E0}-\\u{1D2F3}\\u{1D360}-\\u{1D378}\\u{1D7CE}-\\u{1D7FF}\\u{1E140}-\\u{1E149}\\u{1E2F0}-\\u{1E2F9}\\u{1E4F0}-\\u{1E4F9}\\u{1E5F1}-\\u{1E5FA}\\u{1E8C7}-\\u{1E8CF}\\u{1E950}-\\u{1E959}\\u{1EC71}-\\u{1ECAB}\\u{1ECAD}-\\u{1ECAF}\\u{1ECB1}-\\u{1ECB4}\\u{1ED01}-\\u{1ED2D}\\u{1ED2F}-\\u{1ED3D}\\u{1F100}-\\u{1F10C}\\u{1FBF0}-\\u{1FBF9}]+", "gu"))) != null ? _text_match : []).length;
|
|
39
48
|
};
|
|
40
|
-
/** How many words a page says, its title and headings included. */ export function siteImportPageWords(page) {
|
|
41
|
-
return page.blocks.reduce((total, block)=>total + (block.kind === 'image' ? 0 : wordsIn(block.text)), 0);
|
|
49
|
+
/** How many words a page says, its title and headings included; a link inside a sentence is counted once. */ export function siteImportPageWords(page) {
|
|
50
|
+
return page.blocks.reduce((total, block)=>total + (block.kind === 'image' || block.kind === 'link' && block.inline ? 0 : wordsIn(block.text)), 0);
|
|
42
51
|
}
|
|
43
52
|
/** Between a page's own name and its site's in a browser title: "About | Rosie's Bakery". */ const TITLE_SEPARATOR = /\s+[|\u2013\u2014\u00b7:-]\s+/;
|
|
44
53
|
/**
|
|
@@ -118,28 +127,117 @@ const wordsIn = (text)=>{
|
|
|
118
127
|
else if (block.kind === 'image') lines.push(`${where}[photo: ${block.alt || 'no description'}]`);
|
|
119
128
|
else if (block.kind === 'list-item') lines.push(`${where}- ${block.text}`);
|
|
120
129
|
else if (block.kind === 'quote') lines.push(`${where}> ${block.text}`);
|
|
121
|
-
else
|
|
130
|
+
else if (block.kind === 'link') {
|
|
131
|
+
// A link inside a sentence is in its block's words already.
|
|
132
|
+
if (!block.inline) lines.push(`${where}${block.text}`);
|
|
133
|
+
} else lines.push(`${where}${block.text}`);
|
|
122
134
|
}
|
|
123
135
|
const text = lines.join('\n');
|
|
124
136
|
return text.length > AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS ? text.slice(0, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS) : text;
|
|
125
137
|
}
|
|
126
|
-
/** The page as a wave reads it. */ export function siteImportPageRead(page, at) {
|
|
127
|
-
|
|
138
|
+
/** The page as a wave reads it: its main region's blocks whole, within the read's budget. */ export function siteImportPageRead(page, at) {
|
|
139
|
+
const main = page.blocks.filter((block)=>block.region === 'main');
|
|
140
|
+
const capped = aiSiteImportCapBlocks(main, AI_SITE_IMPORT_READ_MAX_CHARS);
|
|
141
|
+
return _extends({
|
|
128
142
|
at,
|
|
129
143
|
title: siteImportPageTitle(page),
|
|
130
144
|
text: siteImportPageText(page),
|
|
131
145
|
sections: siteImportPageSections(page),
|
|
132
|
-
images: siteImportPageImages(page)
|
|
146
|
+
images: siteImportPageImages(page),
|
|
147
|
+
blocks: capped.blocks
|
|
148
|
+
}, capped.cut ? {
|
|
149
|
+
cut: true
|
|
150
|
+
} : {}, {
|
|
151
|
+
words: siteImportPageWords({
|
|
152
|
+
title: '',
|
|
153
|
+
blocks: main
|
|
154
|
+
}),
|
|
155
|
+
forms: page.forms.kept
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
/** The site's own name: its head's, its structured data's, else its home page's title before a separator. */ export function siteImportSiteName(home) {
|
|
159
|
+
var _home_title;
|
|
160
|
+
var _home_organization;
|
|
161
|
+
return siteImportLabel(home.siteName || ((_home_organization = home.organization) == null ? void 0 : _home_organization.name) || ((_home_title = home.title) != null ? _home_title : '').split(TITLE_SEPARATOR).pop() || '');
|
|
162
|
+
}
|
|
163
|
+
/** Words short enough to be a name, not a sentence. */ const looksLikeName = (text)=>text.length >= 2 && text.length <= AI_SITE_IMPORT_LABEL_MAX_CHARS && text.split(/\s+/).length <= 8;
|
|
164
|
+
/**
|
|
165
|
+
* The home page's header brand (AGL-4083), as the source shows it: the link
|
|
166
|
+
* home in its header, the picture inside it, and whether words sit beside
|
|
167
|
+
* the picture, inside the link or next to it. A header with no picture link
|
|
168
|
+
* home is the name alone.
|
|
169
|
+
*/ export function siteImportHeaderBrand(home) {
|
|
170
|
+
var _siteImportLogoCandidates_find, _ref;
|
|
171
|
+
var _home_brand;
|
|
172
|
+
const logo = (_siteImportLogoCandidates_find = siteImportLogoCandidates(home).find((candidate)=>candidate.source === 'header')) != null ? _siteImportLogoCandidates_find : null;
|
|
173
|
+
const siteName = siteImportSiteName(home);
|
|
174
|
+
const linked = siteImportLabel((_ref = (_home_brand = home.brand) == null ? void 0 : _home_brand.text) != null ? _ref : '');
|
|
175
|
+
const beside = home.blocks.find((block)=>block.region === 'header' && (block.kind === 'paragraph' || block.kind === 'heading' || block.kind === 'link') && !!siteName && block.text.trim().toLowerCase() === siteName.toLowerCase());
|
|
176
|
+
const printed = looksLikeName(linked) ? linked : beside && beside.kind !== 'image' ? siteImportLabel(beside.text) : '';
|
|
177
|
+
if (logo && printed) return {
|
|
178
|
+
kind: 'logo-and-name',
|
|
179
|
+
logo,
|
|
180
|
+
name: true,
|
|
181
|
+
text: printed
|
|
182
|
+
};
|
|
183
|
+
if (logo) return {
|
|
184
|
+
kind: 'logo-only',
|
|
185
|
+
logo,
|
|
186
|
+
name: false,
|
|
187
|
+
text: ''
|
|
188
|
+
};
|
|
189
|
+
return {
|
|
190
|
+
kind: 'name-only',
|
|
191
|
+
logo: null,
|
|
192
|
+
name: true,
|
|
193
|
+
text: printed || siteName
|
|
133
194
|
};
|
|
134
195
|
}
|
|
196
|
+
const SOCIAL_HOSTS = /(?:^|\.)(?:twitter\.com|x\.com|facebook\.com|instagram\.com|linkedin\.com|youtube\.com|tiktok\.com|github\.com|pinterest\.com|threads\.net|yelp\.com|tripadvisor\.com|nextdoor\.com|vimeo\.com|bsky\.app|mastodon\.social)$/i;
|
|
135
197
|
/**
|
|
136
|
-
* The
|
|
137
|
-
*
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
198
|
+
* The home page's own facts (AGL-4083), word for word: its header and footer
|
|
199
|
+
* blocks, its Organization structured data, and the email addresses, phone
|
|
200
|
+
* numbers and profiles they hold. Nothing here is invented or reworded; a
|
|
201
|
+
* fact the page does not state is not here.
|
|
202
|
+
*/ export function siteImportSiteFacts(home) {
|
|
203
|
+
var _home_organization, _ref;
|
|
204
|
+
const header = home.blocks.filter((block)=>block.region === 'header');
|
|
205
|
+
const footer = home.blocks.filter((block)=>block.region === 'footer');
|
|
206
|
+
const organization = (_home_organization = home.organization) != null ? _home_organization : null;
|
|
207
|
+
const emails = [];
|
|
208
|
+
const phones = [];
|
|
209
|
+
const social = [];
|
|
210
|
+
const add = (list, value)=>{
|
|
211
|
+
const clean = value.trim();
|
|
212
|
+
if (clean && !list.some((entry)=>entry.toLowerCase() === clean.toLowerCase())) list.push(clean);
|
|
213
|
+
};
|
|
214
|
+
if (organization == null ? void 0 : organization.email) add(emails, organization.email.toLowerCase());
|
|
215
|
+
if (organization == null ? void 0 : organization.telephone) add(phones, organization.telephone);
|
|
216
|
+
for (const url of (_ref = organization == null ? void 0 : organization.sameAs) != null ? _ref : [])add(social, url);
|
|
217
|
+
for (const block of [
|
|
218
|
+
...header,
|
|
219
|
+
...footer
|
|
220
|
+
]){
|
|
221
|
+
if (block.kind !== 'link') continue;
|
|
222
|
+
if (block.href.startsWith('mailto:')) add(emails, block.href.slice('mailto:'.length));
|
|
223
|
+
else if (block.href.startsWith('tel:')) add(phones, block.text);
|
|
224
|
+
else {
|
|
225
|
+
try {
|
|
226
|
+
if (SOCIAL_HOSTS.test(new URL(block.href).hostname)) add(social, block.href);
|
|
227
|
+
} catch (unused) {
|
|
228
|
+
// A link that is not an address is no profile.
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
header,
|
|
234
|
+
footer,
|
|
235
|
+
organization,
|
|
236
|
+
emails,
|
|
237
|
+
phones,
|
|
238
|
+
social
|
|
239
|
+
};
|
|
240
|
+
}
|
|
143
241
|
/** Why the crawler could not read a page, as an import leaves it out; `status` is the answer an `http-error` got. */ export function siteImportLeftOutReason(reason, status) {
|
|
144
242
|
switch(reason){
|
|
145
243
|
case 'login-form':
|
|
@@ -151,7 +249,9 @@ const wordsIn = (text)=>{
|
|
|
151
249
|
case 'off-origin-redirect':
|
|
152
250
|
return 'off-site';
|
|
153
251
|
case 'http-error':
|
|
154
|
-
return status !== undefined &&
|
|
252
|
+
return status !== undefined && AI_SITE_IMPORT_BLOCKED_STATUSES.has(status) ? 'blocked' : 'unreadable';
|
|
253
|
+
case 'not-reached':
|
|
254
|
+
return 'not-reached';
|
|
155
255
|
default:
|
|
156
256
|
return 'unreadable';
|
|
157
257
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-page.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * ONE CRAWLED PAGE, AS AN IMPORT KEEPS IT (AGL-3976).\n *\n * The crawler's page (`SiteImportPage`) holds everything it extracted; an\n * import keeps much less, and keeps it in two shapes:\n *\n * - **Listed**, for the inventory: its address, title, how it was found and\n * how many words and photos it has. No text: an inventory of a whole site\n * rides the job document.\n * - **Read**, for a wave: its text in order, its sections from its headings,\n * and its photos' addresses. The text is kept as the page says it and is\n * fenced as untrusted data only where a prompt shows it; the section names\n * and the title travel outside the fence, so they are neutralized (no\n * fence marker, no hidden character) and cut short.\n *=========================================*/\n\nimport { neutralizeUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n AI_SITE_IMPORT_IMAGES_PER_PAGE,\n AI_SITE_IMPORT_LABEL_MAX_CHARS,\n AI_SITE_IMPORT_MAX_SECTIONS,\n AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS,\n type AiSiteImportLeftOutReason,\n type AiSiteImportListedPage,\n type AiSiteImportPageRead,\n} from '../model/ai-site-import'\nimport type { SiteImportPage, SiteImportSkipReason } from './site-import-crawl'\n\n/** A line of untrusted text safe to show outside a fence: neutralized, one line, cut short. */\nexport function siteImportLabel(text: string, max = AI_SITE_IMPORT_LABEL_MAX_CHARS): string {\n const line = neutralizeUntrustedText(text).replace(/\\s+/g, ' ').trim()\n return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line\n}\n\nconst wordsIn = (text: string) => (text.match(/[\\p{L}\\p{N}]+/gu) ?? []).length\n\n/** How many words a page says, its title and headings included. */\nexport function siteImportPageWords(page: Pick<SiteImportPage, 'title' | 'blocks'>): number {\n return page.blocks.reduce((total, block) => total + (block.kind === 'image' ? 0 : wordsIn(block.text)), 0)\n}\n\n/** Between a page's own name and its site's in a browser title: \"About | Rosie's Bakery\". */\nconst TITLE_SEPARATOR = /\\s+[|\\u2013\\u2014\\u00b7:-]\\s+/\n\n/**\n * A page's name on the new site: \"Home\" for the home page, else its browser\n * title without the old site's name after a separator (\"About | Rosie's\n * Bakery\" is \"About\"), else its first heading.\n */\nexport function siteImportPageTitle(page: Pick<SiteImportPage, 'path' | 'title' | 'blocks'>): string {\n if ((page.path.replace(/\\/+$/, '') || '/') === '/') return 'Home'\n const own = siteImportLabel((page.title ?? '').split(TITLE_SEPARATOR)[0] ?? '')\n if (own.length >= 2) return own\n return siteImportLabel(page.blocks.find((block) => block.kind === 'heading')?.text ?? '')\n}\n\n/** The page as the inventory lists it. */\nexport function siteImportListedPage(page: SiteImportPage): AiSiteImportListedPage {\n return {\n url: page.url,\n path: page.path,\n title: siteImportPageTitle(page),\n foundBy: page.foundBy,\n words: siteImportPageWords(page),\n images: page.images.length,\n }\n}\n\n/** The page's photos, the ones in its body first, each once. */\nexport function siteImportPageImages(page: Pick<SiteImportPage, 'blocks' | 'images'>): Array<{ url: string; alt: string }> {\n const seen = new Set<string>()\n const images: Array<{ url: string; alt: string }> = []\n const add = (url: string, alt: string) => {\n if (!url || seen.has(url) || images.length >= AI_SITE_IMPORT_IMAGES_PER_PAGE) return\n seen.add(url)\n images.push({ url, alt: siteImportLabel(alt, 250) })\n }\n for (const block of page.blocks) if (block.kind === 'image' && block.region === 'main') add(block.url, block.alt)\n for (const image of page.images) add(image.url, image.alt)\n return images\n}\n\n/**\n * The page's sections, from its headings: the first is its opening (its\n * title or first heading), then one for each second-level heading of its\n * body, at most `AI_SITE_IMPORT_MAX_SECTIONS`. A page with no headings below\n * its title but more to say than an opening gets a second section for it.\n */\nexport function siteImportPageSections(page: Pick<SiteImportPage, 'title' | 'blocks'>): string[] {\n const main = page.blocks.filter((block) => block.region === 'main')\n const headings = main.filter((block): block is Extract<typeof block, { kind: 'heading' }> => block.kind === 'heading')\n const opening = siteImportLabel(headings.find((block) => block.level === 1)?.text || page.title || '') || 'Introduction'\n const sections = [opening]\n const seen = new Set([opening.toLowerCase()])\n for (const heading of headings) {\n if (heading.level !== 2) continue\n const name = siteImportLabel(heading.text)\n if (!name || seen.has(name.toLowerCase())) continue\n seen.add(name.toLowerCase())\n sections.push(name)\n if (sections.length >= AI_SITE_IMPORT_MAX_SECTIONS) break\n }\n if (sections.length === 1 && siteImportPageWords({ title: '', blocks: main }) > 80) sections.push('Details')\n return sections\n}\n\n/** The page's text in order, as a prompt shows it inside its fence. */\nexport function siteImportPageText(page: Pick<SiteImportPage, 'title' | 'description' | 'blocks'>): string {\n const lines: string[] = []\n if (page.title) lines.push(`Title: ${page.title}`)\n if (page.description) lines.push(`Description: ${page.description}`)\n for (const block of page.blocks) {\n const where = block.region === 'main' ? '' : `(${block.region}) `\n if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n else if (block.kind === 'image') lines.push(`${where}[photo: ${block.alt || 'no description'}]`)\n else if (block.kind === 'list-item') lines.push(`${where}- ${block.text}`)\n else if (block.kind === 'quote') lines.push(`${where}> ${block.text}`)\n else lines.push(`${where}${block.text}`)\n }\n const text = lines.join('\\n')\n return text.length > AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS ? text.slice(0, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS) : text\n}\n\n/** The page as a wave reads it. */\nexport function siteImportPageRead(page: SiteImportPage, at: string): AiSiteImportPageRead {\n return {\n at,\n title: siteImportPageTitle(page),\n text: siteImportPageText(page),\n sections: siteImportPageSections(page),\n images: siteImportPageImages(page),\n }\n}\n\n/**\n * The answers bot protection and firewalls turn a crawler away with: asked to\n * sign in, refused, or told to slow down. The owner can let AglynBot through.\n */\nconst SITE_IMPORT_BLOCKED_STATUSES: ReadonlySet<number> = new Set([401, 403, 429])\n\n/** Why the crawler could not read a page, as an import leaves it out; `status` is the answer an `http-error` got. */\nexport function siteImportLeftOutReason(reason: SiteImportSkipReason, status?: number): AiSiteImportLeftOutReason {\n switch (reason) {\n case 'login-form':\n return 'login-form'\n case 'robots':\n return 'robots'\n case 'too-large':\n return 'too-large'\n case 'off-origin-redirect':\n return 'off-site'\n case 'http-error':\n return status !== undefined && SITE_IMPORT_BLOCKED_STATUSES.has(status) ? 'blocked' : 'unreadable'\n default:\n return 'unreadable'\n }\n}\n"],"names":["neutralizeUntrustedText","AI_SITE_IMPORT_IMAGES_PER_PAGE","AI_SITE_IMPORT_LABEL_MAX_CHARS","AI_SITE_IMPORT_MAX_SECTIONS","AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS","siteImportLabel","text","max","line","replace","trim","length","slice","trimEnd","wordsIn","match","siteImportPageWords","page","blocks","reduce","total","block","kind","TITLE_SEPARATOR","siteImportPageTitle","path","own","title","split","find","siteImportListedPage","url","foundBy","words","images","siteImportPageImages","seen","Set","add","alt","has","push","region","image","siteImportPageSections","headings","main","filter","opening","level","sections","toLowerCase","heading","name","siteImportPageText","lines","description","where","repeat","join","siteImportPageRead","at","SITE_IMPORT_BLOCKED_STATUSES","siteImportLeftOutReason","reason","status","undefined"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;2CAc2C,GAE3C,SAASA,uBAAuB,QAAQ,0CAAyC;AACjF,SACEC,8BAA8B,EAC9BC,8BAA8B,EAC9BC,2BAA2B,EAC3BC,kCAAkC,QAI7B,6BAAyB;AAGhC,6FAA6F,GAC7F,OAAO,SAASC,gBAAgBC,IAAY,EAAEC,MAAML,8BAA8B;IAChF,MAAMM,OAAOR,wBAAwBM,MAAMG,OAAO,CAAC,QAAQ,KAAKC,IAAI;IACpE,OAAOF,KAAKG,MAAM,GAAGJ,MAAM,GAAGC,KAAKI,KAAK,CAAC,GAAGL,MAAM,GAAGM,OAAO,GAAG,CAAC,CAAC,GAAGL;AACtE;AAEA,MAAMM,UAAU,CAACR;QAAkBA;WAAD,EAACA,cAAAA,KAAKS,KAAK,CAAC,ynaAAXT,cAAiC,EAAE,EAAEK,MAAM;;AAE9E,iEAAiE,GACjE,OAAO,SAASK,oBAAoBC,IAA8C;IAChF,OAAOA,KAAKC,MAAM,CAACC,MAAM,CAAC,CAACC,OAAOC,QAAUD,QAASC,CAAAA,MAAMC,IAAI,KAAK,UAAU,IAAIR,QAAQO,MAAMf,IAAI,CAAA,GAAI;AAC1G;AAEA,2FAA2F,GAC3F,MAAMiB,kBAAkB;AAExB;;;;CAIC,GACD,OAAO,SAASC,oBAAoBP,IAAuD;QAE7D,SAACA;QAENA;IAHvB,IAAI,AAACA,CAAAA,KAAKQ,IAAI,CAAChB,OAAO,CAAC,QAAQ,OAAO,GAAE,MAAO,KAAK,OAAO;IAC3D,MAAMiB,MAAMrB,iBAAgB,UAAA,EAACY,cAAAA,KAAKU,KAAK,YAAVV,cAAc,IAAIW,KAAK,CAACL,gBAAgB,CAAC,EAAE,YAA5C,UAAgD;IAC5E,IAAIG,IAAIf,MAAM,IAAI,GAAG,OAAOe;IAC5B,OAAOrB,yBAAgBY,oBAAAA,KAAKC,MAAM,CAACW,IAAI,CAAC,CAACR,QAAUA,MAAMC,IAAI,KAAK,+BAA3CL,kBAAuDX,IAAI,mBAAI;AACxF;AAEA,wCAAwC,GACxC,OAAO,SAASwB,qBAAqBb,IAAoB;IACvD,OAAO;QACLc,KAAKd,KAAKc,GAAG;QACbN,MAAMR,KAAKQ,IAAI;QACfE,OAAOH,oBAAoBP;QAC3Be,SAASf,KAAKe,OAAO;QACrBC,OAAOjB,oBAAoBC;QAC3BiB,QAAQjB,KAAKiB,MAAM,CAACvB,MAAM;IAC5B;AACF;AAEA,8DAA8D,GAC9D,OAAO,SAASwB,qBAAqBlB,IAA+C;IAClF,MAAMmB,OAAO,IAAIC;IACjB,MAAMH,SAA8C,EAAE;IACtD,MAAMI,MAAM,CAACP,KAAaQ;QACxB,IAAI,CAACR,OAAOK,KAAKI,GAAG,CAACT,QAAQG,OAAOvB,MAAM,IAAIV,gCAAgC;QAC9EmC,KAAKE,GAAG,CAACP;QACTG,OAAOO,IAAI,CAAC;YAAEV;YAAKQ,KAAKlC,gBAAgBkC,KAAK;QAAK;IACpD;IACA,KAAK,MAAMlB,SAASJ,KAAKC,MAAM,CAAE,IAAIG,MAAMC,IAAI,KAAK,WAAWD,MAAMqB,MAAM,KAAK,QAAQJ,IAAIjB,MAAMU,GAAG,EAAEV,MAAMkB,GAAG;IAChH,KAAK,MAAMI,SAAS1B,KAAKiB,MAAM,CAAEI,IAAIK,MAAMZ,GAAG,EAAEY,MAAMJ,GAAG;IACzD,OAAOL;AACT;AAEA;;;;;CAKC,GACD,OAAO,SAASU,uBAAuB3B,IAA8C;QAGnD4B;IAFhC,MAAMC,OAAO7B,KAAKC,MAAM,CAAC6B,MAAM,CAAC,CAAC1B,QAAUA,MAAMqB,MAAM,KAAK;IAC5D,MAAMG,WAAWC,KAAKC,MAAM,CAAC,CAAC1B,QAA+DA,MAAMC,IAAI,KAAK;IAC5G,MAAM0B,UAAU3C,gBAAgBwC,EAAAA,iBAAAA,SAAShB,IAAI,CAAC,CAACR,QAAUA,MAAM4B,KAAK,KAAK,uBAAzCJ,eAA6CvC,IAAI,KAAIW,KAAKU,KAAK,IAAI,OAAO;IAC1G,MAAMuB,WAAW;QAACF;KAAQ;IAC1B,MAAMZ,OAAO,IAAIC,IAAI;QAACW,QAAQG,WAAW;KAAG;IAC5C,KAAK,MAAMC,WAAWP,SAAU;QAC9B,IAAIO,QAAQH,KAAK,KAAK,GAAG;QACzB,MAAMI,OAAOhD,gBAAgB+C,QAAQ9C,IAAI;QACzC,IAAI,CAAC+C,QAAQjB,KAAKI,GAAG,CAACa,KAAKF,WAAW,KAAK;QAC3Cf,KAAKE,GAAG,CAACe,KAAKF,WAAW;QACzBD,SAAST,IAAI,CAACY;QACd,IAAIH,SAASvC,MAAM,IAAIR,6BAA6B;IACtD;IACA,IAAI+C,SAASvC,MAAM,KAAK,KAAKK,oBAAoB;QAAEW,OAAO;QAAIT,QAAQ4B;IAAK,KAAK,IAAII,SAAST,IAAI,CAAC;IAClG,OAAOS;AACT;AAEA,qEAAqE,GACrE,OAAO,SAASI,mBAAmBrC,IAA8D;IAC/F,MAAMsC,QAAkB,EAAE;IAC1B,IAAItC,KAAKU,KAAK,EAAE4B,MAAMd,IAAI,CAAC,CAAC,OAAO,EAAExB,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKuC,WAAW,EAAED,MAAMd,IAAI,CAAC,CAAC,aAAa,EAAExB,KAAKuC,WAAW,EAAE;IACnE,KAAK,MAAMnC,SAASJ,KAAKC,MAAM,CAAE;QAC/B,MAAMuC,QAAQpC,MAAMqB,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,EAAErB,MAAMqB,MAAM,CAAC,EAAE,CAAC;QACjE,IAAIrB,MAAMC,IAAI,KAAK,WAAWiC,MAAMd,IAAI,CAAC,GAAG,IAAIiB,MAAM,CAACrC,MAAM4B,KAAK,EAAE,CAAC,EAAE5B,MAAMf,IAAI,EAAE;aAC9E,IAAIe,MAAMC,IAAI,KAAK,SAASiC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,QAAQ,EAAEpC,MAAMkB,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAC1F,IAAIlB,MAAMC,IAAI,KAAK,aAAaiC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,EAAE,EAAEpC,MAAMf,IAAI,EAAE;aACpE,IAAIe,MAAMC,IAAI,KAAK,SAASiC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,EAAE,EAAEpC,MAAMf,IAAI,EAAE;aAChEiD,MAAMd,IAAI,CAAC,GAAGgB,QAAQpC,MAAMf,IAAI,EAAE;IACzC;IACA,MAAMA,OAAOiD,MAAMI,IAAI,CAAC;IACxB,OAAOrD,KAAKK,MAAM,GAAGP,qCAAqCE,KAAKM,KAAK,CAAC,GAAGR,sCAAsCE;AAChH;AAEA,iCAAiC,GACjC,OAAO,SAASsD,mBAAmB3C,IAAoB,EAAE4C,EAAU;IACjE,OAAO;QACLA;QACAlC,OAAOH,oBAAoBP;QAC3BX,MAAMgD,mBAAmBrC;QACzBiC,UAAUN,uBAAuB3B;QACjCiB,QAAQC,qBAAqBlB;IAC/B;AACF;AAEA;;;CAGC,GACD,MAAM6C,+BAAoD,IAAIzB,IAAI;IAAC;IAAK;IAAK;CAAI;AAEjF,mHAAmH,GACnH,OAAO,SAAS0B,wBAAwBC,MAA4B,EAAEC,MAAe;IACnF,OAAQD;QACN,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAOC,WAAWC,aAAaJ,6BAA6BtB,GAAG,CAACyB,UAAU,YAAY;QACxF;YACE,OAAO;IACX;AACF"}
|
|
1
|
+
{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-page.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * ONE CRAWLED PAGE, AS AN IMPORT KEEPS IT (AGL-3976).\n *\n * The crawler's page (`SiteImportPage`) holds everything it extracted; an\n * import keeps much less, and keeps it in two shapes:\n *\n * - **Listed**, for the inventory: its address, title, how it was found and\n * how many words and photos it has. No text: an inventory of a whole site\n * rides the job document.\n * - **Read**, for a wave: its text in order, its sections from its headings,\n * and its photos' addresses. The text is kept as the page says it and is\n * fenced as untrusted data only where a prompt shows it; the section names\n * and the title travel outside the fence, so they are neutralized (no\n * fence marker, no hidden character) and cut short. Every block of its\n * main region rides along too (AGL-4083), so a page can be brought in word\n * for word.\n *\n * And from the home page, what the site's header and footer are made of\n * (AGL-4083): its brand as the source draws it (`siteImportHeaderBrand`) and\n * its own facts (`siteImportSiteFacts`), for the header and footer the import\n * builds.\n *=========================================*/\n\nimport { neutralizeUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n AI_SITE_IMPORT_BLOCKED_STATUSES,\n AI_SITE_IMPORT_IMAGES_PER_PAGE,\n AI_SITE_IMPORT_LABEL_MAX_CHARS,\n AI_SITE_IMPORT_MAX_SECTIONS,\n AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS,\n AI_SITE_IMPORT_READ_MAX_CHARS,\n aiSiteImportCapBlocks,\n type AiSiteImportLeftOutReason,\n type AiSiteImportListedPage,\n type AiSiteImportPageRead,\n} from '../model/ai-site-import'\nimport type { SiteImportPage, SiteImportSkipReason } from './site-import-crawl'\nimport type { SiteImportBlock, SiteImportOrganization } from './site-import-html'\nimport { siteImportLogoCandidates, type SiteImportLogoCandidate } from './site-import-images'\n\n/** A line of untrusted text safe to show outside a fence: neutralized, one line, cut short. */\nexport function siteImportLabel(text: string, max = AI_SITE_IMPORT_LABEL_MAX_CHARS): string {\n const line = neutralizeUntrustedText(text).replace(/\\s+/g, ' ').trim()\n return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line\n}\n\nconst wordsIn = (text: string) => (text.match(/[\\p{L}\\p{N}]+/gu) ?? []).length\n\n/** How many words a page says, its title and headings included; a link inside a sentence is counted once. */\nexport function siteImportPageWords(page: Pick<SiteImportPage, 'title' | 'blocks'>): number {\n return page.blocks.reduce(\n (total, block) => total + (block.kind === 'image' || (block.kind === 'link' && block.inline) ? 0 : wordsIn(block.text)),\n 0,\n )\n}\n\n/** Between a page's own name and its site's in a browser title: \"About | Rosie's Bakery\". */\nconst TITLE_SEPARATOR = /\\s+[|\\u2013\\u2014\\u00b7:-]\\s+/\n\n/**\n * A page's name on the new site: \"Home\" for the home page, else its browser\n * title without the old site's name after a separator (\"About | Rosie's\n * Bakery\" is \"About\"), else its first heading.\n */\nexport function siteImportPageTitle(page: Pick<SiteImportPage, 'path' | 'title' | 'blocks'>): string {\n if ((page.path.replace(/\\/+$/, '') || '/') === '/') return 'Home'\n const own = siteImportLabel((page.title ?? '').split(TITLE_SEPARATOR)[0] ?? '')\n if (own.length >= 2) return own\n return siteImportLabel(page.blocks.find((block) => block.kind === 'heading')?.text ?? '')\n}\n\n/** The page as the inventory lists it. */\nexport function siteImportListedPage(page: SiteImportPage): AiSiteImportListedPage {\n return {\n url: page.url,\n path: page.path,\n title: siteImportPageTitle(page),\n foundBy: page.foundBy,\n words: siteImportPageWords(page),\n images: page.images.length,\n }\n}\n\n/** The page's photos, the ones in its body first, each once. */\nexport function siteImportPageImages(page: Pick<SiteImportPage, 'blocks' | 'images'>): Array<{ url: string; alt: string }> {\n const seen = new Set<string>()\n const images: Array<{ url: string; alt: string }> = []\n const add = (url: string, alt: string) => {\n if (!url || seen.has(url) || images.length >= AI_SITE_IMPORT_IMAGES_PER_PAGE) return\n seen.add(url)\n images.push({ url, alt: siteImportLabel(alt, 250) })\n }\n for (const block of page.blocks) if (block.kind === 'image' && block.region === 'main') add(block.url, block.alt)\n for (const image of page.images) add(image.url, image.alt)\n return images\n}\n\n/**\n * The page's sections, from its headings: the first is its opening (its\n * title or first heading), then one for each second-level heading of its\n * body, at most `AI_SITE_IMPORT_MAX_SECTIONS`. A page with no headings below\n * its title but more to say than an opening gets a second section for it.\n */\nexport function siteImportPageSections(page: Pick<SiteImportPage, 'title' | 'blocks'>): string[] {\n const main = page.blocks.filter((block) => block.region === 'main')\n const headings = main.filter((block): block is Extract<typeof block, { kind: 'heading' }> => block.kind === 'heading')\n const opening = siteImportLabel(headings.find((block) => block.level === 1)?.text || page.title || '') || 'Introduction'\n const sections = [opening]\n const seen = new Set([opening.toLowerCase()])\n for (const heading of headings) {\n if (heading.level !== 2) continue\n const name = siteImportLabel(heading.text)\n if (!name || seen.has(name.toLowerCase())) continue\n seen.add(name.toLowerCase())\n sections.push(name)\n if (sections.length >= AI_SITE_IMPORT_MAX_SECTIONS) break\n }\n if (sections.length === 1 && siteImportPageWords({ title: '', blocks: main }) > 80) sections.push('Details')\n return sections\n}\n\n/** The page's text in order, as a prompt shows it inside its fence. */\nexport function siteImportPageText(page: Pick<SiteImportPage, 'title' | 'description' | 'blocks'>): string {\n const lines: string[] = []\n if (page.title) lines.push(`Title: ${page.title}`)\n if (page.description) lines.push(`Description: ${page.description}`)\n for (const block of page.blocks) {\n const where = block.region === 'main' ? '' : `(${block.region}) `\n if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n else if (block.kind === 'image') lines.push(`${where}[photo: ${block.alt || 'no description'}]`)\n else if (block.kind === 'list-item') lines.push(`${where}- ${block.text}`)\n else if (block.kind === 'quote') lines.push(`${where}> ${block.text}`)\n else if (block.kind === 'link') {\n // A link inside a sentence is in its block's words already.\n if (!block.inline) lines.push(`${where}${block.text}`)\n } else lines.push(`${where}${block.text}`)\n }\n const text = lines.join('\\n')\n return text.length > AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS ? text.slice(0, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS) : text\n}\n\n/** The page as a wave reads it: its main region's blocks whole, within the read's budget. */\nexport function siteImportPageRead(page: SiteImportPage, at: string): AiSiteImportPageRead {\n const main = page.blocks.filter((block) => block.region === 'main')\n const capped = aiSiteImportCapBlocks(main, AI_SITE_IMPORT_READ_MAX_CHARS)\n return {\n at,\n title: siteImportPageTitle(page),\n text: siteImportPageText(page),\n sections: siteImportPageSections(page),\n images: siteImportPageImages(page),\n blocks: capped.blocks,\n ...(capped.cut ? { cut: true } : {}),\n words: siteImportPageWords({ title: '', blocks: main }),\n forms: page.forms.kept,\n }\n}\n\n/*------------------------------------------\n * The home page's header and footer (AGL-4083)\n *-----------------------------------------*/\n\n/**\n * How the source's header draws its brand: its logo alone (aglyn.com), its\n * logo with its name beside it, or its name alone. `text` is the name as the\n * header prints it, or the site's own name when it prints nothing at all.\n */\nexport interface SiteImportHeaderBrand {\n kind: 'logo-only' | 'logo-and-name' | 'name-only'\n logo: SiteImportLogoCandidate | null\n /** Whether the header prints the name as words. */\n name: boolean\n text: string\n}\n\n/** The site's own name: its head's, its structured data's, else its home page's title before a separator. */\nexport function siteImportSiteName(home: Pick<SiteImportPage, 'siteName' | 'organization' | 'title'>): string {\n return siteImportLabel(home.siteName || home.organization?.name || (home.title ?? '').split(TITLE_SEPARATOR).pop() || '')\n}\n\n/** Words short enough to be a name, not a sentence. */\nconst looksLikeName = (text: string) => text.length >= 2 && text.length <= AI_SITE_IMPORT_LABEL_MAX_CHARS && text.split(/\\s+/).length <= 8\n\n/**\n * The home page's header brand (AGL-4083), as the source shows it: the link\n * home in its header, the picture inside it, and whether words sit beside\n * the picture, inside the link or next to it. A header with no picture link\n * home is the name alone.\n */\nexport function siteImportHeaderBrand(\n home: Pick<SiteImportPage, 'brand' | 'images' | 'icons' | 'siteName' | 'organization' | 'title' | 'blocks'>,\n): SiteImportHeaderBrand {\n const logo = siteImportLogoCandidates(home).find((candidate) => candidate.source === 'header') ?? null\n const siteName = siteImportSiteName(home)\n const linked = siteImportLabel(home.brand?.text ?? '')\n const beside = home.blocks.find(\n (block) =>\n block.region === 'header' &&\n (block.kind === 'paragraph' || block.kind === 'heading' || block.kind === 'link') &&\n !!siteName &&\n block.text.trim().toLowerCase() === siteName.toLowerCase(),\n )\n const printed = looksLikeName(linked) ? linked : beside && beside.kind !== 'image' ? siteImportLabel(beside.text) : ''\n if (logo && printed) return { kind: 'logo-and-name', logo, name: true, text: printed }\n if (logo) return { kind: 'logo-only', logo, name: false, text: '' }\n return { kind: 'name-only', logo: null, name: true, text: printed || siteName }\n}\n\n/** The facts a site's header and footer carry, as the source writes them. */\nexport interface SiteImportSiteFacts {\n header: SiteImportBlock[]\n footer: SiteImportBlock[]\n organization: SiteImportOrganization | null\n /** Email addresses, from its structured data and its `mailto:` links. */\n emails: string[]\n /** Phone numbers as written, from its structured data and its `tel:` links. */\n phones: string[]\n /** Its profiles on other sites: its structured data's `sameAs` and its links to them. */\n social: string[]\n}\n\nconst SOCIAL_HOSTS =\n /(?:^|\\.)(?:twitter\\.com|x\\.com|facebook\\.com|instagram\\.com|linkedin\\.com|youtube\\.com|tiktok\\.com|github\\.com|pinterest\\.com|threads\\.net|yelp\\.com|tripadvisor\\.com|nextdoor\\.com|vimeo\\.com|bsky\\.app|mastodon\\.social)$/i\n\n/**\n * The home page's own facts (AGL-4083), word for word: its header and footer\n * blocks, its Organization structured data, and the email addresses, phone\n * numbers and profiles they hold. Nothing here is invented or reworded; a\n * fact the page does not state is not here.\n */\nexport function siteImportSiteFacts(home: Pick<SiteImportPage, 'blocks' | 'organization'>): SiteImportSiteFacts {\n const header = home.blocks.filter((block) => block.region === 'header')\n const footer = home.blocks.filter((block) => block.region === 'footer')\n const organization = home.organization ?? null\n const emails: string[] = []\n const phones: string[] = []\n const social: string[] = []\n const add = (list: string[], value: string) => {\n const clean = value.trim()\n if (clean && !list.some((entry) => entry.toLowerCase() === clean.toLowerCase())) list.push(clean)\n }\n if (organization?.email) add(emails, organization.email.toLowerCase())\n if (organization?.telephone) add(phones, organization.telephone)\n for (const url of organization?.sameAs ?? []) add(social, url)\n for (const block of [...header, ...footer]) {\n if (block.kind !== 'link') continue\n if (block.href.startsWith('mailto:')) add(emails, block.href.slice('mailto:'.length))\n else if (block.href.startsWith('tel:')) add(phones, block.text)\n else {\n try {\n if (SOCIAL_HOSTS.test(new URL(block.href).hostname)) add(social, block.href)\n } catch {\n // A link that is not an address is no profile.\n }\n }\n }\n return { header, footer, organization, emails, phones, social }\n}\n\n/** Why the crawler could not read a page, as an import leaves it out; `status` is the answer an `http-error` got. */\nexport function siteImportLeftOutReason(reason: SiteImportSkipReason, status?: number): AiSiteImportLeftOutReason {\n switch (reason) {\n case 'login-form':\n return 'login-form'\n case 'robots':\n return 'robots'\n case 'too-large':\n return 'too-large'\n case 'off-origin-redirect':\n return 'off-site'\n case 'http-error':\n return status !== undefined && AI_SITE_IMPORT_BLOCKED_STATUSES.has(status) ? 'blocked' : 'unreadable'\n case 'not-reached':\n return 'not-reached'\n default:\n return 'unreadable'\n }\n}\n"],"names":["neutralizeUntrustedText","AI_SITE_IMPORT_BLOCKED_STATUSES","AI_SITE_IMPORT_IMAGES_PER_PAGE","AI_SITE_IMPORT_LABEL_MAX_CHARS","AI_SITE_IMPORT_MAX_SECTIONS","AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS","AI_SITE_IMPORT_READ_MAX_CHARS","aiSiteImportCapBlocks","siteImportLogoCandidates","siteImportLabel","text","max","line","replace","trim","length","slice","trimEnd","wordsIn","match","siteImportPageWords","page","blocks","reduce","total","block","kind","inline","TITLE_SEPARATOR","siteImportPageTitle","path","own","title","split","find","siteImportListedPage","url","foundBy","words","images","siteImportPageImages","seen","Set","add","alt","has","push","region","image","siteImportPageSections","headings","main","filter","opening","level","sections","toLowerCase","heading","name","siteImportPageText","lines","description","where","repeat","join","siteImportPageRead","at","capped","cut","forms","kept","siteImportSiteName","home","siteName","organization","pop","looksLikeName","siteImportHeaderBrand","logo","candidate","source","linked","brand","beside","printed","SOCIAL_HOSTS","siteImportSiteFacts","header","footer","emails","phones","social","list","value","clean","some","entry","email","telephone","sameAs","href","startsWith","test","URL","hostname","siteImportLeftOutReason","reason","status","undefined"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;2CAqB2C,GAE3C,SAASA,uBAAuB,QAAQ,0CAAyC;AACjF,SACEC,+BAA+B,EAC/BC,8BAA8B,EAC9BC,8BAA8B,EAC9BC,2BAA2B,EAC3BC,kCAAkC,EAClCC,6BAA6B,EAC7BC,qBAAqB,QAIhB,6BAAyB;AAGhC,SAASC,wBAAwB,QAAsC,0BAAsB;AAE7F,6FAA6F,GAC7F,OAAO,SAASC,gBAAgBC,IAAY,EAAEC,MAAMR,8BAA8B;IAChF,MAAMS,OAAOZ,wBAAwBU,MAAMG,OAAO,CAAC,QAAQ,KAAKC,IAAI;IACpE,OAAOF,KAAKG,MAAM,GAAGJ,MAAM,GAAGC,KAAKI,KAAK,CAAC,GAAGL,MAAM,GAAGM,OAAO,GAAG,CAAC,CAAC,GAAGL;AACtE;AAEA,MAAMM,UAAU,CAACR;QAAkBA;WAAD,EAACA,cAAAA,KAAKS,KAAK,CAAC,ynaAAXT,cAAiC,EAAE,EAAEK,MAAM;;AAE9E,2GAA2G,GAC3G,OAAO,SAASK,oBAAoBC,IAA8C;IAChF,OAAOA,KAAKC,MAAM,CAACC,MAAM,CACvB,CAACC,OAAOC,QAAUD,QAASC,CAAAA,MAAMC,IAAI,KAAK,WAAYD,MAAMC,IAAI,KAAK,UAAUD,MAAME,MAAM,GAAI,IAAIT,QAAQO,MAAMf,IAAI,CAAA,GACrH;AAEJ;AAEA,2FAA2F,GAC3F,MAAMkB,kBAAkB;AAExB;;;;CAIC,GACD,OAAO,SAASC,oBAAoBR,IAAuD;QAE7D,SAACA;QAENA;IAHvB,IAAI,AAACA,CAAAA,KAAKS,IAAI,CAACjB,OAAO,CAAC,QAAQ,OAAO,GAAE,MAAO,KAAK,OAAO;IAC3D,MAAMkB,MAAMtB,iBAAgB,UAAA,EAACY,cAAAA,KAAKW,KAAK,YAAVX,cAAc,IAAIY,KAAK,CAACL,gBAAgB,CAAC,EAAE,YAA5C,UAAgD;IAC5E,IAAIG,IAAIhB,MAAM,IAAI,GAAG,OAAOgB;IAC5B,OAAOtB,yBAAgBY,oBAAAA,KAAKC,MAAM,CAACY,IAAI,CAAC,CAACT,QAAUA,MAAMC,IAAI,KAAK,+BAA3CL,kBAAuDX,IAAI,mBAAI;AACxF;AAEA,wCAAwC,GACxC,OAAO,SAASyB,qBAAqBd,IAAoB;IACvD,OAAO;QACLe,KAAKf,KAAKe,GAAG;QACbN,MAAMT,KAAKS,IAAI;QACfE,OAAOH,oBAAoBR;QAC3BgB,SAAShB,KAAKgB,OAAO;QACrBC,OAAOlB,oBAAoBC;QAC3BkB,QAAQlB,KAAKkB,MAAM,CAACxB,MAAM;IAC5B;AACF;AAEA,8DAA8D,GAC9D,OAAO,SAASyB,qBAAqBnB,IAA+C;IAClF,MAAMoB,OAAO,IAAIC;IACjB,MAAMH,SAA8C,EAAE;IACtD,MAAMI,MAAM,CAACP,KAAaQ;QACxB,IAAI,CAACR,OAAOK,KAAKI,GAAG,CAACT,QAAQG,OAAOxB,MAAM,IAAIb,gCAAgC;QAC9EuC,KAAKE,GAAG,CAACP;QACTG,OAAOO,IAAI,CAAC;YAAEV;YAAKQ,KAAKnC,gBAAgBmC,KAAK;QAAK;IACpD;IACA,KAAK,MAAMnB,SAASJ,KAAKC,MAAM,CAAE,IAAIG,MAAMC,IAAI,KAAK,WAAWD,MAAMsB,MAAM,KAAK,QAAQJ,IAAIlB,MAAMW,GAAG,EAAEX,MAAMmB,GAAG;IAChH,KAAK,MAAMI,SAAS3B,KAAKkB,MAAM,CAAEI,IAAIK,MAAMZ,GAAG,EAAEY,MAAMJ,GAAG;IACzD,OAAOL;AACT;AAEA;;;;;CAKC,GACD,OAAO,SAASU,uBAAuB5B,IAA8C;QAGnD6B;IAFhC,MAAMC,OAAO9B,KAAKC,MAAM,CAAC8B,MAAM,CAAC,CAAC3B,QAAUA,MAAMsB,MAAM,KAAK;IAC5D,MAAMG,WAAWC,KAAKC,MAAM,CAAC,CAAC3B,QAA+DA,MAAMC,IAAI,KAAK;IAC5G,MAAM2B,UAAU5C,gBAAgByC,EAAAA,iBAAAA,SAAShB,IAAI,CAAC,CAACT,QAAUA,MAAM6B,KAAK,KAAK,uBAAzCJ,eAA6CxC,IAAI,KAAIW,KAAKW,KAAK,IAAI,OAAO;IAC1G,MAAMuB,WAAW;QAACF;KAAQ;IAC1B,MAAMZ,OAAO,IAAIC,IAAI;QAACW,QAAQG,WAAW;KAAG;IAC5C,KAAK,MAAMC,WAAWP,SAAU;QAC9B,IAAIO,QAAQH,KAAK,KAAK,GAAG;QACzB,MAAMI,OAAOjD,gBAAgBgD,QAAQ/C,IAAI;QACzC,IAAI,CAACgD,QAAQjB,KAAKI,GAAG,CAACa,KAAKF,WAAW,KAAK;QAC3Cf,KAAKE,GAAG,CAACe,KAAKF,WAAW;QACzBD,SAAST,IAAI,CAACY;QACd,IAAIH,SAASxC,MAAM,IAAIX,6BAA6B;IACtD;IACA,IAAImD,SAASxC,MAAM,KAAK,KAAKK,oBAAoB;QAAEY,OAAO;QAAIV,QAAQ6B;IAAK,KAAK,IAAII,SAAST,IAAI,CAAC;IAClG,OAAOS;AACT;AAEA,qEAAqE,GACrE,OAAO,SAASI,mBAAmBtC,IAA8D;IAC/F,MAAMuC,QAAkB,EAAE;IAC1B,IAAIvC,KAAKW,KAAK,EAAE4B,MAAMd,IAAI,CAAC,CAAC,OAAO,EAAEzB,KAAKW,KAAK,EAAE;IACjD,IAAIX,KAAKwC,WAAW,EAAED,MAAMd,IAAI,CAAC,CAAC,aAAa,EAAEzB,KAAKwC,WAAW,EAAE;IACnE,KAAK,MAAMpC,SAASJ,KAAKC,MAAM,CAAE;QAC/B,MAAMwC,QAAQrC,MAAMsB,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,EAAEtB,MAAMsB,MAAM,CAAC,EAAE,CAAC;QACjE,IAAItB,MAAMC,IAAI,KAAK,WAAWkC,MAAMd,IAAI,CAAC,GAAG,IAAIiB,MAAM,CAACtC,MAAM6B,KAAK,EAAE,CAAC,EAAE7B,MAAMf,IAAI,EAAE;aAC9E,IAAIe,MAAMC,IAAI,KAAK,SAASkC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,QAAQ,EAAErC,MAAMmB,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAC1F,IAAInB,MAAMC,IAAI,KAAK,aAAakC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,EAAE,EAAErC,MAAMf,IAAI,EAAE;aACpE,IAAIe,MAAMC,IAAI,KAAK,SAASkC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,EAAE,EAAErC,MAAMf,IAAI,EAAE;aAChE,IAAIe,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAME,MAAM,EAAEiC,MAAMd,IAAI,CAAC,GAAGgB,QAAQrC,MAAMf,IAAI,EAAE;QACvD,OAAOkD,MAAMd,IAAI,CAAC,GAAGgB,QAAQrC,MAAMf,IAAI,EAAE;IAC3C;IACA,MAAMA,OAAOkD,MAAMI,IAAI,CAAC;IACxB,OAAOtD,KAAKK,MAAM,GAAGV,qCAAqCK,KAAKM,KAAK,CAAC,GAAGX,sCAAsCK;AAChH;AAEA,2FAA2F,GAC3F,OAAO,SAASuD,mBAAmB5C,IAAoB,EAAE6C,EAAU;IACjE,MAAMf,OAAO9B,KAAKC,MAAM,CAAC8B,MAAM,CAAC,CAAC3B,QAAUA,MAAMsB,MAAM,KAAK;IAC5D,MAAMoB,SAAS5D,sBAAsB4C,MAAM7C;IAC3C,OAAO;QACL4D;QACAlC,OAAOH,oBAAoBR;QAC3BX,MAAMiD,mBAAmBtC;QACzBkC,UAAUN,uBAAuB5B;QACjCkB,QAAQC,qBAAqBnB;QAC7BC,QAAQ6C,OAAO7C,MAAM;OACjB6C,OAAOC,GAAG,GAAG;QAAEA,KAAK;IAAK,IAAI,CAAC;QAClC9B,OAAOlB,oBAAoB;YAAEY,OAAO;YAAIV,QAAQ6B;QAAK;QACrDkB,OAAOhD,KAAKgD,KAAK,CAACC,IAAI;;AAE1B;AAmBA,2GAA2G,GAC3G,OAAO,SAASC,mBAAmBC,IAAiE;QAC9BA;QAA5BA;IAAxC,OAAO/D,gBAAgB+D,KAAKC,QAAQ,MAAID,qBAAAA,KAAKE,YAAY,qBAAjBF,mBAAmBd,IAAI,KAAI,EAACc,cAAAA,KAAKxC,KAAK,YAAVwC,cAAc,IAAIvC,KAAK,CAACL,iBAAiB+C,GAAG,MAAM;AACxH;AAEA,qDAAqD,GACrD,MAAMC,gBAAgB,CAAClE,OAAiBA,KAAKK,MAAM,IAAI,KAAKL,KAAKK,MAAM,IAAIZ,kCAAkCO,KAAKuB,KAAK,CAAC,OAAOlB,MAAM,IAAI;AAEzI;;;;;CAKC,GACD,OAAO,SAAS8D,sBACdL,IAA2G;QAE9FhE;QAEkBgE;IAF/B,MAAMM,QAAOtE,iCAAAA,yBAAyBgE,MAAMtC,IAAI,CAAC,CAAC6C,YAAcA,UAAUC,MAAM,KAAK,qBAAxExE,iCAAqF;IAClG,MAAMiE,WAAWF,mBAAmBC;IACpC,MAAMS,SAASxE,yBAAgB+D,cAAAA,KAAKU,KAAK,qBAAVV,YAAY9D,IAAI,mBAAI;IACnD,MAAMyE,SAASX,KAAKlD,MAAM,CAACY,IAAI,CAC7B,CAACT,QACCA,MAAMsB,MAAM,KAAK,YAChBtB,CAAAA,MAAMC,IAAI,KAAK,eAAeD,MAAMC,IAAI,KAAK,aAAaD,MAAMC,IAAI,KAAK,MAAK,KAC/E,CAAC,CAAC+C,YACFhD,MAAMf,IAAI,CAACI,IAAI,GAAG0C,WAAW,OAAOiB,SAASjB,WAAW;IAE5D,MAAM4B,UAAUR,cAAcK,UAAUA,SAASE,UAAUA,OAAOzD,IAAI,KAAK,UAAUjB,gBAAgB0E,OAAOzE,IAAI,IAAI;IACpH,IAAIoE,QAAQM,SAAS,OAAO;QAAE1D,MAAM;QAAiBoD;QAAMpB,MAAM;QAAMhD,MAAM0E;IAAQ;IACrF,IAAIN,MAAM,OAAO;QAAEpD,MAAM;QAAaoD;QAAMpB,MAAM;QAAOhD,MAAM;IAAG;IAClE,OAAO;QAAEgB,MAAM;QAAaoD,MAAM;QAAMpB,MAAM;QAAMhD,MAAM0E,WAAWX;IAAS;AAChF;AAeA,MAAMY,eACJ;AAEF;;;;;CAKC,GACD,OAAO,SAASC,oBAAoBd,IAAqD;QAGlEA;IAFrB,MAAMe,SAASf,KAAKlD,MAAM,CAAC8B,MAAM,CAAC,CAAC3B,QAAUA,MAAMsB,MAAM,KAAK;IAC9D,MAAMyC,SAAShB,KAAKlD,MAAM,CAAC8B,MAAM,CAAC,CAAC3B,QAAUA,MAAMsB,MAAM,KAAK;IAC9D,MAAM2B,gBAAeF,qBAAAA,KAAKE,YAAY,YAAjBF,qBAAqB;IAC1C,MAAMiB,SAAmB,EAAE;IAC3B,MAAMC,SAAmB,EAAE;IAC3B,MAAMC,SAAmB,EAAE;IAC3B,MAAMhD,MAAM,CAACiD,MAAgBC;QAC3B,MAAMC,QAAQD,MAAM/E,IAAI;QACxB,IAAIgF,SAAS,CAACF,KAAKG,IAAI,CAAC,CAACC,QAAUA,MAAMxC,WAAW,OAAOsC,MAAMtC,WAAW,KAAKoC,KAAK9C,IAAI,CAACgD;IAC7F;IACA,IAAIpB,gCAAAA,aAAcuB,KAAK,EAAEtD,IAAI8C,QAAQf,aAAauB,KAAK,CAACzC,WAAW;IACnE,IAAIkB,gCAAAA,aAAcwB,SAAS,EAAEvD,IAAI+C,QAAQhB,aAAawB,SAAS;IAC/D,KAAK,MAAM9D,eAAOsC,gCAAAA,aAAcyB,MAAM,mBAAI,EAAE,CAAExD,IAAIgD,QAAQvD;IAC1D,KAAK,MAAMX,SAAS;WAAI8D;WAAWC;KAAO,CAAE;QAC1C,IAAI/D,MAAMC,IAAI,KAAK,QAAQ;QAC3B,IAAID,MAAM2E,IAAI,CAACC,UAAU,CAAC,YAAY1D,IAAI8C,QAAQhE,MAAM2E,IAAI,CAACpF,KAAK,CAAC,UAAUD,MAAM;aAC9E,IAAIU,MAAM2E,IAAI,CAACC,UAAU,CAAC,SAAS1D,IAAI+C,QAAQjE,MAAMf,IAAI;aACzD;YACH,IAAI;gBACF,IAAI2E,aAAaiB,IAAI,CAAC,IAAIC,IAAI9E,MAAM2E,IAAI,EAAEI,QAAQ,GAAG7D,IAAIgD,QAAQlE,MAAM2E,IAAI;YAC7E,EAAE,eAAM;YACN,+CAA+C;YACjD;QACF;IACF;IACA,OAAO;QAAEb;QAAQC;QAAQd;QAAce;QAAQC;QAAQC;IAAO;AAChE;AAEA,mHAAmH,GACnH,OAAO,SAASc,wBAAwBC,MAA4B,EAAEC,MAAe;IACnF,OAAQD;QACN,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAOC,WAAWC,aAAa3G,gCAAgC4C,GAAG,CAAC8D,UAAU,YAAY;QAC3F,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF"}
|