@aglyn/plugins-ai 1.0.0-beta.291 → 1.0.0-beta.293

package/src/lib/model/ai-site-import.js

@@ -54,8 +54,10 @@ import { aiCreditRangeOrdered, aiCreditRangeText, AI_MEASURED_IMPORT_CREDITS } f
 /**
  * The most pages one import reads, whatever the plan allows: an unlimited
  * plan is still one crawl's worth, and the crawl's own time budget may stop
- * sooner (the inventory then says so).
- */ export const AI_SITE_IMPORT_MAX_PAGES = 200;
+ * sooner (the inventory then says so). Only pages count: a file a link
+ * points at is never fetched as one (AGL-4106), so aglyn.com's 209 pages fit
+ * with room where 136 press-kit files once took 136 of 200.
+ */ export const AI_SITE_IMPORT_MAX_PAGES = 400;
 /** The fewest pages under one address prefix that make it a collection. */ export const AI_SITE_IMPORT_COLLECTION_MIN = 3;
 /** The most photos brought in from one page. */ export const AI_SITE_IMPORT_IMAGES_PER_PAGE = 8;
 /**
@@ -66,15 +68,19 @@ import { aiCreditRangeOrdered, aiCreditRangeText, AI_MEASURED_IMPORT_CREDITS } f
 /** The most sections one imported page is planned with: one for every second-level heading (AGL-4083). */ export const AI_SITE_IMPORT_MAX_SECTIONS = 24;
 /** The most blocks of one page a wave's read keeps. */ export const AI_SITE_IMPORT_READ_MAX_BLOCKS = 2000;
 /** The most characters of one page's blocks a wave's read keeps: a long policy is 40,000. */ export const AI_SITE_IMPORT_READ_MAX_CHARS = 120000;
+/** The most characters one page's layout keeps on the job document (AGL-4106): its icons' drawings go first. */ export const AI_SITE_IMPORT_LAYOUT_MAX_CHARS = 48000;
 /**
- * The most characters of blocks the reads of one import hold at once: they
+ * The most characters of blocks (and their layouts, AGL-4106) the reads of one import hold at once: they
  * ride the job document, which Firestore caps at 1 MiB, beside the
  * inventory and the ledger. A wave's later page past it keeps what fits and
  * says it was cut.
  */ export const AI_SITE_IMPORT_WAVE_READ_MAX_CHARS = 400000;
 /**
- * A page with more words of its own than this is copied as it is, never
- * arranged (AGL-4083): a policy, an article, a long story.
+ * A collection's page longer than this is prose: a policy, an article, a
+ * long story (AGL-4083). A collection's page is copied as it is whatever its
+ * length, and since AGL-4106 a main page is arranged whatever its length:
+ * `/product` at 1,072 words is laid out as its source lays it out, not copied
+ * as one column.
  */ export const AI_SITE_IMPORT_PROSE_WORDS = 900;
 /** The answers bot protection and firewalls turn a crawler away with: asked to sign in, refused, or told to slow down. */ export const AI_SITE_IMPORT_BLOCKED_STATUSES = new Set([
     401,
@@ -97,7 +103,8 @@ export const AI_SITE_IMPORT_LEFT_OUT_REASONS = [
     'blocked',
     'too-large',
     'off-site',
-    'not-reached'
+    'not-reached',
+    'file'
 ];
 /** The reasons that mean the page was there and AglynBot could not read it: the "could not be read" of what was found. */ export const AI_SITE_IMPORT_UNREAD_REASONS = [
     'unreadable',
@@ -105,6 +112,166 @@ export const AI_SITE_IMPORT_LEFT_OUT_REASONS = [
     'too-large',
     'not-reached'
 ];
+export const SITE_IMPORT_GROUP_KINDS = [
+    'cards',
+    'steps',
+    'stats',
+    'faq',
+    'quotes',
+    'logos',
+    'pricing',
+    'list',
+    'text'
+];
+/** Every line a layout's nodes place, in order. */ export function siteImportLayoutLines(nodes) {
+    const out = [];
+    const visit = (list)=>{
+        for (const node of list){
+            switch(node.kind){
+                case 'line':
+                case 'picture':
+                    out.push(node.line);
+                    break;
+                case 'row':
+                    for (const cell of node.cells)visit(cell.nodes);
+                    break;
+                case 'group':
+                    for (const item of node.items)out.push(...item.lines);
+                    break;
+                case 'table':
+                    for (const row of node.rows)for (const cell of row.cells)out.push(...cell.lines);
+                    break;
+                case 'tabs':
+                    for (const tab of node.tabs){
+                        if (tab.label !== null) out.push(tab.label);
+                        visit(tab.panel);
+                    }
+                    break;
+                default:
+                    break;
+            }
+        }
+    };
+    visit(nodes);
+    return out;
+}
+/** How many characters a layout weighs on the job document. */ export function siteImportLayoutChars(layout) {
+    return layout ? JSON.stringify(layout).length : 0;
+}
+/**
+ * A layout over only the first `count` lines (a read the wave's budget cut
+ * short): references past the cut are dropped, and what is left empty goes.
+ */ export function siteImportLayoutWithin(layout, count) {
+    const keep = (ref)=>ref >= 0 && ref < count;
+    const sections = layout.sections.map((section)=>_extends({}, section, {
+            nodes: siteImportFilterNodes(section.nodes, keep),
+            lines: section.lines.filter(keep)
+        })).filter((section)=>section.lines.length);
+    return _extends({}, layout, {
+        sections
+    });
+}
+/** Nodes with only the lines `keep` keeps; what is left empty goes. */ export function siteImportFilterNodes(list, keep) {
+    const nodes = (list)=>list.flatMap((node)=>{
+            switch(node.kind){
+                case 'line':
+                case 'picture':
+                    return keep(node.line) ? [
+                        node
+                    ] : [];
+                case 'icon':
+                case 'form':
+                    return [
+                        node
+                    ];
+                case 'row':
+                    {
+                        const cells = node.cells.map((cell)=>({
+                                nodes: nodes(cell.nodes)
+                            }));
+                        return cells.some((cell)=>cell.nodes.length) ? [
+                            _extends({}, node, {
+                                cells
+                            })
+                        ] : [];
+                    }
+                case 'group':
+                    {
+                        const items = node.items.map((item)=>trimItem(item, keep)).filter((item)=>item.lines.length || item.icon !== undefined);
+                        return items.length ? [
+                            _extends({}, node, {
+                                items
+                            })
+                        ] : [];
+                    }
+                case 'table':
+                    {
+                        const rows = node.rows.map((row)=>({
+                                cells: row.cells.map((cell)=>({
+                                        lines: cell.lines.filter(keep)
+                                    }))
+                            })).filter((row)=>row.cells.some((cell)=>cell.lines.length));
+                        return rows.length ? [
+                            _extends({}, node, {
+                                rows
+                            })
+                        ] : [];
+                    }
+                case 'tabs':
+                    {
+                        const tabs = node.tabs.map((tab)=>_extends({}, tab, {
+                                label: tab.label !== null && keep(tab.label) ? tab.label : null,
+                                panel: nodes(tab.panel)
+                            }));
+                        return tabs.some((tab)=>tab.label !== null || tab.panel.length) ? [
+                            _extends({}, node, {
+                                tabs
+                            })
+                        ] : [];
+                    }
+                default:
+                    return [];
+            }
+        });
+    return nodes(list);
+}
+function trimItem(item, keep) {
+    const out = _extends({}, item, {
+        lines: item.lines.filter(keep)
+    });
+    for (const key of [
+        'title',
+        'figure',
+        'image'
+    ])if (out[key] !== undefined && !keep(out[key])) delete out[key];
+    for (const key of [
+        'text',
+        'list',
+        'link'
+    ]){
+        var _out_key;
+        const list = (_out_key = out[key]) == null ? void 0 : _out_key.filter(keep);
+        if (list == null ? void 0 : list.length) out[key] = list;
+        else delete out[key];
+    }
+    return out;
+}
+/** A layout with its icons' drawings dropped, for a page that must be lighter. */ export function siteImportLayoutWithoutDrawings(layout) {
+    return _extends({}, layout, {
+        icons: layout.icons.map((icon)=>icon.svg ? {
+                viewBox: icon.viewBox
+            } : icon)
+    });
+}
+/**
+ * Whether the site's header and footer are built from the source's own menu
+ * (AGL-4108), with no model: the home page's menu was read. Then the frame
+ * costs nothing and is quoted at nothing.
+ */ export function aiSiteImportFrameIsRead(state) {
+    var _state_home;
+    const nav = state == null ? void 0 : (_state_home = state.home) == null ? void 0 : _state_home.nav;
+    return !!nav && (nav.items.length > 0 || nav.ctas.length > 0);
+}
 const pathOfUrl = (url)=>{
     try {
         return new URL(url).pathname.replace(/\/+$/, '') || '/';
@@ -395,7 +562,7 @@ const segmentsOf = (path)=>String(path != null ? path : '').split('/').filter(Bo
  * The site's inventory cut into waves, every page left out said with its
  * reason. Pure: the step reads the site, this decides what comes in.
  */ export function aiSiteImportInventory(input) {
-    var _input_wavePages, _leftOutCount_unreadable, _input_verdict, _input_reviewId;
+    var _input_wavePages, _leftOutCount_unreadable, _input_verdict;
     var _waves_;
     const wavePages = Math.max(1, Math.floor((_input_wavePages = input.wavePages) != null ? _input_wavePages : AI_SITE_IMPORT_WAVE_PAGES));
     const limit = input.pageLimit === null || !Number.isFinite(input.pageLimit) ? null : Math.max(0, Math.floor(input.pageLimit));
@@ -427,7 +594,6 @@ const segmentsOf = (path)=>String(path != null ? path : '').split('/').filter(Bo
     // Home first: a site read without its home page plans nothing at all.
     const verdict = (_input_verdict = input.verdict) != null ? _input_verdict : inventoryVerdict(input);
     if (aiSiteImportReadStops(verdict)) {
-        var _input_reviewId1;
         return {
             domain: input.domain,
             siteName: clipLabel(input.siteName) || input.domain,
@@ -443,7 +609,6 @@ const segmentsOf = (path)=>String(path != null ? path : '').split('/').filter(Bo
             leftOut,
             leftOutCount,
             frame: null,
-            reviewId: (_input_reviewId1 = input.reviewId) != null ? _input_reviewId1 : null,
             verdict
         };
     }
@@ -529,7 +694,6 @@ const segmentsOf = (path)=>String(path != null ? path : '').split('/').filter(Bo
         frame: ((_waves_ = waves[0]) == null ? void 0 : _waves_.frame) ? {
             draftId: input.mintId()
         } : null,
-        reviewId: (_input_reviewId = input.reviewId) != null ? _input_reviewId : null,
         verdict
     };
 }
@@ -665,8 +829,21 @@ const withWave = (state, index, patch)=>_extends({}, state, {
 }
 const readChars = (read)=>{
     var _ref;
-    return ((_ref = read == null ? void 0 : read.blocks) != null ? _ref : []).reduce((total, block)=>total + aiSiteImportBlockChars(block), 0);
+    return ((_ref = read == null ? void 0 : read.blocks) != null ? _ref : []).reduce((total, block)=>total + aiSiteImportBlockChars(block), 0) + siteImportLayoutChars(read == null ? void 0 : read.layout);
 };
+/**
+ * A page's layout in the room the wave has left after its blocks (AGL-4106):
+ * cut to the blocks kept, its drawings dropped when it would not fit with
+ * them, and left off when it would not fit at all (the page is then laid out
+ * from its lines alone).
+ */ function layoutWithin(layout, blocks, left) {
+    if (!layout) return null;
+    const room = Math.min(left, AI_SITE_IMPORT_LAYOUT_MAX_CHARS);
+    const cut = siteImportLayoutWithin(layout, blocks);
+    if (siteImportLayoutChars(cut) <= room) return cut;
+    const light = siteImportLayoutWithoutDrawings(cut);
+    return siteImportLayoutChars(light) <= room ? light : null;
+}
 /**
  * More of a running wave's pages, read now. Their blocks share
  * `AI_SITE_IMPORT_WAVE_READ_MAX_CHARS` with every read still held: a page
@@ -693,6 +870,13 @@ const readChars = (read)=>{
             }, capped.cut || read.cut ? {
                 cut: true
             } : {});
+            if (read.layout) {
+                const layout = layoutWithin(read.layout, capped.blocks.length, Math.max(0, AI_SITE_IMPORT_WAVE_READ_MAX_CHARS - held));
+                held += siteImportLayoutChars(layout);
+                kept = _extends({}, kept, {
+                    layout
+                });
+            }
         }
         next = withPage(next, slot, {
             read: kept,
@@ -713,7 +897,8 @@ const readChars = (read)=>{
     return withPage(state, slot, {
         read: _extends({}, page.read, {
             text: '',
-            blocks: []
+            blocks: [],
+            layout: null
         })
     });
 }
@@ -730,6 +915,26 @@ const readChars = (read)=>{
     for (const slot of (_ref = (_state_waves_find = state.waves.find((wave)=>wave.index === index)) == null ? void 0 : _state_waves_find.slots) != null ? _ref : [])next = aiSiteImportPageSettled(next, slot);
     return next;
 }
+/**
+ * The state of an import a person canceled (AGL-4107): every wave settled. A
+ * wave that ran, or was waiting to, is stopped; one already in, for review or
+ * done, is in. On 10/29 a canceled import still showed its second wave as
+ * running, with nothing running it.
+ */ export function aiSiteImportCanceled(state, at) {
+    return _extends({}, state, {
+        autoContinue: false,
+        waves: state.waves.map((wave)=>{
+            if (wave.status === 'running' || wave.status === 'waiting') return _extends({}, wave, {
+                status: 'stopped',
+                finishedAt: wave.status === 'running' ? at : wave.finishedAt
+            });
+            if (wave.status === 'review') return _extends({}, wave, {
+                status: 'done'
+            });
+            return wave;
+        })
+    });
+}
 /** The state as the console reads it: every page's read text left off. */ export function aiSiteImportWire(state) {
     if (!state) return null;
     return _extends({}, state, {
@@ -738,7 +943,8 @@ const readChars = (read)=>{
                     text: '',
                     images: [],
                     blocks: [],
-                    forms: []
+                    forms: [],
+                    layout: null
                 })
             }) : page)
     });
@@ -747,11 +953,11 @@ const readChars = (read)=>{
  * Credits
  *-----------------------------------------*/ /**
  * How a page is built (AGL-4083), decided by code: every page of a collection
- * (a blog, the legal pages, news) and any page longer than
- * `AI_SITE_IMPORT_PROSE_WORDS` is copied as it is; the main pages are
- * arranged by Aglyn AI, their words kept as written.
+ * (a blog, the legal pages, news) is copied as it is; every main page is
+ * arranged by Aglyn AI whatever its length (AGL-4106), its words kept as
+ * written. Both are laid out from the source's own layout.
  */ export function aiSiteImportPageBuild(page) {
-    return page.group !== 'main' || page.words > AI_SITE_IMPORT_PROSE_WORDS ? 'copied' : 'arranged';
+    return page.group !== 'main' ? 'copied' : 'arranged';
 }
 const whole = (value)=>Math.max(0, Math.floor(Number.isFinite(value) ? value : 0));
 /**
@@ -773,11 +979,12 @@ const whole = (value)=>Math.max(0, Math.floor(Number.isFinite(value) ? value : 0
 }
 /** What a wave still owes, by how its pages are built, and whether its header and footer are owed. */ export function aiSiteImportWaveOwed(state, wave, items) {
     const owed = (slot)=>items ? aiSiteImportItemOpen(items, slot) : true;
+    // A header and footer drawn from the source's own menu ask no model (AGL-4108), so are owed nothing.
     const counts = {
         pages: 0,
         arranged: 0,
         copied: 0,
-        frame: wave.frame && !!state.frame && owed(AI_SITE_IMPORT_FRAME_SLOT)
+        frame: wave.frame && !!state.frame && !aiSiteImportFrameIsRead(state) && owed(AI_SITE_IMPORT_FRAME_SLOT)
     };
     for (const slot of wave.slots){
         const page = state.pages.find((entry)=>entry.slot === slot);
@@ -834,7 +1041,8 @@ const whole = (value)=>Math.max(0, Math.floor(Number.isFinite(value) ? value : 0
     blocked: 'The site’s bot protection or firewall turned AglynBot away. Allow AglynBot in your site’s bot or firewall settings, then bring the site in again.',
     'too-large': 'Larger than a page can be to bring in.',
     'off-site': 'It sends visitors to another website.',
-    'not-reached': 'Not reached: the read stopped before it got to this page.'
+    'not-reached': 'Not reached: the read stopped before it got to this page.',
+    file: 'A file, not a page.'
 };
 /** "3 pages" / "1 page". */ export function aiSiteImportPagesText(count) {
     return `${count.toLocaleString('en-US')} ${count === 1 ? 'page' : 'pages'}`;