@aglyn/plugins-ai 1.0.0-beta.288 → 1.0.0-beta.290

package/src/lib/model/ai-site-import.d.ts

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import type { SiteImportBlock, SiteImportForm } from '../site-import/site-import-html';
 import { type AiCreditRange } from './ai-credit-estimate';
 import type { AiJobItemLedger } from './ai-jobs.types';
 /** The job kind. */
@@ -32,10 +33,32 @@ export declare const AI_SITE_IMPORT_MAX_PAGES = 200;
 export declare const AI_SITE_IMPORT_COLLECTION_MIN = 3;
 /** The most photos brought in from one page. */
 export declare const AI_SITE_IMPORT_IMAGES_PER_PAGE = 8;
-/** The most characters of one page's text a wave keeps for its model answer. */
+/**
+ * The most characters of one page's text a wave keeps for the page step's
+ * prompt, which reads no more than this; every word is kept on the read's
+ * `blocks` (AGL-4083).
+ */
 export declare const AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS = 6000;
-/** The most sections one imported page is planned with. */
-export declare const AI_SITE_IMPORT_MAX_SECTIONS = 6;
+/** The most sections one imported page is planned with: one for every second-level heading (AGL-4083). */
+export declare const AI_SITE_IMPORT_MAX_SECTIONS = 24;
+/** The most blocks of one page a wave's read keeps. */
+export declare const AI_SITE_IMPORT_READ_MAX_BLOCKS = 2000;
+/** The most characters of one page's blocks a wave's read keeps: a long policy is 40,000. */
+export declare const AI_SITE_IMPORT_READ_MAX_CHARS = 120000;
+/**
+ * The most characters of blocks the reads of one import hold at once: they
+ * ride the job document, which Firestore caps at 1 MiB, beside the
+ * inventory and the ledger. A wave's later page past it keeps what fits and
+ * says it was cut.
+ */
+export declare const AI_SITE_IMPORT_WAVE_READ_MAX_CHARS = 400000;
+/**
+ * A page with more words of its own than this is copied as it is, never
+ * arranged (AGL-4083): a policy, an article, a long story.
+ */
+export declare const AI_SITE_IMPORT_PROSE_WORDS = 900;
+/** The answers bot protection and firewalls turn a crawler away with: asked to sign in, refused, or told to slow down. */
+export declare const AI_SITE_IMPORT_BLOCKED_STATUSES: ReadonlySet<number>;
 /** The most characters a section's name, a page's title or a label keeps. */
 export declare const AI_SITE_IMPORT_LABEL_MAX_CHARS = 60;
 /** A page with fewer words than this and no photo has nothing to bring in. */
@@ -44,6 +67,8 @@ export declare const AI_SITE_IMPORT_MIN_WORDS = 5;
 export declare const AI_SITE_IMPORT_FRAME_SLOT = "frame";
 /** The ceiling one imported page reserves: its answer pass and its listing pass, at 50 each. */
 export declare const AI_SITE_IMPORT_PAGE_CEILING_CREDITS = 100;
+/** The ceiling a page copied as it is reserves: its listing pass alone. */
+export declare const AI_SITE_IMPORT_COPIED_CEILING_CREDITS = 50;
 /** The ceiling the header and footer reserve: one answer at 50. */
 export declare const AI_SITE_IMPORT_FRAME_CEILING_CREDITS = 50;
 /** The most left-out pages listed by address; the rest are counted. */
@@ -83,8 +108,12 @@ export type AiSiteImportLeftOutReason =
 /** Over the size one page may be. */
  | 'too-large'
 /** It sends visitors to another website. */
- | 'off-site';
+ | 'off-site'
+/** The read stopped (its clock, its bytes, its page count) before it got to it (AGL-4083). */
+ | 'not-reached';
 export declare const AI_SITE_IMPORT_LEFT_OUT_REASONS: readonly AiSiteImportLeftOutReason[];
+/** The reasons that mean the page was there and AglynBot could not read it: the "could not be read" of what was found. */
+export declare const AI_SITE_IMPORT_UNREAD_REASONS: readonly AiSiteImportLeftOutReason[];
 export interface AiSiteImportLeftOut {
     url: string;
     path: string;
@@ -98,6 +127,8 @@ export interface AiSiteImportPhoto {
     width?: number;
     height?: number;
     alt?: string;
+    /** The address on the old site it was read from, where a page that is copied places it (AGL-4084). */
+    from?: string;
 }
 /** What a wave's read found on one page, for its model answer. Never on the wire. */
 export interface AiSiteImportPageRead {
@@ -112,7 +143,23 @@ export interface AiSiteImportPageRead {
         url: string;
         alt: string;
     }>;
+    /**
+     * Every block of the page's main region, in order (AGL-4083): its words as
+     * written, its headings, lists, links and pictures, within
+     * `AI_SITE_IMPORT_READ_MAX_BLOCKS` and `AI_SITE_IMPORT_READ_MAX_CHARS`.
+     * Untrusted data: fenced wherever a prompt reads it. Emptied once the page
+     * is in; absent on a read made before it was kept.
+     */
+    blocks?: SiteImportBlock[];
+    /** Set when `blocks` had to stop short of the page, at its budget or the wave's. */
+    cut?: boolean;
+    /** The words of the page's main region, as the old page has them. */
+    words?: number;
+    /** The forms the page asks visitors to fill in, by their fields. */
+    forms?: SiteImportForm[];
 }
+/** How a page is built from its old page, as the step chose: its words copied as they are, or arranged by Aglyn AI. */
+export type AiSiteImportBuild = 'copied' | 'arranged';
 export interface AiSiteImportPage {
     /** The item ledger slot: `p0`, `p1`, … */
     slot: string;
@@ -131,6 +178,12 @@ export interface AiSiteImportPage {
     read?: AiSiteImportPageRead | null;
     /** Set when the wave's read found the page cannot be brought in after all. */
     refused?: AiSiteImportLeftOutReason | null;
+    /** How the step builds the page (`aiSiteImportPageBuild`); absent until it chooses, and the page is priced as an AI page. */
+    build?: AiSiteImportBuild | null;
+    /** What came through once the page is in: its words on the new page. */
+    brought?: {
+        words: number;
+    } | null;
     /** Filled once the page's photos are in the library. */
     photos?: {
         brought: number;
@@ -184,7 +237,139 @@ export interface AiSiteImportState {
     } | null;
     /** The Trust & Safety row this import is logged under. */
     reviewId: string | null;
+    /**
+     * "Start every remaining wave" (AGL-4083): each wave closes and the next
+     * starts with no review between them; the last wave's close is the stop.
+     * Off unless the person chose it.
+     */
+    autoContinue?: boolean;
+    /**
+     * The wave "Start every remaining wave" was chosen at (AGL-4084): the one
+     * start the person made, which the day's limit counts; the waves it starts
+     * after it are the same start.
+     */
+    autoContinueFrom?: number | null;
+    /** What the read found of the site as a whole; a stopping verdict plans nothing. */
+    verdict?: AiSiteImportReadVerdict | null;
+    /** What the home page says of the site as a whole (AGL-4084): its look, its header and footer, its listing. */
+    home?: AiSiteImportHome | null;
+    /** The look pass, once it ran in the first wave (AGL-4084). */
+    look?: AiSiteImportLook | null;
+}
+/**
+ * What the home page says of the site as a whole (AGL-4084), kept from the
+ * read: the site's listing in its own words, its header's brand as the
+ * source draws it, and the words its header and footer carry. Untrusted
+ * data from the owner's site, fenced wherever a prompt reads it.
+ */
+export interface AiSiteImportHome {
+    url: string;
+    title: string;
+    description: string;
+    ogImage: string | null;
+    siteName: string;
+    brand: {
+        kind: 'logo-only' | 'logo-and-name' | 'name-only';
+        /** The name as the header prints it, where it prints one. */
+        text: string;
+        logo: {
+            url: string;
+            alt: string;
+            type: 'svg' | 'raster' | null;
+        } | null;
+    };
+    facts: {
+        /** The header's and footer's lines, as written, in order. */
+        lines: string[];
+        emails: string[];
+        phones: string[];
+        social: string[];
+        address: string;
+    };
+}
+/** What the look pass did (AGL-4084): no model is asked. */
+export interface AiSiteImportLook {
+    at: string;
+    /** The theme: drawn from the source (`applied`), the owner's kept (`kept`), or the source's look could not be read. */
+    theme: 'applied' | 'kept' | 'unread';
+    /** The source's colors it was drawn from, the strongest first. */
+    colors: string[];
+    /** The source's logo, as the library holds it, for the header's brand. */
+    logo: AiSiteImportPhoto | null;
+    /** Whether the site's listing was taken from the home page's own words. */
+    seo: boolean;
+    /** The look's seed, which every page of the import is drawn with. */
+    seed: number | null;
+    /** The look's header arrangement, band rhythm and motion, which the header, the footer and every page are drawn in. */
+    style?: {
+        headerAlign?: string;
+        rhythm?: string;
+        motion?: string;
+    } | null;
+}
+/**
+ * - `home-blocked`: bot protection or a firewall turned AglynBot away from
+ *   the home page (401, 403 or 429).
+ * - `home-unreadable`: the home page could not be read for any other reason,
+ *   or read as an empty shell (a site that needs a browser to show its words).
+ * - `mostly-blocked`: the home page came in, but more than half the site's
+ *   addresses were turned away; the import goes on, and says so.
+ * - `ok`: anything else.
+ */
+export type AiSiteImportReadVerdictKind = 'ok' | 'home-blocked' | 'home-unreadable' | 'mostly-blocked';
+/** Why the home page did not come in. */
+export type AiSiteImportHomeProblem = 'blocked' | 'http-error' | 'unreachable' | 'not-html' | 'too-large' | 'off-site' | 'robots' | 'login-form' | 'not-reached' | 'empty';
+export interface AiSiteImportReadVerdict {
+    kind: AiSiteImportReadVerdictKind;
+    /** Addresses turned away by bot protection or a firewall, the home page among them. */
+    blocked: number;
+    /** The site's addresses the read knew of: the sitemaps', or everything it asked for. */
+    total: number;
+    /** What the home page answered, when it answered with an error. */
+    homeStatus: number | null;
+    homeProblem: AiSiteImportHomeProblem | null;
+    /** Whose edge turned the home page away, when it said. */
+    edge: 'vercel' | 'cloudflare' | null;
+}
+/** A crawl as the verdict reads it; the crawler's `SiteImportInventory` is one. */
+export interface AiSiteImportReadInput {
+    pages: ReadonlyArray<{
+        path: string;
+        foundBy: string;
+        blocks: ReadonlyArray<{
+            kind: string;
+            text?: string;
+            inline?: boolean;
+        }>;
+        images: readonly unknown[];
+    }>;
+    skipped: ReadonlyArray<{
+        url: string;
+        reason: string;
+        status?: number;
+        edge?: string | null;
+    }>;
+    sitemapUrls: number;
 }
+/**
+ * The read's verdict (AGL-4083), pure over the crawl: whether the home page
+ * came in, and how much of the site bot protection turned away. An import
+ * whose verdict stops (`aiSiteImportReadStops`) plans no wave, quotes
+ * nothing and spends nothing: the person is told why and checks again.
+ */
+export declare function aiSiteImportReadVerdict(crawled: AiSiteImportReadInput): AiSiteImportReadVerdict;
+/** Whether a verdict stops the import before anything is planned: its home page is not in. */
+export declare function aiSiteImportReadStops(verdict: AiSiteImportReadVerdict | null | undefined): boolean;
+/** The review an import that stopped at its read waits under: `blocked` or `unreadable`; `null` when it goes on. */
+export declare function aiSiteImportReadVerdictReason(verdict: AiSiteImportReadVerdict | null | undefined): 'blocked' | 'unreadable' | null;
+/** Where a site owner lets AglynBot through, on the edges that name themselves. */
+export declare const AI_SITE_IMPORT_EDGE_HINTS: Readonly<Record<'vercel' | 'cloudflare', string>>;
+/**
+ * What the person is told about the read (AGL-4083), in console words: why
+ * nothing was brought in and what to do, or that most of the site was turned
+ * away. `null` when there is nothing to say.
+ */
+export declare function aiSiteImportReadVerdictMessage(verdict: AiSiteImportReadVerdict | null | undefined, domain: string): string | null;
 /** A collection's prefix for a page address: its first segment, a dated archive's `posts`; `null` for a top-level page. */
 export declare function aiSiteImportPrefix(path: string): string | null;
 /** "news-and-events" → "News and events". */
@@ -221,6 +406,12 @@ export interface AiSiteImportInventoryInput {
     /** Skips the crawl only counted. */
     skippedOverflow?: number;
     stoppedBy: string | null;
+    /**
+     * The read's verdict from the crawl itself (`aiSiteImportReadVerdict`), with
+     * the status and edge the home page answered with; read from `pages` and
+     * `skipped` when absent.
+     */
+    verdict?: AiSiteImportReadVerdict | null;
     /** The plan's page count; `null` for none. */
     pageLimit: number | null;
     existingPages: number;
@@ -276,26 +467,60 @@ export declare function aiSiteImportReviewWave(state: AiSiteImportState | null |
 export declare function aiSiteImportNextWork(state: AiSiteImportState | null | undefined, items: readonly AiJobItemLedger[] | null | undefined): AiSiteImportWork;
 /** A wave started: running, with what its read found on each page. */
 export declare function aiSiteImportWaveStarted(state: AiSiteImportState, index: number, at: string, reads: ReadonlyMap<string, AiSiteImportPageRead | AiSiteImportLeftOutReason>): AiSiteImportState;
-/** More of a running wave's pages, read now. */
+/** How many characters a block holds, as the job document stores it. */
+export declare function aiSiteImportBlockChars(block: SiteImportBlock): number;
+/** A page's blocks in order, as many as fit `maxChars` and `maxBlocks`, and whether any had to go. */
+export declare function aiSiteImportCapBlocks(blocks: readonly SiteImportBlock[], maxChars: number, maxBlocks?: number): {
+    blocks: SiteImportBlock[];
+    chars: number;
+    cut: boolean;
+};
+/**
+ * More of a running wave's pages, read now. Their blocks share
+ * `AI_SITE_IMPORT_WAVE_READ_MAX_CHARS` with every read still held: a page
+ * past it keeps what fits, in order, and is marked `cut`.
+ */
 export declare function aiSiteImportPagesRead(state: AiSiteImportState, reads: ReadonlyMap<string, AiSiteImportPageRead | AiSiteImportLeftOutReason>): AiSiteImportState;
 /** A page's photos are in the library. */
 export declare function aiSiteImportPhotosIn(state: AiSiteImportState, slot: string, photos: NonNullable<AiSiteImportPage['photos']>): AiSiteImportState;
-/** A page is in: its text is no longer kept. */
+/** A page is in: its text and its blocks are no longer kept. */
 export declare function aiSiteImportPageSettled(state: AiSiteImportState, slot: string): AiSiteImportState;
 /** A wave is in: waiting for its review, or done when no wave follows it. */
 export declare function aiSiteImportWaveClosed(state: AiSiteImportState, index: number, at: string): AiSiteImportState;
 /** The state as the console reads it: every page's read text left off. */
 export declare function aiSiteImportWire(state: AiSiteImportState | null | undefined): AiSiteImportState | null;
+/**
+ * How a page is built (AGL-4083), decided by code: every page of a collection
+ * (a blog, the legal pages, news) and any page longer than
+ * `AI_SITE_IMPORT_PROSE_WORDS` is copied as it is; the main pages are
+ * arranged by Aglyn AI, their words kept as written.
+ */
+export declare function aiSiteImportPageBuild(page: Pick<AiSiteImportPage, 'group' | 'words'>): AiSiteImportBuild;
 /**
  * What bringing in this many pages costs, quoted before it starts (AGL-3976):
  * each page an AI page as measured, its ceiling its two passes, and the
- * site's header and footer once when the wave builds them.
+ * site's header and footer once when the wave builds them. Pages the step has
+ * chosen a build for (AGL-4083) are priced by it: `arranged` at the
+ * arrangement answer and its listing, `copied` at the listing alone.
  */
 export declare function aiSiteImportCreditRange(pages: number, options?: {
     frame?: boolean;
+    arranged?: number;
+    copied?: number;
 }): AiCreditRange;
+/** What a wave still owes, by how its pages are built, and whether its header and footer are owed. */
+export declare function aiSiteImportWaveOwed(state: AiSiteImportState, wave: AiSiteImportWave, items?: readonly AiJobItemLedger[] | null): {
+    pages: number;
+    arranged: number;
+    copied: number;
+    frame: boolean;
+};
 /** A wave's quote: the pages it has still to bring in, and its header and footer while they are owed. */
 export declare function aiSiteImportWaveCreditRange(state: AiSiteImportState, wave: AiSiteImportWave, items?: readonly AiJobItemLedger[] | null): AiCreditRange;
+/** Every wave still waiting, quoted at once: "Start every remaining wave" (AGL-4083). */
+export declare function aiSiteImportRemainingCreditRange(state: AiSiteImportState, items?: readonly AiJobItemLedger[] | null): AiCreditRange;
+/** "About 95 credits (up to 600): 7 pages arranged by Aglyn AI, 3 copied as they are"; the build is named once the step has chosen. */
+export declare function aiSiteImportQuoteText(range: AiCreditRange, owed: Pick<ReturnType<typeof aiSiteImportWaveOwed>, 'arranged' | 'copied'>): string;
 /** Why a page is not brought in, as a person reads it. */
 export declare const AI_SITE_IMPORT_LEFT_OUT_COPY: Readonly<Record<AiSiteImportLeftOutReason, string>>;
 /** "3 pages" / "1 page". */