@relvehq/agent-ready 0.0.0-stage → 0.2.1

package/dist/refine/prompts.d.ts

@@ -0,0 +1,49 @@
+import { type PageInput } from "./types";
+export interface PromptSettings {
+    pageType: string;
+    version: number;
+    requiredSections: string[];
+    tokenBudget: number;
+    /** Kept from the POC table; not enforced yet. */
+    recallThreshold: number | null;
+}
+export interface RefinePrompt {
+    /** "<page_type>-v<version>", e.g. "tool-v1": the doc's prompt_version. */
+    id: string;
+    system: string;
+    user: string;
+    settings: PromptSettings;
+}
+export declare function parsePrompt(source: string): RefinePrompt;
+export interface FilledPrompt {
+    system: string;
+    user: string;
+}
+export interface FillOptions {
+    generatedAt: string;
+    /** {{SITE_NAME}}. Default: the page's host without "www.". */
+    siteName?: string;
+    /** {{PAGE_KIND_HINT}}. Default: guessPageKind(input). */
+    kindHint?: PageKind;
+}
+/** Put the page into the prompt. Every {{PLACEHOLDER}} the recipe defines is filled. */
+export declare function fillPrompt(prompt: RefinePrompt, input: PageInput, opts: FillOptions): FilledPrompt;
+/** The page kinds guessPageKind can name; passed to a `page` prompt as {{PAGE_KIND_HINT}}. */
+export declare const PAGE_KINDS: readonly ["home", "product", "pricing", "article", "docs", "listing", "event", "about", "contact", "legal", "other"];
+export type PageKind = (typeof PAGE_KINDS)[number];
+/**
+ * A first guess at the page's kind, from the path and JSON-LD. Only a hint:
+ * the prompt tells the model to confirm it from the page or pick better.
+ * Path rules win over generic JSON-LD (every page has an Organization block).
+ */
+export declare function guessPageKind(input: Pick<PageInput, "url" | "jsonLd">): PageKind;
+/** The prompt key that takes any path no other prompt claims. */
+export declare const FALLBACK_PROMPT = "page";
+/** The prompt for a path: a matching page-type prompt, else the any-page prompt, else null. */
+export declare function promptFor(path: string, prompts: Record<string, RefinePrompt>, rules?: PageTypeRules): RefinePrompt | null;
+/** Which prompt a path uses: the first matching prefix. Relve's types by default. */
+export type PageTypeRules = ReadonlyArray<readonly [prefix: string, pageType: string]>;
+export declare const DEFAULT_PAGE_TYPES: PageTypeRules;
+export declare function pageTypeFor(path: string, rules?: PageTypeRules): string | null;
+/** Many prompt files -> the newest version per page type. */
+export declare function promptsByType(prompts: RefinePrompt[]): Record<string, RefinePrompt>;

package/dist/refine/prompts.js

@@ -0,0 +1,178 @@
+"use strict";
+// Refine, step 2: a prompt file -> the model's messages.
+//
+// A prompt file is the POC's format (changes/markdown-poc/prompts/*-v1.md):
+//
+//   ## SYSTEM     the rules (may hold {{SOURCE_HASH}}, {{PROMPT_VERSION}},
+//                 {{GENERATED_AT}} in its front matter template)
+//   ## USER       the template the page is filled into ({{PAGE_URL}},
+//                 {{PAGE_TEXT}}, {{PAGE_LINKS}}, {{JSON_LD}})
+//   ---
+//   ## Settings   a table: page_type, version, required_sections,
+//                 token_budget, recall_threshold
+//
+// The settings are checked by code (refine/check.ts), never by the model.
+// EDGE-SAFE (pure).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_PAGE_TYPES = exports.FALLBACK_PROMPT = exports.PAGE_KINDS = void 0;
+exports.parsePrompt = parsePrompt;
+exports.fillPrompt = fillPrompt;
+exports.guessPageKind = guessPageKind;
+exports.promptFor = promptFor;
+exports.pageTypeFor = pageTypeFor;
+exports.promptsByType = promptsByType;
+const types_1 = require("./types");
+function parsePrompt(source) {
+    const md = source.replace(/^/, "").replace(/\r\n?/g, "\n");
+    const sys = /^## SYSTEM[ \t]*$/m.exec(md);
+    const usr = /^## USER[ \t]*$/m.exec(md);
+    const set = /^## Settings\b.*$/m.exec(md);
+    if (!sys || !usr || !set || !(sys.index < usr.index && usr.index < set.index)) {
+        throw new Error("agent-ready refine: a prompt needs '## SYSTEM', '## USER' and '## Settings', in that order.");
+    }
+    const system = md.slice(sys.index + sys[0].length, usr.index).trim();
+    const user = md
+        .slice(usr.index + usr[0].length, set.index)
+        .trim()
+        .replace(/\n-{3,}\s*$/, "")
+        .trim();
+    const rows = {};
+    for (const m of md.slice(set.index).matchAll(/^\|\s*`(\w+)`\s*\|(.+)\|\s*$/gm)) {
+        rows[m[1]] = Array.from(m[2].matchAll(/`([^`]+)`/g), (v) => v[1]);
+    }
+    const one = (k) => rows[k]?.[0];
+    const pageType = one("page_type");
+    const version = Number(one("version"));
+    const tokenBudget = Number(one("token_budget"));
+    if (!pageType || !Number.isFinite(version) || !Number.isFinite(tokenBudget)) {
+        throw new Error("agent-ready refine: the Settings table needs page_type, version and token_budget.");
+    }
+    const recall = Number(one("recall_threshold"));
+    return {
+        id: `${pageType}-v${version}`,
+        system,
+        user,
+        settings: {
+            pageType,
+            version,
+            requiredSections: rows.required_sections ?? [],
+            tokenBudget,
+            recallThreshold: Number.isFinite(recall) ? recall : null,
+        },
+    };
+}
+/** Put the page into the prompt. Every {{PLACEHOLDER}} the recipe defines is filled. */
+function fillPrompt(prompt, input, opts) {
+    const u = new URL(input.url);
+    const vars = {
+        PAGE_URL: input.url,
+        PAGE_TEXT: input.text,
+        PAGE_LINKS: (0, types_1.formatLinks)(input.links),
+        JSON_LD: JSON.stringify(input.jsonLd, null, 2),
+        SOURCE_HASH: input.sourceHash,
+        PROMPT_VERSION: prompt.id,
+        GENERATED_AT: opts.generatedAt,
+        SITE_NAME: opts.siteName ?? u.hostname.replace(/^www\./, ""),
+        SITE_URL: u.origin,
+        PAGE_KIND_HINT: opts.kindHint ?? guessPageKind(input),
+    };
+    // A function replacer, so "$" in page text is never read as a pattern.
+    const fill = (s) => s.replace(/\{\{([A-Z_]+)\}\}/g, (m, k) => (k in vars ? vars[k] : m));
+    return { system: fill(prompt.system), user: fill(prompt.user) };
+}
+/** The page kinds guessPageKind can name; passed to a `page` prompt as {{PAGE_KIND_HINT}}. */
+exports.PAGE_KINDS = ["home", "product", "pricing", "article", "docs", "listing", "event", "about", "contact", "legal", "other"];
+const KIND_BY_LD = [
+    [/^(Article|BlogPosting|NewsArticle|TechArticle|Report|ScholarlyArticle)$/, "article"],
+    [/^(Event|BusinessEvent|EducationEvent|Hackathon|SocialEvent)$/, "event"],
+    [/^(Product|SoftwareApplication|WebApplication|MobileApplication|Service|Course)$/, "product"],
+    [/^(CollectionPage|ItemList|SearchResultsPage)$/, "listing"],
+    [/^(AboutPage|ProfilePage)$/, "about"],
+    [/^(ContactPage)$/, "contact"],
+    [/^(HowTo)$/, "docs"],
+];
+const KIND_BY_PATH = [
+    [/^\/(pricing|plans|prices)(\/|$)/, "pricing"],
+    [/^\/(blog|news|articles?|posts?|stories|insights|press|case-stud(y|ies)|resources\/blog)(\/|$)/, "article"],
+    [/^\/(docs?|documentation|help|support|guides?|kb|knowledge-base|faq|learn|api)(\/|$)/, "docs"],
+    [/^\/(events?|webinars?|conferences?)(\/|$)/, "event"],
+    [/^\/(about|about-us|company|team|careers|jobs|mission)(\/|$)/, "about"],
+    [/^\/(contact|contact-us|locations?|stores?)(\/|$)/, "contact"],
+    [/^\/(privacy|terms|legal|cookies?|policies|policy|refunds?|security|gdpr|tos)(-[\w-]+)?(\/|$)/, "legal"],
+    [/^\/(products?|services?|solutions?|features?|tools?|apps?|platform|integrations?)\/[^/]+/, "product"],
+    [/^\/(products?|services?|solutions?|tools?|category|categories|collections?|shop|catalog|directory|tags?)(\/|$)/, "listing"],
+];
+/**
+ * A first guess at the page's kind, from the path and JSON-LD. Only a hint:
+ * the prompt tells the model to confirm it from the page or pick better.
+ * Path rules win over generic JSON-LD (every page has an Organization block).
+ */
+function guessPageKind(input) {
+    let path = "/";
+    try {
+        path = new URL(input.url).pathname.replace(/\/+$/, "") || "/";
+    }
+    catch {
+        // keep "/"
+    }
+    if (path === "/")
+        return "home";
+    for (const [re, kind] of KIND_BY_PATH)
+        if (re.test(path.toLowerCase()))
+            return kind;
+    for (const t of ldTypes(input.jsonLd))
+        for (const [re, kind] of KIND_BY_LD)
+            if (re.test(t))
+                return kind;
+    return "other";
+}
+function ldTypes(blocks) {
+    const out = [];
+    const visit = (v, depth) => {
+        if (!v || typeof v !== "object" || depth > 3)
+            return;
+        if (Array.isArray(v))
+            return v.forEach((x) => visit(x, depth + 1));
+        const o = v;
+        const t = o["@type"];
+        if (typeof t === "string")
+            out.push(t);
+        else if (Array.isArray(t))
+            out.push(...t.filter((x) => typeof x === "string"));
+        if (o["@graph"])
+            visit(o["@graph"], depth + 1);
+        if (o.mainEntity)
+            visit(o.mainEntity, depth + 1);
+    };
+    visit(blocks, 0);
+    return out;
+}
+/** The prompt key that takes any path no other prompt claims. */
+exports.FALLBACK_PROMPT = "page";
+/** The prompt for a path: a matching page-type prompt, else the any-page prompt, else null. */
+function promptFor(path, prompts, rules = exports.DEFAULT_PAGE_TYPES) {
+    const type = pageTypeFor(path, rules);
+    return (type && prompts[type]) || prompts[exports.FALLBACK_PROMPT] || null;
+}
+exports.DEFAULT_PAGE_TYPES = [
+    ["/tool/", "tool"],
+    ["/agent/", "agent"],
+    ["/events/", "event"],
+    ["/blog/", "blog"],
+];
+function pageTypeFor(path, rules = exports.DEFAULT_PAGE_TYPES) {
+    for (const [prefix, type] of rules)
+        if (path.startsWith(prefix) && path.length > prefix.length)
+            return type;
+    return null;
+}
+/** Many prompt files -> the newest version per page type. */
+function promptsByType(prompts) {
+    const out = {};
+    for (const p of prompts) {
+        const cur = out[p.settings.pageType];
+        if (!cur || p.settings.version > cur.settings.version)
+            out[p.settings.pageType] = p;
+    }
+    return out;
+}

package/dist/refine/refine.d.ts

@@ -0,0 +1,38 @@
+import { type CheckIssue, type CheckOptions } from "./check";
+import { type ModelClient } from "./model";
+import { type RefinePrompt } from "./prompts";
+import type { PageInput } from "./types";
+export type RefineStatus = "ok" | "unchanged" | "rejected" | "error";
+export interface RefineOptions extends CheckOptions {
+    /** Absolute page URL. */
+    url: string;
+    prompt: RefinePrompt;
+    model: ModelClient;
+    /** The page's HTML, when the caller already has it (else it is fetched). */
+    html?: string;
+    fetch?: typeof fetch;
+    /** The stored doc's fingerprint: equal to the page's -> "unchanged", no model call. */
+    previousHash?: string | null;
+    /** Extra attempts after a failed check, each told what failed. Default 1. */
+    retries?: number;
+    now?: () => Date;
+    /** {{SITE_NAME}} for the any-page prompt. Default: the host. */
+    siteName?: string;
+}
+export interface RefineResult {
+    status: RefineStatus;
+    url: string;
+    input: PageInput | null;
+    /** The last document the model wrote (kept on "rejected", for review). */
+    doc: string | null;
+    issues: CheckIssue[];
+    attempts: number;
+    tokens: number | null;
+    generatedAt: string | null;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    error: string | null;
+}
+export declare function refinePage(opts: RefineOptions): Promise<RefineResult>;

package/dist/refine/refine.js

@@ -0,0 +1,75 @@
+"use strict";
+// Refine one page: fetch -> extract -> (unchanged? stop) -> fill the prompt ->
+// model -> check -> retry once with the issues -> result.
+// Nothing is stored here; the runner decides (refine/run.ts).
+//
+// NODE ONLY (extract uses linkedom).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.refinePage = refinePage;
+const extract_1 = require("./extract");
+const check_1 = require("./check");
+const model_1 = require("./model");
+const prompts_1 = require("./prompts");
+async function refinePage(opts) {
+    const result = {
+        status: "error",
+        url: opts.url,
+        input: null,
+        doc: null,
+        issues: [],
+        attempts: 0,
+        tokens: null,
+        generatedAt: null,
+        usage: { inputTokens: 0, outputTokens: 0 },
+        error: null,
+    };
+    try {
+        const html = opts.html ?? (await fetchPage(opts.url, opts.fetch ?? fetch));
+        const input = await (0, extract_1.extractInput)(html, { url: opts.url });
+        result.input = input;
+        if (opts.previousHash && opts.previousHash === input.sourceHash) {
+            result.status = "unchanged";
+            return result;
+        }
+        const generatedAt = (opts.now ?? (() => new Date()))().toISOString().replace(/\.\d{3}Z$/, "Z");
+        result.generatedAt = generatedAt;
+        const filled = (0, prompts_1.fillPrompt)(opts.prompt, input, { generatedAt, siteName: opts.siteName });
+        const maxAttempts = 1 + Math.max(0, opts.retries ?? 1);
+        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+            result.attempts = attempt;
+            const user = attempt === 1 ? filled.user : `${filled.user}\n\n${retryNote(result.issues)}`;
+            const reply = await opts.model({ system: filled.system, user });
+            result.usage.inputTokens += reply.inputTokens ?? 0;
+            result.usage.outputTokens += reply.outputTokens ?? 0;
+            const doc = (0, model_1.cleanReply)(reply.text);
+            const check = (0, check_1.checkDoc)(doc, input, opts.prompt.settings, opts);
+            result.doc = doc;
+            result.issues = check.issues;
+            result.tokens = check.tokens;
+            if (check.ok) {
+                result.status = "ok";
+                return result;
+            }
+        }
+        result.status = "rejected";
+        return result;
+    }
+    catch (err) {
+        result.status = "error";
+        result.error = err?.message ?? String(err);
+        return result;
+    }
+}
+function retryNote(issues) {
+    const list = issues.map((i) => `- ${i.kind}: ${i.detail}`).join("\n");
+    return `<check_failed>\nYour previous document failed the automatic check:\n${list}\nRemove anything that is not in the input and fix the rest. Output the corrected document only.\n</check_failed>`;
+}
+async function fetchPage(url, f) {
+    const res = await f(url, {
+        headers: { accept: "text/html", "user-agent": "agent-ready-refine/0.1" },
+        redirect: "follow",
+    });
+    if (!res.ok)
+        throw new Error(`page fetch failed: HTTP ${res.status}`);
+    return res.text();
+}

package/dist/refine/run.d.ts

@@ -0,0 +1,52 @@
+import { type RefineResult } from "./refine";
+import { type PageTypeRules, type RefinePrompt } from "./prompts";
+import type { CheckOptions } from "./check";
+import type { ModelClient } from "./model";
+export interface StoredDoc {
+    path: string;
+    url: string;
+    markdown: string;
+    sourceHash: string;
+    promptId: string;
+    generatedAt: string;
+    tokens: number;
+}
+export interface RefineStore {
+    /** The fingerprint the stored doc was written from, or null. */
+    lastHash(path: string): Promise<string | null>;
+    save(doc: StoredDoc): Promise<void>;
+}
+export interface RunOptions extends CheckOptions {
+    /** Public origin, e.g. "https://relvehq.com". */
+    site: string;
+    paths: string[];
+    /** Prompts by page type (promptsByType). */
+    prompts: Record<string, RefinePrompt>;
+    pageTypes?: PageTypeRules;
+    model: ModelClient;
+    store: RefineStore;
+    /** Refine even when the fingerprint is unchanged. */
+    force?: boolean;
+    /** Pages at a time. Default 2. */
+    concurrency?: number;
+    retries?: number;
+    fetch?: typeof fetch;
+    now?: () => Date;
+    siteName?: string;
+    onResult?: (r: RunItem) => void;
+}
+export interface RunItem extends Partial<Omit<RefineResult, "status">> {
+    path: string;
+    status: RefineResult["status"] | "no-prompt";
+    promptId: string | null;
+    saved: boolean;
+}
+export interface RunReport {
+    items: RunItem[];
+    counts: Record<RunItem["status"], number>;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+}
+export declare function runRefine(opts: RunOptions): Promise<RunReport>;

package/dist/refine/run.js

@@ -0,0 +1,66 @@
+"use strict";
+// Refine, step 5: the runner. Refines a list of pages, skips pages whose
+// fingerprint hasn't changed ("stay fresh"), saves what passes the check.
+//
+//   await runRefine({ site, paths, prompts, model, store })
+//
+// The store is anything with lastHash() and save(): the file store
+// (refine/file-store.ts), a database, a test double.
+// NODE ONLY (through refinePage).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runRefine = runRefine;
+const refine_1 = require("./refine");
+const prompts_1 = require("./prompts");
+async function runRefine(opts) {
+    const site = opts.site.replace(/\/+$/, "");
+    const items = new Array(opts.paths.length);
+    let next = 0;
+    const one = async (path) => {
+        const prompt = (0, prompts_1.promptFor)(path, opts.prompts, opts.pageTypes);
+        if (!prompt)
+            return { path, status: "no-prompt", promptId: null, saved: false };
+        const previousHash = opts.force ? null : await opts.store.lastHash(path);
+        const r = await (0, refine_1.refinePage)({
+            url: site + path,
+            prompt,
+            model: opts.model,
+            previousHash,
+            retries: opts.retries,
+            fetch: opts.fetch,
+            now: opts.now,
+            siteName: opts.siteName,
+            countTokens: opts.countTokens,
+        });
+        let saved = false;
+        if (r.status === "ok" && r.doc && r.input && r.generatedAt) {
+            await opts.store.save({
+                path,
+                url: r.url,
+                markdown: r.doc,
+                sourceHash: r.input.sourceHash,
+                promptId: prompt.id,
+                generatedAt: r.generatedAt,
+                tokens: r.tokens ?? 0,
+            });
+            saved = true;
+        }
+        return { ...r, path, promptId: prompt.id, saved };
+    };
+    const worker = async () => {
+        while (next < opts.paths.length) {
+            const i = next++;
+            const item = await one(opts.paths[i]);
+            items[i] = item;
+            opts.onResult?.(item);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(opts.concurrency ?? 2, opts.paths.length)) }, worker));
+    const counts = { ok: 0, unchanged: 0, rejected: 0, error: 0, "no-prompt": 0 };
+    const usage = { inputTokens: 0, outputTokens: 0 };
+    for (const it of items) {
+        counts[it.status]++;
+        usage.inputTokens += it.usage?.inputTokens ?? 0;
+        usage.outputTokens += it.usage?.outputTokens ?? 0;
+    }
+    return { items, counts, usage };
+}

package/dist/refine/types.d.ts

@@ -0,0 +1,17 @@
+export interface PageLink {
+    url: string;
+    text: string;
+}
+/** The model's input for one page (see refine/extract.ts). */
+export interface PageInput {
+    /** Absolute page URL. */
+    url: string;
+    text: string;
+    links: PageLink[];
+    jsonLd: unknown[];
+    /** SHA-256 hex of `text + "\n" + JSON.stringify(jsonLd)`. */
+    sourceHash: string;
+    mainSource: "main" | "article" | "body";
+}
+/** Links as the prompt's {{PAGE_LINKS}} block: "url | text" per line. */
+export declare function formatLinks(links: PageLink[]): string;

package/dist/refine/types.js

@@ -0,0 +1,8 @@
+"use strict";
+// Shared shapes for refine. EDGE-SAFE (no imports).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatLinks = formatLinks;
+/** Links as the prompt's {{PAGE_LINKS}} block: "url | text" per line. */
+function formatLinks(links) {
+    return links.map((l) => (l.text ? `${l.url} | ${l.text}` : l.url)).join("\n");
+}