@relvehq/agent-ready 0.0.0-stage → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +338 -3
  3. package/SETUP.md +337 -0
  4. package/dist/bin/agent-ready.d.ts +23 -0
  5. package/dist/bin/agent-ready.js +350 -0
  6. package/dist/cli/init.d.ts +70 -0
  7. package/dist/cli/init.js +366 -0
  8. package/dist/core/accept.d.ts +8 -0
  9. package/dist/core/accept.js +76 -0
  10. package/dist/core/config.d.ts +145 -0
  11. package/dist/core/config.js +103 -0
  12. package/dist/core/convert.d.ts +22 -0
  13. package/dist/core/convert.js +334 -0
  14. package/dist/core/detect.d.ts +30 -0
  15. package/dist/core/detect.js +82 -0
  16. package/dist/core/easy.d.ts +19 -0
  17. package/dist/core/easy.js +117 -0
  18. package/dist/core/enrich.d.ts +30 -0
  19. package/dist/core/enrich.js +155 -0
  20. package/dist/core/llms.d.ts +11 -0
  21. package/dist/core/llms.js +71 -0
  22. package/dist/core/report.d.ts +15 -0
  23. package/dist/core/report.js +46 -0
  24. package/dist/core/resolve.d.ts +43 -0
  25. package/dist/core/resolve.js +204 -0
  26. package/dist/core/respond.d.ts +18 -0
  27. package/dist/core/respond.js +59 -0
  28. package/dist/core/sitemap.d.ts +31 -0
  29. package/dist/core/sitemap.js +140 -0
  30. package/dist/index.d.ts +14 -0
  31. package/dist/index.js +76 -0
  32. package/dist/next/dashboard.d.ts +14 -0
  33. package/dist/next/dashboard.js +56 -0
  34. package/dist/next/index.d.ts +42 -0
  35. package/dist/next/index.js +100 -0
  36. package/dist/next/route.d.ts +26 -0
  37. package/dist/next/route.js +167 -0
  38. package/dist/node/index.d.ts +8 -0
  39. package/dist/node/index.js +191 -0
  40. package/dist/proof/audit.d.ts +47 -0
  41. package/dist/proof/audit.js +253 -0
  42. package/dist/proof/bots.d.ts +24 -0
  43. package/dist/proof/bots.js +60 -0
  44. package/dist/proof/dashboard.d.ts +8 -0
  45. package/dist/proof/dashboard.js +77 -0
  46. package/dist/proof/hits.d.ts +18 -0
  47. package/dist/proof/hits.js +51 -0
  48. package/dist/proof/jsonl.d.ts +5 -0
  49. package/dist/proof/jsonl.js +80 -0
  50. package/dist/proof/stats.d.ts +39 -0
  51. package/dist/proof/stats.js +127 -0
  52. package/dist/refine/check.d.ts +17 -0
  53. package/dist/refine/check.js +134 -0
  54. package/dist/refine/extract.d.ts +7 -0
  55. package/dist/refine/extract.js +120 -0
  56. package/dist/refine/file-store.d.ts +6 -0
  57. package/dist/refine/file-store.js +87 -0
  58. package/dist/refine/model.d.ts +30 -0
  59. package/dist/refine/model.js +105 -0
  60. package/dist/refine/prompts.d.ts +49 -0
  61. package/dist/refine/prompts.js +178 -0
  62. package/dist/refine/refine.d.ts +38 -0
  63. package/dist/refine/refine.js +75 -0
  64. package/dist/refine/run.d.ts +52 -0
  65. package/dist/refine/run.js +66 -0
  66. package/dist/refine/types.d.ts +17 -0
  67. package/dist/refine/types.js +8 -0
  68. package/dist/wp/relve-agent-ready/includes/class-relve-ar-admin.php +363 -0
  69. package/dist/wp/relve-agent-ready/includes/class-relve-ar-converter.php +762 -0
  70. package/dist/wp/relve-agent-ready/includes/class-relve-ar-llms.php +192 -0
  71. package/dist/wp/relve-agent-ready/includes/class-relve-ar-markdown-builder.php +447 -0
  72. package/dist/wp/relve-agent-ready/includes/class-relve-ar-negotiation.php +334 -0
  73. package/dist/wp/relve-agent-ready/includes/class-relve-ar-response.php +226 -0
  74. package/dist/wp/relve-agent-ready/includes/class-relve-ar-router.php +421 -0
  75. package/dist/wp/relve-agent-ready/includes/class-relve-ar-settings.php +115 -0
  76. package/dist/wp/relve-agent-ready/includes/class-relve-ar-store.php +543 -0
  77. package/dist/wp/relve-agent-ready/includes/index.php +2 -0
  78. package/dist/wp/relve-agent-ready/index.php +2 -0
  79. package/dist/wp/relve-agent-ready/readme.txt +75 -0
  80. package/dist/wp/relve-agent-ready/relve-agent-ready.php +70 -0
  81. package/dist/wp/relve-agent-ready/uninstall.php +24 -0
  82. package/package.json +64 -6
@@ -0,0 +1,127 @@
1
+ "use strict";
2
+ // Proof: the numbers. HitEvents -> one summary for the dashboard and the CLI.
3
+ // EDGE-SAFE (pure).
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.summarize = summarize;
6
+ exports.formatSummary = formatSummary;
7
+ const bots_1 = require("./bots");
8
+ const VIA_LABEL = {
9
+ accept: "Accept: text/markdown",
10
+ suffix: ".md URL",
11
+ query: "?format=md",
12
+ llms: "llms.txt",
13
+ direct: "direct",
14
+ };
15
+ const SOURCE_LABEL = {
16
+ native: "template",
17
+ stored: "saved file",
18
+ convert: "live conversion",
19
+ llms: "llms.txt",
20
+ };
21
+ const KIND_LABEL = {
22
+ ai_assistant: "AI assistant",
23
+ ai_search: "AI search",
24
+ ai_training: "AI training",
25
+ search: "Search engine",
26
+ tool: "Script",
27
+ unknown: "Other",
28
+ };
29
+ function summarize(events, opts = {}) {
30
+ const sorted = events.slice().sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0));
31
+ const bots = new Map();
32
+ const kinds = new Map();
33
+ const paths = new Map();
34
+ const vias = new Map();
35
+ const sources = new Map();
36
+ const statuses = new Map();
37
+ const days = new Map();
38
+ let served = 0;
39
+ let answer = 0;
40
+ let tokensServed = 0;
41
+ const durations = [];
42
+ const bump = (m, key, label, tokens) => {
43
+ const c = m.get(key) ?? { key, label, hits: 0, tokens: 0 };
44
+ c.hits++;
45
+ c.tokens += tokens;
46
+ m.set(key, c);
47
+ };
48
+ for (const e of sorted) {
49
+ const ok = (e.status >= 200 && e.status < 300) || e.status === 304;
50
+ const tokens = ok ? e.tokens ?? 0 : 0;
51
+ if (ok)
52
+ served++;
53
+ tokensServed += tokens;
54
+ if (Number.isFinite(e.durationMs))
55
+ durations.push(e.durationMs);
56
+ const bot = (0, bots_1.classifyAgent)(e.userAgent);
57
+ if ((0, bots_1.isAnswerBot)(bot))
58
+ answer++;
59
+ const b = bots.get(bot.id) ?? { key: bot.id, label: bot.label, operator: bot.operator, kind: bot.kind, hits: 0, tokens: 0 };
60
+ b.hits++;
61
+ b.tokens += tokens;
62
+ bots.set(bot.id, b);
63
+ bump(kinds, bot.kind, KIND_LABEL[bot.kind], tokens);
64
+ bump(paths, e.path, e.path, tokens);
65
+ bump(vias, e.via, VIA_LABEL[e.via] ?? e.via, tokens);
66
+ bump(sources, e.source ?? "none", e.source ? SOURCE_LABEL[e.source] ?? e.source : "nothing served", tokens);
67
+ bump(statuses, String(e.status), String(e.status), tokens);
68
+ const day = e.ts.slice(0, 10);
69
+ const d = days.get(day) ?? { hits: 0, tokens: 0 };
70
+ d.hits++;
71
+ d.tokens += tokens;
72
+ days.set(day, d);
73
+ }
74
+ const rank = (m, top) => {
75
+ const list = Array.from(m.values()).sort((a, b) => b.hits - a.hits || a.label.localeCompare(b.label));
76
+ return top ? list.slice(0, top) : list;
77
+ };
78
+ durations.sort((a, b) => a - b);
79
+ const from = sorted[0]?.ts ?? null;
80
+ const to = sorted[sorted.length - 1]?.ts ?? null;
81
+ return {
82
+ from,
83
+ to,
84
+ total: sorted.length,
85
+ served,
86
+ answerBotHits: answer,
87
+ tokensServed,
88
+ medianMs: durations.length ? durations[Math.floor((durations.length - 1) / 2)] : null,
89
+ byBot: rank(bots),
90
+ byKind: rank(kinds),
91
+ byPath: rank(paths, opts.topPaths ?? 20),
92
+ byVia: rank(vias),
93
+ bySource: rank(sources),
94
+ byStatus: rank(statuses),
95
+ daily: fillDays(days, from, to),
96
+ };
97
+ }
98
+ function fillDays(days, from, to) {
99
+ if (!from || !to)
100
+ return [];
101
+ const out = [];
102
+ const end = Date.parse(to.slice(0, 10) + "T00:00:00Z");
103
+ for (let t = Date.parse(from.slice(0, 10) + "T00:00:00Z"); t <= end && out.length < 400; t += 86400000) {
104
+ const day = new Date(t).toISOString().slice(0, 10);
105
+ out.push({ day, ...(days.get(day) ?? { hits: 0, tokens: 0 }) });
106
+ }
107
+ return out;
108
+ }
109
+ /** The summary as terminal lines (`agent-ready stats`). */
110
+ function formatSummary(s) {
111
+ if (!s.total)
112
+ return "No agent visits recorded yet.";
113
+ const n = (x) => x.toLocaleString("en-US");
114
+ const table = (title, rows, max = 10) => [title, ...rows.slice(0, max).map((r) => ` ${String(r.hits).padStart(6)} ${r.label}${r.tokens ? ` (${n(r.tokens)} tokens)` : ""}`)].join("\n");
115
+ return [
116
+ `Agent visits ${s.from?.slice(0, 10)} to ${s.to?.slice(0, 10)}`,
117
+ ` ${n(s.total)} requests · ${n(s.served)} served · ${n(s.answerBotHits)} from AI assistants and AI search · ${n(s.tokensServed)} tokens served${s.medianMs !== null ? ` · median ${s.medianMs} ms` : ""}`,
118
+ "",
119
+ table("By bot", s.byBot),
120
+ "",
121
+ table("Top pages", s.byPath),
122
+ "",
123
+ table("How agents asked", s.byVia),
124
+ "",
125
+ table("Where the Markdown came from", s.bySource),
126
+ ].join("\n");
127
+ }
@@ -0,0 +1,17 @@
1
+ import type { PageInput } from "./types";
2
+ import type { PromptSettings } from "./prompts";
3
+ export type IssueKind = "number" | "url" | "frontmatter" | "hash" | "section" | "h1" | "html" | "placeholder" | "tokens" | "empty";
4
+ export interface CheckIssue {
5
+ kind: IssueKind;
6
+ detail: string;
7
+ }
8
+ export interface CheckResult {
9
+ ok: boolean;
10
+ issues: CheckIssue[];
11
+ tokens: number;
12
+ }
13
+ export interface CheckOptions {
14
+ /** Token counter. Default chars/4; pass a real tokenizer for exact budgets. */
15
+ countTokens?: (text: string) => number;
16
+ }
17
+ export declare function checkDoc(doc: string, input: PageInput, settings: PromptSettings, opts?: CheckOptions): CheckResult;
@@ -0,0 +1,134 @@
1
+ "use strict";
2
+ // Refine, step 4: check the model's document before it is stored.
3
+ // The POC's mechanical fact check, as code (the model's own checking is not
4
+ // trusted). A document passes only with no issues:
5
+ //
6
+ // number a number in the body that is not in the input
7
+ // url a URL in the body that is not in the input (the page URL and
8
+ // input URLs with their query removed are allowed)
9
+ // frontmatter missing, or `canonical` is not the page URL
10
+ // hash `source_hash` is not the input's fingerprint
11
+ // section a required section (prompt settings) is missing
12
+ // h1 not exactly one H1
13
+ // html HTML tags in the document
14
+ // placeholder a {{PLACEHOLDER}} or <Template field> left in
15
+ // tokens over the prompt's token budget
16
+ // empty no real body
17
+ //
18
+ // Frontmatter values are not number-checked: they hold the hash, timestamps
19
+ // and dates reformatted as YYYY-MM-DD by design.
20
+ // EDGE-SAFE (pure). It cannot catch a reworded claim; people still read.
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.checkDoc = checkDoc;
23
+ const enrich_1 = require("../core/enrich");
24
+ const URL_RE = /https?:\/\/[^\s<>()[\]|"'`]+/g;
25
+ const NUM_RE = /\d+(?:[.,]\d+)*/g;
26
+ const HTML_RE = /<\/?(?:a|b|br|div|em|h[1-6]|i|img|li|ol|p|script|span|strong|style|table|td|th|tr|ul)\b[^>]*>/i;
27
+ function checkDoc(doc, input, settings, opts = {}) {
28
+ const issues = [];
29
+ const add = (kind, detail) => issues.push({ kind, detail });
30
+ const md = doc.replace(/\r\n?/g, "\n").trim();
31
+ const tokens = (opts.countTokens ?? enrich_1.estimateTokens)(md);
32
+ // Frontmatter.
33
+ const fm = /^---\n([\s\S]*?)\n---\n?/.exec(md);
34
+ const body = fm ? md.slice(fm[0].length) : md;
35
+ if (!fm) {
36
+ add("frontmatter", "no frontmatter block");
37
+ }
38
+ else {
39
+ const meta = Object.fromEntries(fm[1]
40
+ .split("\n")
41
+ .map((l) => /^([\w-]+):\s*(.*)$/.exec(l))
42
+ .filter((m) => !!m)
43
+ .map((m) => [m[1], m[2].replace(/^["']|["']$/g, "").trim()]));
44
+ if (!meta.canonical)
45
+ add("frontmatter", "no canonical");
46
+ else if (normUrl(meta.canonical) !== normUrl(input.url))
47
+ add("frontmatter", `canonical ${meta.canonical} is not ${input.url}`);
48
+ if (meta.source_hash !== input.sourceHash)
49
+ add("hash", `source_hash ${meta.source_hash ?? "(none)"} does not match the input`);
50
+ }
51
+ const prose = body.replace(/^```[\s\S]*?^```/gm, "");
52
+ if (prose.replace(/[#|\-\s>*_]/g, "").length < 40)
53
+ add("empty", "the body is (almost) empty");
54
+ const h1s = (prose.match(/^# \S/gm) ?? []).length;
55
+ if (h1s !== 1)
56
+ add("h1", `${h1s} H1 headings`);
57
+ for (const name of settings.requiredSections) {
58
+ if (!new RegExp(`^##\\s+${escapeRe(name)}\\s*$`, "m").test(prose))
59
+ add("section", `missing "## ${name}"`);
60
+ }
61
+ if (HTML_RE.test(prose))
62
+ add("html", "HTML tags in the document");
63
+ const left = /\{\{[A-Z_]+\}\}|<(?:Tool|Event|Agent|Post|page) name>|<the kind you chose>/.exec(md);
64
+ if (left)
65
+ add("placeholder", `left in: ${left[0]}`);
66
+ if (tokens > settings.tokenBudget)
67
+ add("tokens", `${tokens} tokens, budget ${settings.tokenBudget}`);
68
+ // Facts: everything the input contains, as one haystack.
69
+ const haystack = [input.text, input.links.map((l) => `${l.url} ${l.text}`).join("\n"), JSON.stringify(input.jsonLd), input.url].join("\n");
70
+ const known = new Set();
71
+ const knownNoQuery = new Set();
72
+ for (const u of haystack.match(URL_RE) ?? []) {
73
+ known.add(normUrl(trimUrl(u)));
74
+ knownNoQuery.add(noQuery(trimUrl(u)));
75
+ }
76
+ known.add(normUrl(input.url));
77
+ // Relative links in the input (e.g. "[x](/events/y)" inside JSON-LD): the
78
+ // prompt asks for absolute URLs, so the model is right to resolve them.
79
+ for (const m of haystack.matchAll(/(?:\]\(|href=\\?["']|^|\s)(\/[A-Za-z0-9][^\s)"'\\<>|]*)/gm)) {
80
+ try {
81
+ const abs = new URL(m[1], input.url).href;
82
+ known.add(normUrl(abs));
83
+ knownNoQuery.add(noQuery(abs));
84
+ }
85
+ catch {
86
+ // not a path
87
+ }
88
+ }
89
+ for (const raw of body.match(URL_RE) ?? []) {
90
+ const u = trimUrl(raw);
91
+ if (!known.has(normUrl(u)) && !knownNoQuery.has(noQuery(u)))
92
+ add("url", u);
93
+ }
94
+ const knownNums = new Set((haystack.match(NUM_RE) ?? []).map(canonNum));
95
+ const seen = new Set();
96
+ for (const n of body.replace(URL_RE, " ").match(NUM_RE) ?? []) {
97
+ const c = canonNum(n);
98
+ if (knownNums.has(c) || seen.has(c))
99
+ continue;
100
+ seen.add(c);
101
+ add("number", n);
102
+ }
103
+ return { ok: issues.length === 0, issues, tokens };
104
+ }
105
+ /** "1,400" = "1400", "0.10" = "0.1", "05" = "5". */
106
+ function canonNum(s) {
107
+ const plain = s.replace(/,(?=\d{3}\b)/g, "");
108
+ const n = Number(plain);
109
+ return Number.isFinite(n) ? String(n) : plain;
110
+ }
111
+ function trimUrl(u) {
112
+ return u.replace(/[.,;:!?*_]+$/, "");
113
+ }
114
+ function normUrl(u) {
115
+ try {
116
+ const x = new URL(u);
117
+ return `${x.protocol}//${x.host.toLowerCase()}${x.pathname.replace(/\/+$/, "")}${x.search}`;
118
+ }
119
+ catch {
120
+ return u;
121
+ }
122
+ }
123
+ function noQuery(u) {
124
+ try {
125
+ const x = new URL(u);
126
+ return `${x.protocol}//${x.host.toLowerCase()}${x.pathname.replace(/\/+$/, "")}`;
127
+ }
128
+ catch {
129
+ return u;
130
+ }
131
+ }
132
+ function escapeRe(s) {
133
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
134
+ }
@@ -0,0 +1,7 @@
1
+ import type { PageInput } from "./types";
2
+ export { formatLinks, type PageInput, type PageLink } from "./types";
3
+ export declare function extractInput(html: string, opts: {
4
+ url: string;
5
+ }): Promise<PageInput>;
6
+ /** The page fingerprint: SHA-256 hex of the text and the JSON-LD. */
7
+ export declare function sourceHashOf(text: string, jsonLd: unknown[]): Promise<string>;
@@ -0,0 +1,120 @@
1
+ "use strict";
2
+ // Refine, step 1: a page's HTML -> the model's input (changes/agent-ready-engine,
3
+ // "refine"). The same recipe as the Markdown POC:
4
+ //
5
+ // text the page's <main> (<article>, then <body>, when there is none),
6
+ // late-loading Next.js sections put back first, as readable
7
+ // lines; table cells joined with " | "; images as [image: alt]
8
+ // links every http(s) link inside it, absolute, "url | text", deduped
9
+ // jsonLd every JSON-LD block on the page (top-level arrays flattened)
10
+ // sourceHash SHA-256 of `text + "\n" + JSON.stringify(jsonLd)`: the page's
11
+ // fingerprint; a new hash means the page changed
12
+ //
13
+ // Unlike convert, nothing is judged here: buttons and labels stay in, and the
14
+ // writing step decides what to leave out. NODE ONLY (linkedom).
15
+ Object.defineProperty(exports, "__esModule", { value: true });
16
+ exports.formatLinks = void 0;
17
+ exports.extractInput = extractInput;
18
+ exports.sourceHashOf = sourceHashOf;
19
+ const linkedom_1 = require("linkedom");
20
+ const convert_1 = require("../core/convert");
21
+ var types_1 = require("./types");
22
+ Object.defineProperty(exports, "formatLinks", { enumerable: true, get: function () { return types_1.formatLinks; } });
23
+ const DROP = "script,style,noscript,template,svg,canvas,iframe,object,embed,[hidden]";
24
+ const BLOCK = new Set("address article aside blockquote dd details div dl dt fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hr li main nav ol p pre section summary table tbody thead tfoot tr ul".split(" "));
25
+ async function extractInput(html, opts) {
26
+ // linkedom needs a whole document to give a <body>.
27
+ const { document } = (0, linkedom_1.parseHTML)(/<html[\s>]/i.test(html) ? html : `<!doctype html><html><head></head>${html}</html>`);
28
+ // JSON-LD first: the <script> tags are dropped below.
29
+ const jsonLd = [];
30
+ for (const s of Array.from(document.querySelectorAll('script[type="application/ld+json"]'))) {
31
+ try {
32
+ const v = JSON.parse(s.textContent || "");
33
+ if (Array.isArray(v))
34
+ jsonLd.push(...v);
35
+ else
36
+ jsonLd.push(v);
37
+ }
38
+ catch {
39
+ // Broken JSON-LD is skipped, as in convert.
40
+ }
41
+ }
42
+ (0, convert_1.unfoldNextStreaming)(document);
43
+ for (const el of Array.from(document.querySelectorAll(DROP)))
44
+ el.parentNode?.removeChild(el);
45
+ for (const el of Array.from(document.querySelectorAll('[style*="display:none"], [style*="display: none"]'))) {
46
+ el.parentNode?.removeChild(el);
47
+ }
48
+ const main = document.querySelector("main");
49
+ const article = main ? null : document.querySelector("article");
50
+ const root = main || article || document.body || document.documentElement;
51
+ const mainSource = main ? "main" : article ? "article" : "body";
52
+ const text = readableText(root);
53
+ const links = collectLinks(root, opts.url);
54
+ return { url: opts.url, text, links, jsonLd, sourceHash: await sourceHashOf(text, jsonLd), mainSource };
55
+ }
56
+ /** The page fingerprint: SHA-256 hex of the text and the JSON-LD. */
57
+ async function sourceHashOf(text, jsonLd) {
58
+ const bytes = new TextEncoder().encode(`${text}\n${JSON.stringify(jsonLd)}`);
59
+ const digest = await crypto.subtle.digest("SHA-256", bytes);
60
+ return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
61
+ }
62
+ function readableText(root) {
63
+ const out = [];
64
+ const walk = (node) => {
65
+ if (node.nodeType === 3) {
66
+ // Source newlines are layout, not content: only blocks and <br> break lines.
67
+ out.push(String(node.data).replace(/\s+/g, " "));
68
+ return;
69
+ }
70
+ if (node.nodeType !== 1)
71
+ return;
72
+ const tag = String(node.nodeName).toLowerCase();
73
+ if (tag === "br")
74
+ return void out.push("\n");
75
+ if (tag === "img") {
76
+ const alt = (node.getAttribute("alt") || "").trim();
77
+ if (alt)
78
+ out.push(` [image: ${alt}] `);
79
+ return;
80
+ }
81
+ const block = BLOCK.has(tag);
82
+ if (block)
83
+ out.push("\n");
84
+ for (const child of Array.from(node.childNodes))
85
+ walk(child);
86
+ if (tag === "td" || tag === "th")
87
+ out.push("\t");
88
+ if (block)
89
+ out.push("\n");
90
+ };
91
+ walk(root);
92
+ return out
93
+ .join("")
94
+ .split("\n")
95
+ .map((line) => line
96
+ .split("\t")
97
+ .map((cell) => cell.replace(/\s+/g, " ").trim())
98
+ .filter(Boolean)
99
+ .join(" | "))
100
+ .filter(Boolean)
101
+ .join("\n");
102
+ }
103
+ function collectLinks(root, pageUrl) {
104
+ const seen = new Set();
105
+ const links = [];
106
+ for (const a of Array.from(root.querySelectorAll("a[href]"))) {
107
+ let url;
108
+ try {
109
+ url = new URL(String(a.getAttribute("href")).trim(), pageUrl).href;
110
+ }
111
+ catch {
112
+ continue;
113
+ }
114
+ if (!/^https?:/i.test(url) || seen.has(url))
115
+ continue;
116
+ seen.add(url);
117
+ links.push({ url, text: String(a.textContent || "").replace(/\s+/g, " ").trim() });
118
+ }
119
+ return links;
120
+ }
@@ -0,0 +1,6 @@
1
+ import type { MarkdownStore } from "../core/config";
2
+ import type { RefineStore } from "./run";
3
+ export interface FileStore extends RefineStore, MarkdownStore {
4
+ dir: string;
5
+ }
6
+ export declare function fileStore(dir: string): FileStore;
@@ -0,0 +1,87 @@
1
+ "use strict";
2
+ // A folder of refined docs, like the POC's changes/markdown-poc/md/:
3
+ //
4
+ // <dir>/tool/notion.md the document
5
+ // <dir>/tool/notion.json { url, sourceHash, promptId, generatedAt, tokens }
6
+ // <dir>/index.md the home page
7
+ //
8
+ // It is a RefineStore (the runner writes it) and a MarkdownStore (the engine's
9
+ // "stored" step can read it): `store: fileStore(".agent-ready/md")` in the
10
+ // config. Rejected docs are not written here; review them from the run report.
11
+ //
12
+ // NODE ONLY. node:fs is loaded on first use, so importing this file does not
13
+ // pull fs into a bundle that never calls it.
14
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
15
+ if (k2 === undefined) k2 = k;
16
+ var desc = Object.getOwnPropertyDescriptor(m, k);
17
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
18
+ desc = { enumerable: true, get: function() { return m[k]; } };
19
+ }
20
+ Object.defineProperty(o, k2, desc);
21
+ }) : (function(o, m, k, k2) {
22
+ if (k2 === undefined) k2 = k;
23
+ o[k2] = m[k];
24
+ }));
25
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
26
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
27
+ }) : function(o, v) {
28
+ o["default"] = v;
29
+ });
30
+ var __importStar = (this && this.__importStar) || (function () {
31
+ var ownKeys = function(o) {
32
+ ownKeys = Object.getOwnPropertyNames || function (o) {
33
+ var ar = [];
34
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
35
+ return ar;
36
+ };
37
+ return ownKeys(o);
38
+ };
39
+ return function (mod) {
40
+ if (mod && mod.__esModule) return mod;
41
+ var result = {};
42
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
43
+ __setModuleDefault(result, mod);
44
+ return result;
45
+ };
46
+ })();
47
+ Object.defineProperty(exports, "__esModule", { value: true });
48
+ exports.fileStore = fileStore;
49
+ function fileStore(dir) {
50
+ const files = (path) => {
51
+ const clean = path.replace(/^\/+|\/+$/g, "");
52
+ if (clean.split("/").some((seg) => seg === ".." || seg === "." || seg.includes("\\"))) {
53
+ throw new Error(`agent-ready: bad path ${path}`);
54
+ }
55
+ const base = `${dir.replace(/[\\/]+$/, "")}/${clean || "index"}`;
56
+ return { md: `${base}.md`, meta: `${base}.json` };
57
+ };
58
+ const fs = () => Promise.resolve().then(() => __importStar(require("node:fs/promises")));
59
+ return {
60
+ dir,
61
+ async lastHash(path) {
62
+ try {
63
+ const meta = JSON.parse(await (await fs()).readFile(files(path).meta, "utf8"));
64
+ return typeof meta.sourceHash === "string" ? meta.sourceHash : null;
65
+ }
66
+ catch {
67
+ return null;
68
+ }
69
+ },
70
+ async save(doc) {
71
+ const { md, meta } = files(doc.path);
72
+ const f = await fs();
73
+ await f.mkdir(md.slice(0, md.lastIndexOf("/")), { recursive: true });
74
+ await f.writeFile(md, doc.markdown, "utf8");
75
+ const { markdown: _markdown, ...rest } = doc;
76
+ await f.writeFile(meta, JSON.stringify(rest, null, 2) + "\n", "utf8");
77
+ },
78
+ async get(path) {
79
+ try {
80
+ return await (await fs()).readFile(files(path).md, "utf8");
81
+ }
82
+ catch {
83
+ return null;
84
+ }
85
+ },
86
+ };
87
+ }
@@ -0,0 +1,30 @@
1
+ export interface ModelRequest {
2
+ system: string;
3
+ user: string;
4
+ /** Output cap. Default 8000 (documents are budgeted at 2,200-3,000 tokens). */
5
+ maxTokens?: number;
6
+ }
7
+ export interface ModelReply {
8
+ text: string;
9
+ model: string;
10
+ inputTokens: number | null;
11
+ outputTokens: number | null;
12
+ }
13
+ export type ModelClient = (req: ModelRequest) => Promise<ModelReply>;
14
+ export interface ProviderOptions {
15
+ apiKey: string;
16
+ model?: string;
17
+ baseUrl?: string;
18
+ fetch?: typeof fetch;
19
+ timeoutMs?: number;
20
+ }
21
+ export declare const DEFAULT_CLAUDE_MODEL = "claude-opus-5-5";
22
+ export declare const DEFAULT_OPENAI_MODEL = "gpt-4o";
23
+ export declare function anthropicModel(opts: ProviderOptions): ModelClient;
24
+ export declare function openaiModel(opts: ProviderOptions & {
25
+ model: string;
26
+ }): ModelClient;
27
+ /** The model the env points at, or null when no key is set. */
28
+ export declare function modelFromEnv(env: Record<string, string | undefined>, f?: typeof fetch): ModelClient | null;
29
+ /** The document only: drops a ```markdown fence or any words before the frontmatter. */
30
+ export declare function cleanReply(text: string): string;
@@ -0,0 +1,105 @@
1
+ "use strict";
2
+ // Refine, step 3: one model call per page. A model is just a function, so
3
+ // tests pass a stub and a venture can bring any provider.
4
+ //
5
+ // ANTHROPIC_API_KEY -> Claude (default model claude-opus-5-5, the model the
6
+ // POC files were written with)
7
+ // OPENAI_API_KEY -> OpenAI (default model gpt-4o, the strongest model
8
+ // Relve already uses elsewhere)
9
+ // AGENT_READY_MODEL -> the model id, e.g. claude-haiku-5-5 or gpt-4o-mini;
10
+ // a gpt-* / o* id picks OpenAI when both keys are set
11
+ //
12
+ // Plain fetch, no SDK. Errors throw with the status and a short body excerpt,
13
+ // never the key. EDGE-SAFE (global fetch only).
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.DEFAULT_OPENAI_MODEL = exports.DEFAULT_CLAUDE_MODEL = void 0;
16
+ exports.anthropicModel = anthropicModel;
17
+ exports.openaiModel = openaiModel;
18
+ exports.modelFromEnv = modelFromEnv;
19
+ exports.cleanReply = cleanReply;
20
+ exports.DEFAULT_CLAUDE_MODEL = "claude-opus-5-5";
21
+ exports.DEFAULT_OPENAI_MODEL = "gpt-4o";
22
+ function anthropicModel(opts) {
23
+ const model = opts.model ?? exports.DEFAULT_CLAUDE_MODEL;
24
+ const base = (opts.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, "");
25
+ return async (req) => {
26
+ const json = await post(opts, `${base}/v1/messages`, {
27
+ "x-api-key": opts.apiKey,
28
+ "anthropic-version": "2023-06-01",
29
+ }, {
30
+ model,
31
+ max_tokens: req.maxTokens ?? 8000,
32
+ system: req.system,
33
+ messages: [{ role: "user", content: req.user }],
34
+ });
35
+ const text = Array.isArray(json.content)
36
+ ? json.content.filter((c) => c?.type === "text").map((c) => c.text).join("")
37
+ : "";
38
+ return { text, model: json.model ?? model, inputTokens: json.usage?.input_tokens ?? null, outputTokens: json.usage?.output_tokens ?? null };
39
+ };
40
+ }
41
+ function openaiModel(opts) {
42
+ const base = (opts.baseUrl ?? "https://api.openai.com").replace(/\/+$/, "");
43
+ return async (req) => {
44
+ const json = await post(opts, `${base}/v1/chat/completions`, { authorization: `Bearer ${opts.apiKey}` }, {
45
+ model: opts.model,
46
+ max_completion_tokens: req.maxTokens ?? 8000,
47
+ messages: [
48
+ { role: "system", content: req.system },
49
+ { role: "user", content: req.user },
50
+ ],
51
+ });
52
+ return {
53
+ text: json.choices?.[0]?.message?.content ?? "",
54
+ model: json.model ?? opts.model,
55
+ inputTokens: json.usage?.prompt_tokens ?? null,
56
+ outputTokens: json.usage?.completion_tokens ?? null,
57
+ };
58
+ };
59
+ }
60
+ /** The model the env points at, or null when no key is set. */
61
+ function modelFromEnv(env, f) {
62
+ const model = env.AGENT_READY_MODEL?.trim() || undefined;
63
+ const anthropicKey = env.ANTHROPIC_API_KEY?.trim();
64
+ const openaiKey = env.OPENAI_API_KEY?.trim();
65
+ const wantsOpenai = !!model && /^(gpt|o\d)/i.test(model);
66
+ if (anthropicKey && !wantsOpenai)
67
+ return anthropicModel({ apiKey: anthropicKey, model, fetch: f });
68
+ if (openaiKey) {
69
+ if (model && !wantsOpenai)
70
+ throw new Error(`agent-ready refine: ${model} is not an OpenAI model, and ANTHROPIC_API_KEY is not set.`);
71
+ return openaiModel({ apiKey: openaiKey, model: model ?? exports.DEFAULT_OPENAI_MODEL, fetch: f });
72
+ }
73
+ return null;
74
+ }
75
+ /** The document only: drops a ```markdown fence or any words before the frontmatter. */
76
+ function cleanReply(text) {
77
+ let t = text.replace(/\r\n?/g, "\n").trim();
78
+ const fence = /^```(?:markdown|md)?\n([\s\S]*?)\n```$/.exec(t);
79
+ if (fence)
80
+ t = fence[1].trim();
81
+ const start = t.indexOf("---\n");
82
+ if (start > 0 && !t.startsWith("---"))
83
+ t = t.slice(start);
84
+ return t + "\n";
85
+ }
86
+ async function post(opts, url, headers, body) {
87
+ const f = opts.fetch ?? fetch;
88
+ const ctl = new AbortController();
89
+ const timer = setTimeout(() => ctl.abort(), opts.timeoutMs ?? 180000);
90
+ try {
91
+ const res = await f(url, {
92
+ method: "POST",
93
+ headers: { "content-type": "application/json", ...headers },
94
+ body: JSON.stringify(body),
95
+ signal: ctl.signal,
96
+ });
97
+ const text = await res.text();
98
+ if (!res.ok)
99
+ throw new Error(`model call failed: HTTP ${res.status} ${text.slice(0, 300)}`);
100
+ return JSON.parse(text);
101
+ }
102
+ finally {
103
+ clearTimeout(timer);
104
+ }
105
+ }