katman-mcp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/LICENSE +13 -0
  2. package/README.md +160 -0
  3. package/dist/cli.js +71 -0
  4. package/dist/context.js +43 -0
  5. package/dist/crawlers.js +86 -0
  6. package/dist/index.js +25 -0
  7. package/dist/license.js +448 -0
  8. package/dist/meta.js +28 -0
  9. package/dist/net.js +128 -0
  10. package/dist/server.js +17 -0
  11. package/dist/state.js +81 -0
  12. package/dist/tools/account/account.js +64 -0
  13. package/dist/tools/account/cli.js +21 -0
  14. package/dist/tools/account/index.js +48 -0
  15. package/dist/tools/account/sync.js +142 -0
  16. package/dist/tools/account/upsell.js +290 -0
  17. package/dist/tools/crawl/audit/analyze.js +719 -0
  18. package/dist/tools/crawl/audit/bots.js +114 -0
  19. package/dist/tools/crawl/audit/checks.js +126 -0
  20. package/dist/tools/crawl/audit/html.js +382 -0
  21. package/dist/tools/crawl/audit/index.js +364 -0
  22. package/dist/tools/crawl/audit/model.js +1 -0
  23. package/dist/tools/crawl/audit/report.js +199 -0
  24. package/dist/tools/crawl/audit/robots.js +199 -0
  25. package/dist/tools/crawl/audit/sitemap.js +147 -0
  26. package/dist/tools/crawl/cli.js +106 -0
  27. package/dist/tools/crawl/common/advice.js +61 -0
  28. package/dist/tools/crawl/common/findings.js +57 -0
  29. package/dist/tools/crawl/common/routes.js +80 -0
  30. package/dist/tools/crawl/common/text.js +89 -0
  31. package/dist/tools/crawl/common/urls.js +64 -0
  32. package/dist/tools/crawl/index.js +226 -0
  33. package/dist/tools/crawl/indexnow.js +271 -0
  34. package/dist/tools/crawl/keywords.js +178 -0
  35. package/dist/tools/crawl/scan/framework.js +645 -0
  36. package/dist/tools/crawl/scan/index.js +175 -0
  37. package/dist/tools/crawl/scan/project.js +180 -0
  38. package/dist/tools/crawl/scan/routes.js +354 -0
  39. package/dist/tools/crawl/scan/signals.js +414 -0
  40. package/dist/tools/method/cli.js +42 -0
  41. package/dist/tools/method/core/brief.js +302 -0
  42. package/dist/tools/method/core/extract.js +435 -0
  43. package/dist/tools/method/core/generate.js +504 -0
  44. package/dist/tools/method/core/lint.js +556 -0
  45. package/dist/tools/method/core/plan.js +405 -0
  46. package/dist/tools/method/core/prompt.js +150 -0
  47. package/dist/tools/method/core/questions.js +196 -0
  48. package/dist/tools/method/core/report.js +182 -0
  49. package/dist/tools/method/core/research-run.js +148 -0
  50. package/dist/tools/method/core/research.js +448 -0
  51. package/dist/tools/method/core/snippets.js +292 -0
  52. package/dist/tools/method/core/start.js +277 -0
  53. package/dist/tools/method/data/anatomy.js +141 -0
  54. package/dist/tools/method/data/appendices.js +202 -0
  55. package/dist/tools/method/data/cloudflare.js +192 -0
  56. package/dist/tools/method/data/evidence.js +137 -0
  57. package/dist/tools/method/data/geo-jobs.js +128 -0
  58. package/dist/tools/method/data/index.js +19 -0
  59. package/dist/tools/method/data/landing.js +67 -0
  60. package/dist/tools/method/data/lessons.js +127 -0
  61. package/dist/tools/method/data/measurement.js +142 -0
  62. package/dist/tools/method/data/method.js +141 -0
  63. package/dist/tools/method/data/offsite.js +148 -0
  64. package/dist/tools/method/data/pages.js +167 -0
  65. package/dist/tools/method/data/prompts.js +633 -0
  66. package/dist/tools/method/data/publish.js +113 -0
  67. package/dist/tools/method/data/render.js +117 -0
  68. package/dist/tools/method/data/research.js +131 -0
  69. package/dist/tools/method/data/roadmap.js +105 -0
  70. package/dist/tools/method/data/traps.js +74 -0
  71. package/dist/tools/method/data/verify.js +88 -0
  72. package/dist/tools/method/data/viral-list.js +39 -0
  73. package/dist/tools/method/index.js +209 -0
  74. package/dist/tools/method/lib/calls.js +53 -0
  75. package/dist/tools/method/lib/framework.js +525 -0
  76. package/dist/tools/method/lib/l10n.js +17 -0
  77. package/dist/tools/method/lib/routes.js +118 -0
  78. package/dist/tools/method/lib/state.js +454 -0
  79. package/dist/tools/method/lib/text.js +165 -0
  80. package/dist/tools/method/mcp/prompts.js +79 -0
  81. package/dist/tools/method/mcp/resources.js +50 -0
  82. package/dist/tools/visibility/analyze.js +463 -0
  83. package/dist/tools/visibility/cli.js +86 -0
  84. package/dist/tools/visibility/estimate.js +105 -0
  85. package/dist/tools/visibility/index.js +108 -0
  86. package/dist/tools/visibility/judge.js +156 -0
  87. package/dist/tools/visibility/manual.js +275 -0
  88. package/dist/tools/visibility/models.js +277 -0
  89. package/dist/tools/visibility/monitoring.js +165 -0
  90. package/dist/tools/visibility/openrouter.js +449 -0
  91. package/dist/tools/visibility/render.js +295 -0
  92. package/dist/tools/visibility/run.js +684 -0
  93. package/dist/tools/visibility/summary.js +164 -0
  94. package/dist/types.js +3 -0
  95. package/dist/util.js +92 -0
  96. package/package.json +54 -0
  97. package/server.json +34 -0
@@ -0,0 +1,118 @@
1
+ import { normPath } from "./text.js";
2
+ const DYNAMIC_SEG = /^(\[.*\]|:.+|\$.*|\*.*|\(.*\))$/;
3
+ function segments(p) {
4
+ return p
5
+ .split(/[?#]/)[0]
6
+ .split("/")
7
+ .filter(Boolean)
8
+ .filter((s) => !/^\(.*\)$/.test(s)); // Next.js route groups aren't URL segments
9
+ }
10
+ /** Default app paths from book appendix A, used only when there is no scan. */
11
+ export const DEFAULT_APP_PREFIXES = ["/app", "/admin", "/api", "/dashboard"];
12
+ /** App/API prefixes to Disallow (never bare prefixes: callers always add a trailing slash). */
13
+ export function appPrefixes(scan, extra = []) {
14
+ const set = new Set();
15
+ const conflicts = [];
16
+ for (const r of scan?.routes ?? []) {
17
+ if (r.kind !== "app" && r.kind !== "api")
18
+ continue;
19
+ const segs = segments(r.path);
20
+ if (!segs.length)
21
+ continue;
22
+ let prefix;
23
+ if (DYNAMIC_SEG.test(segs[0])) {
24
+ if (segs.length < 2 || DYNAMIC_SEG.test(segs[1]))
25
+ continue;
26
+ prefix = `/*/${segs[1]}`;
27
+ }
28
+ else
29
+ prefix = `/${segs[0]}`;
30
+ set.add(prefix.toLowerCase());
31
+ }
32
+ for (const p of extra) {
33
+ const s = p.trim();
34
+ if (!s || s === "/")
35
+ continue;
36
+ set.add(("/" + s.replace(/^\/+/, "").replace(/\/+$/, "")).toLowerCase());
37
+ }
38
+ const defaulted = !scan && set.size === 0;
39
+ if (defaulted)
40
+ DEFAULT_APP_PREFIXES.forEach((p) => set.add(p));
41
+ // never close a prefix that public pages live under
42
+ const publicPaths = (scan?.routes ?? []).filter((r) => r.kind === "public" || r.kind === "legal").map((r) => normPath(r.path));
43
+ const prefixes = [];
44
+ for (const p of set) {
45
+ if (p.startsWith("/*/")) {
46
+ prefixes.push(p);
47
+ continue;
48
+ }
49
+ if (publicPaths.some((pp) => pp === p || pp.startsWith(p + "/")))
50
+ conflicts.push(p);
51
+ else
52
+ prefixes.push(p);
53
+ }
54
+ return { prefixes: prefixes.sort(), conflicts, defaulted };
55
+ }
56
+ export function appPathsText(scan, extra = []) {
57
+ const a = appPrefixes(scan, extra);
58
+ if (!a.prefixes.length)
59
+ return scan ? "none found by the scan — close only real app/API paths" : DEFAULT_APP_PREFIXES.map((p) => p + "/").join(", ");
60
+ return a.prefixes.map((p) => p + "/").join(", ");
61
+ }
62
+ /** Login, sign-up, checkout, thank-you and search routes: noindex,follow (and not Disallowed). */
63
+ export function noindexRoutes(scan, closed = []) {
64
+ const out = [];
65
+ for (const r of scan?.routes ?? []) {
66
+ if (r.kind !== "auth" && r.kind !== "checkout" && r.kind !== "search")
67
+ continue;
68
+ const p = normPath(r.path);
69
+ if (closed.some((c) => !c.startsWith("/*/") && (p === c || p.startsWith(c + "/"))))
70
+ continue;
71
+ out.push(r.path);
72
+ }
73
+ return [...new Set(out)];
74
+ }
75
+ export const PAGE_PATTERNS = {
76
+ pricing: /\/(pricing|prices|plans|fiyat\w*|precios|preise|tarifs?|prezzi|precos|tarifler)(\/|$)/,
77
+ faq: /\/(faq|faqs|sss|help|questions|frequently-asked-questions|sikca-sorulan-sorular)(\/|$)/,
78
+ about: /\/(about|about-us|hakkimizda|hakkinda|company|team|uber-uns|a-propos|chi-siamo|quienes-somos|sobre)(\/|$)/,
79
+ contact: /\/(contact|contact-us|iletisim|kontakt|contacto|contatti)(\/|$)/,
80
+ privacy: /\/(privacy|privacy-policy|gizlilik\w*|kvkk|datenschutz|privacidad)(\/|$)/,
81
+ terms: /\/(terms|tos|terms-of-service|terms-and-conditions|sartlar|kosullar|kullanim-kosullari|agb)(\/|$)/,
82
+ comparison: /(\/(compare|comparison|alternatives?|vs|versus|karsilastirma|alternatif\w*)(\/|-|$))|-vs-|\/best-/,
83
+ guide: /\/(guides?|rehber\w*|blog|learn|resources|articles?)(\/|$)/,
84
+ glossary: /\/(glossary|sozluk)(\/|$)/,
85
+ };
86
+ /** Known public paths: scan routes that aren't app/auth/api + audit pages that returned 200. */
87
+ export function knownPublicPaths(scan, audit) {
88
+ const m = new Map();
89
+ for (const r of scan?.routes ?? []) {
90
+ if (r.kind === "public" || r.kind === "legal" || r.kind === "unknown")
91
+ m.set(normPath(r.path), "scan");
92
+ }
93
+ for (const p of audit?.pages ?? []) {
94
+ if (p.status >= 200 && p.status < 300)
95
+ m.set(normPath(p.url), "live");
96
+ }
97
+ return m;
98
+ }
99
+ export function findPage(paths, key) {
100
+ if (key === "home") {
101
+ const s = paths.get("/");
102
+ return s ? { path: "/", source: s } : undefined;
103
+ }
104
+ const re = PAGE_PATTERNS[key];
105
+ let best;
106
+ for (const [p, source] of paths) {
107
+ if (!re.test(p) || /[[:*$]/.test(p))
108
+ continue;
109
+ if (!best || p.length < best.path.length || (source === "live" && best.source === "scan"))
110
+ best = { path: p, source };
111
+ }
112
+ return best;
113
+ }
114
+ export function pathKnown(paths, url) {
115
+ if (!url)
116
+ return undefined;
117
+ return paths.get(normPath(url));
118
+ }
@@ -0,0 +1,454 @@
1
+ // Defensive readers for the .katman/ files other modules write (scan, audits, runs) and for plan/research.
2
+ // Those files may be missing, partial or hand-edited: every reader returns a well-formed shape or undefined.
3
+ import path from "node:path";
4
+ import { existsSync, statSync, promises as fs } from "node:fs";
5
+ import { KATMAN_DIR, katmanPath, listFiles, readJson } from "../../../state.js";
6
+ import { domainOf } from "../../../util.js";
7
+ const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
8
+ const arr = (x) => (Array.isArray(x) ? x : []);
9
+ const str = (x) => (typeof x === "string" ? x : undefined);
10
+ const num = (x, d = 0) => (typeof x === "number" && Number.isFinite(x) ? x : d);
11
+ const bool = (x) => x === true;
12
+ export function hasKatmanDir(root) {
13
+ return existsSync(path.join(root, KATMAN_DIR));
14
+ }
15
+ // ---------------------------------------------------------------------------
16
+ // research
17
+ export function normalizeResearch(raw) {
18
+ if (!isObj(raw))
19
+ return undefined;
20
+ const site = isObj(raw.site) ? raw.site : {};
21
+ const brand = isObj(raw.brand) ? raw.brand : {};
22
+ const url = str(site.url) ?? "";
23
+ const r = {
24
+ version: 1,
25
+ updatedAt: str(raw.updatedAt) ?? "",
26
+ site: {
27
+ url,
28
+ domain: str(site.domain) ?? (url && /^https?:/i.test(url) ? domainOf(url) : ""),
29
+ languages: arr(site.languages).filter((x) => typeof x === "string"),
30
+ ...(Array.isArray(site.previewUrls) ? { previewUrls: arr(site.previewUrls).filter((x) => typeof x === "string") } : {}),
31
+ },
32
+ brand: {
33
+ ...brand,
34
+ name: str(brand.name) ?? "",
35
+ category: str(brand.category) ?? "",
36
+ },
37
+ situations: arr(raw.situations)
38
+ .filter((s) => isObj(s) && typeof s.sentence === "string")
39
+ .map((s) => ({
40
+ id: str(s.id) ?? "",
41
+ sentence: String(s.sentence),
42
+ ...(str(s.url) ? { url: String(s.url) } : {}),
43
+ ...(s.status === "live" || s.status === "planned" ? { status: s.status } : {}),
44
+ })),
45
+ competitors: arr(raw.competitors)
46
+ .filter((c) => isObj(c) && typeof c.name === "string")
47
+ .map((c) => ({ ...c, name: String(c.name), url: str(c.url) ?? "" })),
48
+ proof: arr(raw.proof)
49
+ .filter((p) => isObj(p) && typeof p.claim === "string")
50
+ .map((p) => ({ ...p, claim: String(p.claim), source: str(p.source) ?? "" })),
51
+ keywords: arr(raw.keywords)
52
+ .filter((k) => isObj(k) && typeof k.term === "string")
53
+ .map((k) => k),
54
+ questions: arr(raw.questions)
55
+ .filter((q) => isObj(q) && typeof q.text === "string")
56
+ .map((q) => ({
57
+ id: str(q.id) ?? "",
58
+ type: (str(q.type) ?? "situation"),
59
+ text: String(q.text),
60
+ ...(str(q.lang) ? { lang: String(q.lang) } : {}),
61
+ })),
62
+ };
63
+ return r;
64
+ }
65
+ export async function loadResearchSafe(root) {
66
+ return normalizeResearch(await readJson(root, "research.json"));
67
+ }
68
+ export async function loadConfigSafe(root) {
69
+ const c = await readJson(root, "config.json");
70
+ return isObj(c) ? c : {};
71
+ }
72
+ // ---------------------------------------------------------------------------
73
+ // scan.json
74
+ function fileInfo(x) {
75
+ if (!isObj(x))
76
+ return { exists: false };
77
+ return { exists: bool(x.exists), ...(str(x.path) ? { path: str(x.path) } : {}), ...(x.generated === true ? { generated: true } : {}) };
78
+ }
79
+ export function normalizeFinding(x) {
80
+ if (!isObj(x) || typeof x.title !== "string")
81
+ return undefined;
82
+ const sev = ["critical", "high", "medium", "low", "info"].includes(String(x.severity)) ? x.severity : "medium";
83
+ const gate = ["read", "quote", "fit", "mention", "measure"].includes(String(x.gate)) ? x.gate : "read";
84
+ const prompt = /^P([0-9]|1[0-4])$/.test(String(x.prompt)) ? x.prompt : undefined;
85
+ return {
86
+ id: str(x.id) ?? x.title.toLowerCase().replace(/[^a-z0-9]+/g, "-").slice(0, 40),
87
+ title: x.title,
88
+ severity: sev,
89
+ gate,
90
+ detail: str(x.detail) ?? "",
91
+ ...(Array.isArray(x.evidence) ? { evidence: arr(x.evidence).map(String) } : {}),
92
+ ...(Array.isArray(x.files) ? { files: arr(x.files).map(String) } : {}),
93
+ ...(Array.isArray(x.urls) ? { urls: arr(x.urls).map(String) } : {}),
94
+ ...(str(x.fix) ? { fix: str(x.fix) } : {}),
95
+ ...(prompt ? { prompt } : {}),
96
+ ...(str(x.accept) ? { accept: str(x.accept) } : {}),
97
+ };
98
+ }
99
+ export function normalizeScan(raw, root) {
100
+ if (!isObj(raw))
101
+ return undefined;
102
+ const fw = isObj(raw.framework) ? raw.framework : {};
103
+ const rend = isObj(raw.rendering) ? raw.rendering : {};
104
+ const host = isObj(raw.hosting) ? raw.hosting : {};
105
+ const files = isObj(raw.files) ? raw.files : {};
106
+ const i18n = isObj(raw.i18n) ? raw.i18n : {};
107
+ const an = isObj(raw.analytics) ? raw.analytics : {};
108
+ const verdicts = ["ssr", "ssg", "hybrid", "csr", "static-html", "unknown"];
109
+ const kinds = ["public", "app", "auth", "api", "legal", "checkout", "search", "unknown"];
110
+ return {
111
+ version: 1,
112
+ ts: str(raw.ts) ?? "",
113
+ root: str(raw.root) ?? root,
114
+ framework: {
115
+ name: str(fw.name) ?? "unknown",
116
+ ...(str(fw.version) ? { version: str(fw.version) } : {}),
117
+ ...(str(fw.router) ? { router: str(fw.router) } : {}),
118
+ ...(str(fw.builder) ? { builder: str(fw.builder) } : {}),
119
+ },
120
+ rendering: {
121
+ verdict: (verdicts.includes(String(rend.verdict)) ? rend.verdict : "unknown"),
122
+ confidence: (["high", "medium", "low"].includes(String(rend.confidence)) ? rend.confidence : "low"),
123
+ evidence: arr(rend.evidence).map(String),
124
+ },
125
+ hosting: {
126
+ ...(str(host.provider) ? { provider: str(host.provider) } : {}),
127
+ previewPatterns: arr(host.previewPatterns).map(String),
128
+ evidence: arr(host.evidence).map(String),
129
+ },
130
+ ...(typeof raw.staticDir === "string" ? { staticDir: raw.staticDir } : {}),
131
+ routes: arr(raw.routes)
132
+ .filter((r) => isObj(r) && typeof r.path === "string")
133
+ .map((r) => ({
134
+ path: String(r.path),
135
+ ...(str(r.file) ? { file: str(r.file) } : {}),
136
+ kind: (kinds.includes(String(r.kind)) ? r.kind : "unknown"),
137
+ dynamic: bool(r.dynamic) || /[[:*$]/.test(String(r.path)),
138
+ })),
139
+ files: {
140
+ robots: fileInfo(files.robots),
141
+ sitemap: fileInfo(files.sitemap),
142
+ llms: fileInfo(files.llms),
143
+ llmsFull: fileInfo(files.llmsFull),
144
+ notFound: fileInfo(files.notFound),
145
+ ...(isObj(files.headSource) ? { headSource: fileInfo(files.headSource) } : {}),
146
+ },
147
+ jsonLdTypes: arr(raw.jsonLdTypes).map(String),
148
+ i18n: {
149
+ detected: bool(i18n.detected),
150
+ ...(str(i18n.library) ? { library: str(i18n.library) } : {}),
151
+ locales: arr(i18n.locales).map(String),
152
+ ...(str(i18n.htmlLang) ? { htmlLang: str(i18n.htmlLang) } : {}),
153
+ },
154
+ analytics: { providers: arr(an.providers).map(String), tracksAiChannel: bool(an.tracksAiChannel) },
155
+ findings: arr(raw.findings).map(normalizeFinding).filter((f) => !!f),
156
+ ...(Array.isArray(raw.domains) ? { domains: arr(raw.domains).filter((d) => typeof d === "string" && d.includes(".")) } : {}),
157
+ };
158
+ }
159
+ export async function loadScan(root) {
160
+ return normalizeScan(await readJson(root, "scan.json"), root);
161
+ }
162
+ // ---------------------------------------------------------------------------
163
+ // audits/<stamp>.json
164
+ function normalizePage(x) {
165
+ if (!isObj(x) || typeof x.url !== "string")
166
+ return undefined;
167
+ return {
168
+ url: x.url,
169
+ status: num(x.status),
170
+ ...(str(x.contentType) ? { contentType: str(x.contentType) } : {}),
171
+ noindex: bool(x.noindex),
172
+ ...(str(x.title) !== undefined ? { title: str(x.title) } : {}),
173
+ ...(str(x.description) !== undefined ? { description: str(x.description) } : {}),
174
+ ...(str(x.canonical) ? { canonical: str(x.canonical) } : {}),
175
+ ...(typeof x.canonicalSelf === "boolean" ? { canonicalSelf: x.canonicalSelf } : {}),
176
+ h1Count: num(x.h1Count),
177
+ headingSkips: num(x.headingSkips),
178
+ wordsJsOff: num(x.wordsJsOff),
179
+ ...(str(x.firstParagraph) ? { firstParagraph: str(x.firstParagraph) } : {}),
180
+ jsonLdTypes: arr(x.jsonLdTypes).map(String),
181
+ jsonLdErrors: num(x.jsonLdErrors),
182
+ ...(str(x.ogImage) ? { ogImage: str(x.ogImage) } : {}),
183
+ ...(str(x.lang) ? { lang: str(x.lang) } : {}),
184
+ hreflang: arr(x.hreflang).filter(isObj).map((h) => ({ lang: String(h.lang ?? ""), href: String(h.href ?? "") })),
185
+ imagesMissingAlt: num(x.imagesMissingAlt),
186
+ internalLinksOut: num(x.internalLinksOut),
187
+ internalLinksIn: num(x.internalLinksIn),
188
+ inSitemap: bool(x.inSitemap),
189
+ ...(str(x.dateModified) ? { dateModified: str(x.dateModified) } : {}),
190
+ bytes: num(x.bytes),
191
+ ms: num(x.ms),
192
+ };
193
+ }
194
+ function gate(x) {
195
+ if (!isObj(x))
196
+ return { pass: 0, total: 0, notes: [] };
197
+ return { pass: num(x.pass), total: num(x.total), notes: arr(x.notes).map(String) };
198
+ }
199
+ export function normalizeAudit(raw) {
200
+ if (!isObj(raw))
201
+ return undefined;
202
+ const site = isObj(raw.site) ? raw.site : {};
203
+ const robots = isObj(site.robots) ? site.robots : {};
204
+ const llms = isObj(site.llmsTxt) ? site.llmsTxt : {};
205
+ const gates = isObj(raw.gates) ? raw.gates : {};
206
+ return {
207
+ version: 1,
208
+ ts: str(raw.ts) ?? "",
209
+ siteUrl: str(raw.siteUrl) ?? "",
210
+ pages: arr(raw.pages).map(normalizePage).filter((p) => !!p),
211
+ site: {
212
+ ...(typeof site.httpsRedirect === "boolean" ? { httpsRedirect: site.httpsRedirect } : {}),
213
+ ...(typeof site.hostRedirect === "boolean" ? { hostRedirect: site.hostRedirect } : {}),
214
+ ...(typeof site.hsts === "boolean" ? { hsts: site.hsts } : {}),
215
+ ...(site.compression !== undefined ? { compression: (str(site.compression) ?? null) } : {}),
216
+ ...(typeof site.soft404 === "boolean" ? { soft404: site.soft404 } : {}),
217
+ llmsTxt: { exists: bool(llms.exists), ...(typeof llms.bytes === "number" ? { bytes: llms.bytes } : {}) },
218
+ robots: {
219
+ exists: bool(robots.exists),
220
+ sitemaps: arr(robots.sitemaps).map(String),
221
+ blocksAll: bool(robots.blocksAll),
222
+ blockedAiTokens: arr(robots.blockedAiTokens).map(String),
223
+ invalidLines: arr(robots.invalidLines).map(String),
224
+ },
225
+ sitemapUrls: num(site.sitemapUrls),
226
+ ...(str(site.cdn) ? { cdn: str(site.cdn) } : {}),
227
+ bots: arr(site.bots)
228
+ .filter(isObj)
229
+ .map((b) => ({
230
+ token: String(b.token ?? ""),
231
+ purpose: String(b.purpose ?? ""),
232
+ status: num(b.status),
233
+ blocked: bool(b.blocked),
234
+ ...(str(b.reason) ? { reason: str(b.reason) } : {}),
235
+ })),
236
+ },
237
+ gates: { read: gate(gates.read), quote: gate(gates.quote), fit: gate(gates.fit) },
238
+ findings: arr(raw.findings).map(normalizeFinding).filter((f) => !!f),
239
+ };
240
+ }
241
+ /** Latest audit by file name (stamps sort chronologically). Skips files that don't parse. */
242
+ export async function loadLatestAudit(root) {
243
+ const names = (await listFiles(root, "audits")).filter((n) => n.endsWith(".json"));
244
+ for (let i = names.length - 1; i >= 0; i--) {
245
+ const a = normalizeAudit(await readJson(root, path.join("audits", names[i])));
246
+ if (a)
247
+ return { audit: a, file: names[i], count: names.length };
248
+ }
249
+ return undefined;
250
+ }
251
+ // ---------------------------------------------------------------------------
252
+ // runs/<stamp>.json
253
+ function normalizeAnswer(x) {
254
+ if (!isObj(x))
255
+ return undefined;
256
+ return {
257
+ questionId: String(x.questionId ?? ""),
258
+ question: String(x.question ?? ""),
259
+ questionType: (str(x.questionType) ?? "situation"),
260
+ model: String(x.model ?? ""),
261
+ family: String(x.family ?? x.model ?? ""),
262
+ source: x.source === "manual" ? "manual" : "api",
263
+ ...(str(x.surface) ? { surface: str(x.surface) } : {}),
264
+ ts: String(x.ts ?? ""),
265
+ ...(typeof x.temperature === "number" ? { temperature: x.temperature } : {}),
266
+ mentioned: bool(x.mentioned),
267
+ recommendedFirst: bool(x.recommendedFirst),
268
+ rank: typeof x.rank === "number" ? x.rank : null,
269
+ cited: bool(x.cited),
270
+ namedProducts: arr(x.namedProducts).map(String),
271
+ competitorsNamed: arr(x.competitorsNamed).map(String),
272
+ citations: arr(x.citations)
273
+ .filter(isObj)
274
+ .map((c) => ({ url: String(c.url ?? ""), ...(str(c.title) ? { title: str(c.title) } : {}), domain: String(c.domain ?? domainOf(String(c.url ?? ""))) })),
275
+ answerExcerpt: String(x.answerExcerpt ?? ""),
276
+ ...(typeof x.identityCorrect === "boolean" || x.identityCorrect === null ? { identityCorrect: x.identityCorrect } : {}),
277
+ ...(typeof x.costUsd === "number" ? { costUsd: x.costUsd } : {}),
278
+ ...(str(x.error) ? { error: str(x.error) } : {}),
279
+ ...(typeof x.searched === "boolean" ? { searched: x.searched } : {}),
280
+ ...(typeof x.sample === "number" ? { sample: x.sample } : {}),
281
+ ...(str(x.lang) ? { lang: str(x.lang) } : {}),
282
+ };
283
+ }
284
+ /** Recompute the summary from answers (used when a run file has none or a partial one). */
285
+ export function summarizeAnswers(answers) {
286
+ const ok = answers.filter((a) => !a.error);
287
+ const byFamily = {};
288
+ const byLanguage = {};
289
+ const comp = new Map();
290
+ const doms = new Map();
291
+ for (const a of ok) {
292
+ const buckets = [(byFamily[a.family || a.model] ??= { answered: 0, mentioned: 0, recommendedFirst: 0, cited: 0 })];
293
+ if (a.lang)
294
+ buckets.push((byLanguage[a.lang] ??= { answered: 0, mentioned: 0, recommendedFirst: 0, cited: 0 }));
295
+ for (const f of buckets) {
296
+ f.answered++;
297
+ if (a.mentioned)
298
+ f.mentioned++;
299
+ if (a.recommendedFirst)
300
+ f.recommendedFirst++;
301
+ if (a.cited)
302
+ f.cited++;
303
+ }
304
+ for (const c of a.competitorsNamed)
305
+ comp.set(c, (comp.get(c) ?? 0) + 1);
306
+ for (const c of a.citations)
307
+ if (c.domain)
308
+ doms.set(c.domain, (doms.get(c.domain) ?? 0) + 1);
309
+ }
310
+ const n = ok.length || 1;
311
+ const top = (m) => [...m.entries()].sort((a, b) => b[1] - a[1]).slice(0, 15);
312
+ return {
313
+ answered: ok.length,
314
+ mentionRate: ok.filter((a) => a.mentioned).length / n,
315
+ recommendedFirstRate: ok.filter((a) => a.recommendedFirst).length / n,
316
+ citationRate: ok.filter((a) => a.cited).length / n,
317
+ byFamily,
318
+ topCompetitors: top(comp).map(([name, count]) => ({ name, count })),
319
+ topCitedDomains: top(doms).map(([domain, count]) => ({ domain, count })),
320
+ ...(Object.keys(byLanguage).length ? { byLanguage } : {}),
321
+ };
322
+ }
323
+ export function normalizeRun(raw, fallbackId) {
324
+ if (!isObj(raw))
325
+ return undefined;
326
+ const answers = arr(raw.answers).map(normalizeAnswer).filter((a) => !!a);
327
+ const s = isObj(raw.summary) ? raw.summary : undefined;
328
+ const computed = summarizeAnswers(answers);
329
+ const summary = s
330
+ ? {
331
+ answered: typeof s.answered === "number" ? s.answered : computed.answered,
332
+ mentionRate: typeof s.mentionRate === "number" ? s.mentionRate : computed.mentionRate,
333
+ recommendedFirstRate: typeof s.recommendedFirstRate === "number" ? s.recommendedFirstRate : computed.recommendedFirstRate,
334
+ citationRate: typeof s.citationRate === "number" ? s.citationRate : computed.citationRate,
335
+ byFamily: isObj(s.byFamily) ? s.byFamily : computed.byFamily,
336
+ topCompetitors: Array.isArray(s.topCompetitors) ? s.topCompetitors : computed.topCompetitors,
337
+ topCitedDomains: Array.isArray(s.topCitedDomains) ? s.topCitedDomains : computed.topCitedDomains,
338
+ ...(isObj(s.byLanguage)
339
+ ? { byLanguage: s.byLanguage }
340
+ : computed.byLanguage
341
+ ? { byLanguage: computed.byLanguage }
342
+ : {}),
343
+ }
344
+ : computed;
345
+ return {
346
+ id: str(raw.id) ?? fallbackId,
347
+ ts: str(raw.ts) ?? "",
348
+ brand: String(raw.brand ?? ""),
349
+ domain: String(raw.domain ?? ""),
350
+ models: arr(raw.models)
351
+ .filter(isObj)
352
+ .map((m) => ({ id: String(m.id ?? ""), family: String(m.family ?? m.id ?? ""), search: (["native", "plugin", "none"].includes(String(m.search)) ? m.search : "none") })),
353
+ temperature: typeof raw.temperature === "number" ? raw.temperature : null,
354
+ ...(raw.status === "complete" || raw.status === "partial" ? { status: raw.status } : {}),
355
+ answers,
356
+ summary,
357
+ ...(typeof raw.totalCostUsd === "number" ? { totalCostUsd: raw.totalCostUsd } : {}),
358
+ };
359
+ }
360
+ /** All runs, oldest first. */
361
+ export async function loadRuns(root) {
362
+ const names = (await listFiles(root, "runs")).filter((n) => n.endsWith(".json"));
363
+ const out = [];
364
+ for (const n of names) {
365
+ const r = normalizeRun(await readJson(root, path.join("runs", n)), n.replace(/\.json$/, ""));
366
+ if (r)
367
+ out.push(r);
368
+ }
369
+ return out.sort((a, b) => (a.ts || a.id).localeCompare(b.ts || b.id));
370
+ }
371
+ export function runDate(r) {
372
+ const d = new Date(r.ts);
373
+ if (!Number.isNaN(d.getTime()))
374
+ return d;
375
+ const m = /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})Z/.exec(r.id);
376
+ if (m)
377
+ return new Date(`${m[1]}T${m[2]}:${m[3]}:${m[4]}Z`);
378
+ return undefined;
379
+ }
380
+ // ---------------------------------------------------------------------------
381
+ // project files (read-only, never secrets)
382
+ const SECRET_RE = /(^|[\\/])(\.env[^\\/]*|\.dev\.vars[^\\/]*)$|\.(pem|key|p12|pfx)$|secret|credential/i;
383
+ export function isSecretPath(p) {
384
+ return SECRET_RE.test(p);
385
+ }
386
+ /** Resolve a path relative to the project root; undefined when it escapes the root. */
387
+ export function insideRoot(root, rel) {
388
+ const base = path.resolve(root);
389
+ const p = path.resolve(base, rel);
390
+ if (p !== base && !p.startsWith(base + path.sep))
391
+ return undefined;
392
+ return p;
393
+ }
394
+ export async function readProjectFile(root, rel, maxBytes = 2_000_000) {
395
+ const abs = insideRoot(root, rel);
396
+ if (!abs)
397
+ return { error: `${rel} is outside the project root ${root}.` };
398
+ if (isSecretPath(abs))
399
+ return { error: `Katman never reads secret files (${path.basename(abs)}).` };
400
+ try {
401
+ const st = await fs.stat(abs);
402
+ if (!st.isFile())
403
+ return { abs, error: `${rel} is not a file.` };
404
+ if (st.size > maxBytes)
405
+ return { abs, error: `${rel} is larger than ${Math.round(maxBytes / 1e6)} MB.` };
406
+ return { abs, text: await fs.readFile(abs, "utf8") };
407
+ }
408
+ catch {
409
+ return { abs, error: `${rel} not found.` };
410
+ }
411
+ }
412
+ export async function mtimeOf(p) {
413
+ try {
414
+ return (await fs.stat(p)).mtime;
415
+ }
416
+ catch {
417
+ return undefined;
418
+ }
419
+ }
420
+ export async function katmanMtime(root, rel) {
421
+ return mtimeOf(katmanPath(root, rel));
422
+ }
423
+ // ---------------------------------------------------------------------------
424
+ // static dir (folder served as-is at the site root)
425
+ const STATIC_BY_FRAMEWORK = {
426
+ sveltekit: "static", hugo: "static", gatsby: "static", jekyll: "", eleventy: "", "static-html": "",
427
+ };
428
+ export function cleanRel(p) {
429
+ const s = p.replace(/\\/g, "/").replace(/^\.\/?/, "").replace(/\/+$/, "");
430
+ return s === "." ? "" : s;
431
+ }
432
+ /** scan.staticDir → existing public/ or static/ → framework convention → project root. */
433
+ export function staticDirFor(root, scan) {
434
+ if (scan && typeof scan.staticDir === "string")
435
+ return { dir: cleanRel(scan.staticDir), source: "scan" };
436
+ for (const d of ["public", "static"]) {
437
+ try {
438
+ if (statSync(path.join(root, d)).isDirectory())
439
+ return { dir: d, source: "detected" };
440
+ }
441
+ catch {
442
+ // not there
443
+ }
444
+ }
445
+ const name = scan?.framework.name ?? "";
446
+ if (name in STATIC_BY_FRAMEWORK)
447
+ return { dir: STATIC_BY_FRAMEWORK[name], source: "convention" };
448
+ if (name && name !== "unknown")
449
+ return { dir: "public", source: "convention" };
450
+ return { dir: "", source: "root" };
451
+ }
452
+ export function joinRel(...parts) {
453
+ return parts.filter((p) => p !== "").join("/").replace(/\/+/g, "/");
454
+ }