katman-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +13 -0
- package/README.md +160 -0
- package/dist/cli.js +71 -0
- package/dist/context.js +43 -0
- package/dist/crawlers.js +86 -0
- package/dist/index.js +25 -0
- package/dist/license.js +448 -0
- package/dist/meta.js +28 -0
- package/dist/net.js +128 -0
- package/dist/server.js +17 -0
- package/dist/state.js +81 -0
- package/dist/tools/account/account.js +64 -0
- package/dist/tools/account/cli.js +21 -0
- package/dist/tools/account/index.js +48 -0
- package/dist/tools/account/sync.js +142 -0
- package/dist/tools/account/upsell.js +290 -0
- package/dist/tools/crawl/audit/analyze.js +719 -0
- package/dist/tools/crawl/audit/bots.js +114 -0
- package/dist/tools/crawl/audit/checks.js +126 -0
- package/dist/tools/crawl/audit/html.js +382 -0
- package/dist/tools/crawl/audit/index.js +364 -0
- package/dist/tools/crawl/audit/model.js +1 -0
- package/dist/tools/crawl/audit/report.js +199 -0
- package/dist/tools/crawl/audit/robots.js +199 -0
- package/dist/tools/crawl/audit/sitemap.js +147 -0
- package/dist/tools/crawl/cli.js +106 -0
- package/dist/tools/crawl/common/advice.js +61 -0
- package/dist/tools/crawl/common/findings.js +57 -0
- package/dist/tools/crawl/common/routes.js +80 -0
- package/dist/tools/crawl/common/text.js +89 -0
- package/dist/tools/crawl/common/urls.js +64 -0
- package/dist/tools/crawl/index.js +226 -0
- package/dist/tools/crawl/indexnow.js +271 -0
- package/dist/tools/crawl/keywords.js +178 -0
- package/dist/tools/crawl/scan/framework.js +645 -0
- package/dist/tools/crawl/scan/index.js +175 -0
- package/dist/tools/crawl/scan/project.js +180 -0
- package/dist/tools/crawl/scan/routes.js +354 -0
- package/dist/tools/crawl/scan/signals.js +414 -0
- package/dist/tools/method/cli.js +42 -0
- package/dist/tools/method/core/brief.js +302 -0
- package/dist/tools/method/core/extract.js +435 -0
- package/dist/tools/method/core/generate.js +504 -0
- package/dist/tools/method/core/lint.js +556 -0
- package/dist/tools/method/core/plan.js +405 -0
- package/dist/tools/method/core/prompt.js +150 -0
- package/dist/tools/method/core/questions.js +196 -0
- package/dist/tools/method/core/report.js +182 -0
- package/dist/tools/method/core/research-run.js +148 -0
- package/dist/tools/method/core/research.js +448 -0
- package/dist/tools/method/core/snippets.js +292 -0
- package/dist/tools/method/core/start.js +277 -0
- package/dist/tools/method/data/anatomy.js +141 -0
- package/dist/tools/method/data/appendices.js +202 -0
- package/dist/tools/method/data/cloudflare.js +192 -0
- package/dist/tools/method/data/evidence.js +137 -0
- package/dist/tools/method/data/geo-jobs.js +128 -0
- package/dist/tools/method/data/index.js +19 -0
- package/dist/tools/method/data/landing.js +67 -0
- package/dist/tools/method/data/lessons.js +127 -0
- package/dist/tools/method/data/measurement.js +142 -0
- package/dist/tools/method/data/method.js +141 -0
- package/dist/tools/method/data/offsite.js +148 -0
- package/dist/tools/method/data/pages.js +167 -0
- package/dist/tools/method/data/prompts.js +633 -0
- package/dist/tools/method/data/publish.js +113 -0
- package/dist/tools/method/data/render.js +117 -0
- package/dist/tools/method/data/research.js +131 -0
- package/dist/tools/method/data/roadmap.js +105 -0
- package/dist/tools/method/data/traps.js +74 -0
- package/dist/tools/method/data/verify.js +88 -0
- package/dist/tools/method/data/viral-list.js +39 -0
- package/dist/tools/method/index.js +209 -0
- package/dist/tools/method/lib/calls.js +53 -0
- package/dist/tools/method/lib/framework.js +525 -0
- package/dist/tools/method/lib/l10n.js +17 -0
- package/dist/tools/method/lib/routes.js +118 -0
- package/dist/tools/method/lib/state.js +454 -0
- package/dist/tools/method/lib/text.js +165 -0
- package/dist/tools/method/mcp/prompts.js +79 -0
- package/dist/tools/method/mcp/resources.js +50 -0
- package/dist/tools/visibility/analyze.js +463 -0
- package/dist/tools/visibility/cli.js +86 -0
- package/dist/tools/visibility/estimate.js +105 -0
- package/dist/tools/visibility/index.js +108 -0
- package/dist/tools/visibility/judge.js +156 -0
- package/dist/tools/visibility/manual.js +275 -0
- package/dist/tools/visibility/models.js +277 -0
- package/dist/tools/visibility/monitoring.js +165 -0
- package/dist/tools/visibility/openrouter.js +449 -0
- package/dist/tools/visibility/render.js +295 -0
- package/dist/tools/visibility/run.js +684 -0
- package/dist/tools/visibility/summary.js +164 -0
- package/dist/types.js +3 -0
- package/dist/util.js +92 -0
- package/package.json +54 -0
- package/server.json +34 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { normPath } from "./text.js";
|
|
2
|
+
const DYNAMIC_SEG = /^(\[.*\]|:.+|\$.*|\*.*|\(.*\))$/;
|
|
3
|
+
function segments(p) {
|
|
4
|
+
return p
|
|
5
|
+
.split(/[?#]/)[0]
|
|
6
|
+
.split("/")
|
|
7
|
+
.filter(Boolean)
|
|
8
|
+
.filter((s) => !/^\(.*\)$/.test(s)); // Next.js route groups aren't URL segments
|
|
9
|
+
}
|
|
10
|
+
/** Default app paths from book appendix A, used only when there is no scan. */
|
|
11
|
+
export const DEFAULT_APP_PREFIXES = ["/app", "/admin", "/api", "/dashboard"];
|
|
12
|
+
/** App/API prefixes to Disallow (never bare prefixes: callers always add a trailing slash). */
|
|
13
|
+
export function appPrefixes(scan, extra = []) {
|
|
14
|
+
const set = new Set();
|
|
15
|
+
const conflicts = [];
|
|
16
|
+
for (const r of scan?.routes ?? []) {
|
|
17
|
+
if (r.kind !== "app" && r.kind !== "api")
|
|
18
|
+
continue;
|
|
19
|
+
const segs = segments(r.path);
|
|
20
|
+
if (!segs.length)
|
|
21
|
+
continue;
|
|
22
|
+
let prefix;
|
|
23
|
+
if (DYNAMIC_SEG.test(segs[0])) {
|
|
24
|
+
if (segs.length < 2 || DYNAMIC_SEG.test(segs[1]))
|
|
25
|
+
continue;
|
|
26
|
+
prefix = `/*/${segs[1]}`;
|
|
27
|
+
}
|
|
28
|
+
else
|
|
29
|
+
prefix = `/${segs[0]}`;
|
|
30
|
+
set.add(prefix.toLowerCase());
|
|
31
|
+
}
|
|
32
|
+
for (const p of extra) {
|
|
33
|
+
const s = p.trim();
|
|
34
|
+
if (!s || s === "/")
|
|
35
|
+
continue;
|
|
36
|
+
set.add(("/" + s.replace(/^\/+/, "").replace(/\/+$/, "")).toLowerCase());
|
|
37
|
+
}
|
|
38
|
+
const defaulted = !scan && set.size === 0;
|
|
39
|
+
if (defaulted)
|
|
40
|
+
DEFAULT_APP_PREFIXES.forEach((p) => set.add(p));
|
|
41
|
+
// never close a prefix that public pages live under
|
|
42
|
+
const publicPaths = (scan?.routes ?? []).filter((r) => r.kind === "public" || r.kind === "legal").map((r) => normPath(r.path));
|
|
43
|
+
const prefixes = [];
|
|
44
|
+
for (const p of set) {
|
|
45
|
+
if (p.startsWith("/*/")) {
|
|
46
|
+
prefixes.push(p);
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (publicPaths.some((pp) => pp === p || pp.startsWith(p + "/")))
|
|
50
|
+
conflicts.push(p);
|
|
51
|
+
else
|
|
52
|
+
prefixes.push(p);
|
|
53
|
+
}
|
|
54
|
+
return { prefixes: prefixes.sort(), conflicts, defaulted };
|
|
55
|
+
}
|
|
56
|
+
export function appPathsText(scan, extra = []) {
|
|
57
|
+
const a = appPrefixes(scan, extra);
|
|
58
|
+
if (!a.prefixes.length)
|
|
59
|
+
return scan ? "none found by the scan — close only real app/API paths" : DEFAULT_APP_PREFIXES.map((p) => p + "/").join(", ");
|
|
60
|
+
return a.prefixes.map((p) => p + "/").join(", ");
|
|
61
|
+
}
|
|
62
|
+
/** Login, sign-up, checkout, thank-you and search routes: noindex,follow (and not Disallowed). */
|
|
63
|
+
export function noindexRoutes(scan, closed = []) {
|
|
64
|
+
const out = [];
|
|
65
|
+
for (const r of scan?.routes ?? []) {
|
|
66
|
+
if (r.kind !== "auth" && r.kind !== "checkout" && r.kind !== "search")
|
|
67
|
+
continue;
|
|
68
|
+
const p = normPath(r.path);
|
|
69
|
+
if (closed.some((c) => !c.startsWith("/*/") && (p === c || p.startsWith(c + "/"))))
|
|
70
|
+
continue;
|
|
71
|
+
out.push(r.path);
|
|
72
|
+
}
|
|
73
|
+
return [...new Set(out)];
|
|
74
|
+
}
|
|
75
|
+
export const PAGE_PATTERNS = {
|
|
76
|
+
pricing: /\/(pricing|prices|plans|fiyat\w*|precios|preise|tarifs?|prezzi|precos|tarifler)(\/|$)/,
|
|
77
|
+
faq: /\/(faq|faqs|sss|help|questions|frequently-asked-questions|sikca-sorulan-sorular)(\/|$)/,
|
|
78
|
+
about: /\/(about|about-us|hakkimizda|hakkinda|company|team|uber-uns|a-propos|chi-siamo|quienes-somos|sobre)(\/|$)/,
|
|
79
|
+
contact: /\/(contact|contact-us|iletisim|kontakt|contacto|contatti)(\/|$)/,
|
|
80
|
+
privacy: /\/(privacy|privacy-policy|gizlilik\w*|kvkk|datenschutz|privacidad)(\/|$)/,
|
|
81
|
+
terms: /\/(terms|tos|terms-of-service|terms-and-conditions|sartlar|kosullar|kullanim-kosullari|agb)(\/|$)/,
|
|
82
|
+
comparison: /(\/(compare|comparison|alternatives?|vs|versus|karsilastirma|alternatif\w*)(\/|-|$))|-vs-|\/best-/,
|
|
83
|
+
guide: /\/(guides?|rehber\w*|blog|learn|resources|articles?)(\/|$)/,
|
|
84
|
+
glossary: /\/(glossary|sozluk)(\/|$)/,
|
|
85
|
+
};
|
|
86
|
+
/** Known public paths: scan routes that aren't app/auth/api + audit pages that returned 200. */
|
|
87
|
+
export function knownPublicPaths(scan, audit) {
|
|
88
|
+
const m = new Map();
|
|
89
|
+
for (const r of scan?.routes ?? []) {
|
|
90
|
+
if (r.kind === "public" || r.kind === "legal" || r.kind === "unknown")
|
|
91
|
+
m.set(normPath(r.path), "scan");
|
|
92
|
+
}
|
|
93
|
+
for (const p of audit?.pages ?? []) {
|
|
94
|
+
if (p.status >= 200 && p.status < 300)
|
|
95
|
+
m.set(normPath(p.url), "live");
|
|
96
|
+
}
|
|
97
|
+
return m;
|
|
98
|
+
}
|
|
99
|
+
export function findPage(paths, key) {
|
|
100
|
+
if (key === "home") {
|
|
101
|
+
const s = paths.get("/");
|
|
102
|
+
return s ? { path: "/", source: s } : undefined;
|
|
103
|
+
}
|
|
104
|
+
const re = PAGE_PATTERNS[key];
|
|
105
|
+
let best;
|
|
106
|
+
for (const [p, source] of paths) {
|
|
107
|
+
if (!re.test(p) || /[[:*$]/.test(p))
|
|
108
|
+
continue;
|
|
109
|
+
if (!best || p.length < best.path.length || (source === "live" && best.source === "scan"))
|
|
110
|
+
best = { path: p, source };
|
|
111
|
+
}
|
|
112
|
+
return best;
|
|
113
|
+
}
|
|
114
|
+
export function pathKnown(paths, url) {
|
|
115
|
+
if (!url)
|
|
116
|
+
return undefined;
|
|
117
|
+
return paths.get(normPath(url));
|
|
118
|
+
}
|
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
// Defensive readers for the .katman/ files other modules write (scan, audits, runs) and for plan/research.
|
|
2
|
+
// Those files may be missing, partial or hand-edited: every reader returns a well-formed shape or undefined.
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { existsSync, statSync, promises as fs } from "node:fs";
|
|
5
|
+
import { KATMAN_DIR, katmanPath, listFiles, readJson } from "../../../state.js";
|
|
6
|
+
import { domainOf } from "../../../util.js";
|
|
7
|
+
const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
8
|
+
const arr = (x) => (Array.isArray(x) ? x : []);
|
|
9
|
+
const str = (x) => (typeof x === "string" ? x : undefined);
|
|
10
|
+
const num = (x, d = 0) => (typeof x === "number" && Number.isFinite(x) ? x : d);
|
|
11
|
+
const bool = (x) => x === true;
|
|
12
|
+
export function hasKatmanDir(root) {
|
|
13
|
+
return existsSync(path.join(root, KATMAN_DIR));
|
|
14
|
+
}
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
// research
|
|
17
|
+
export function normalizeResearch(raw) {
|
|
18
|
+
if (!isObj(raw))
|
|
19
|
+
return undefined;
|
|
20
|
+
const site = isObj(raw.site) ? raw.site : {};
|
|
21
|
+
const brand = isObj(raw.brand) ? raw.brand : {};
|
|
22
|
+
const url = str(site.url) ?? "";
|
|
23
|
+
const r = {
|
|
24
|
+
version: 1,
|
|
25
|
+
updatedAt: str(raw.updatedAt) ?? "",
|
|
26
|
+
site: {
|
|
27
|
+
url,
|
|
28
|
+
domain: str(site.domain) ?? (url && /^https?:/i.test(url) ? domainOf(url) : ""),
|
|
29
|
+
languages: arr(site.languages).filter((x) => typeof x === "string"),
|
|
30
|
+
...(Array.isArray(site.previewUrls) ? { previewUrls: arr(site.previewUrls).filter((x) => typeof x === "string") } : {}),
|
|
31
|
+
},
|
|
32
|
+
brand: {
|
|
33
|
+
...brand,
|
|
34
|
+
name: str(brand.name) ?? "",
|
|
35
|
+
category: str(brand.category) ?? "",
|
|
36
|
+
},
|
|
37
|
+
situations: arr(raw.situations)
|
|
38
|
+
.filter((s) => isObj(s) && typeof s.sentence === "string")
|
|
39
|
+
.map((s) => ({
|
|
40
|
+
id: str(s.id) ?? "",
|
|
41
|
+
sentence: String(s.sentence),
|
|
42
|
+
...(str(s.url) ? { url: String(s.url) } : {}),
|
|
43
|
+
...(s.status === "live" || s.status === "planned" ? { status: s.status } : {}),
|
|
44
|
+
})),
|
|
45
|
+
competitors: arr(raw.competitors)
|
|
46
|
+
.filter((c) => isObj(c) && typeof c.name === "string")
|
|
47
|
+
.map((c) => ({ ...c, name: String(c.name), url: str(c.url) ?? "" })),
|
|
48
|
+
proof: arr(raw.proof)
|
|
49
|
+
.filter((p) => isObj(p) && typeof p.claim === "string")
|
|
50
|
+
.map((p) => ({ ...p, claim: String(p.claim), source: str(p.source) ?? "" })),
|
|
51
|
+
keywords: arr(raw.keywords)
|
|
52
|
+
.filter((k) => isObj(k) && typeof k.term === "string")
|
|
53
|
+
.map((k) => k),
|
|
54
|
+
questions: arr(raw.questions)
|
|
55
|
+
.filter((q) => isObj(q) && typeof q.text === "string")
|
|
56
|
+
.map((q) => ({
|
|
57
|
+
id: str(q.id) ?? "",
|
|
58
|
+
type: (str(q.type) ?? "situation"),
|
|
59
|
+
text: String(q.text),
|
|
60
|
+
...(str(q.lang) ? { lang: String(q.lang) } : {}),
|
|
61
|
+
})),
|
|
62
|
+
};
|
|
63
|
+
return r;
|
|
64
|
+
}
|
|
65
|
+
export async function loadResearchSafe(root) {
|
|
66
|
+
return normalizeResearch(await readJson(root, "research.json"));
|
|
67
|
+
}
|
|
68
|
+
export async function loadConfigSafe(root) {
|
|
69
|
+
const c = await readJson(root, "config.json");
|
|
70
|
+
return isObj(c) ? c : {};
|
|
71
|
+
}
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
// scan.json
|
|
74
|
+
function fileInfo(x) {
|
|
75
|
+
if (!isObj(x))
|
|
76
|
+
return { exists: false };
|
|
77
|
+
return { exists: bool(x.exists), ...(str(x.path) ? { path: str(x.path) } : {}), ...(x.generated === true ? { generated: true } : {}) };
|
|
78
|
+
}
|
|
79
|
+
export function normalizeFinding(x) {
|
|
80
|
+
if (!isObj(x) || typeof x.title !== "string")
|
|
81
|
+
return undefined;
|
|
82
|
+
const sev = ["critical", "high", "medium", "low", "info"].includes(String(x.severity)) ? x.severity : "medium";
|
|
83
|
+
const gate = ["read", "quote", "fit", "mention", "measure"].includes(String(x.gate)) ? x.gate : "read";
|
|
84
|
+
const prompt = /^P([0-9]|1[0-4])$/.test(String(x.prompt)) ? x.prompt : undefined;
|
|
85
|
+
return {
|
|
86
|
+
id: str(x.id) ?? x.title.toLowerCase().replace(/[^a-z0-9]+/g, "-").slice(0, 40),
|
|
87
|
+
title: x.title,
|
|
88
|
+
severity: sev,
|
|
89
|
+
gate,
|
|
90
|
+
detail: str(x.detail) ?? "",
|
|
91
|
+
...(Array.isArray(x.evidence) ? { evidence: arr(x.evidence).map(String) } : {}),
|
|
92
|
+
...(Array.isArray(x.files) ? { files: arr(x.files).map(String) } : {}),
|
|
93
|
+
...(Array.isArray(x.urls) ? { urls: arr(x.urls).map(String) } : {}),
|
|
94
|
+
...(str(x.fix) ? { fix: str(x.fix) } : {}),
|
|
95
|
+
...(prompt ? { prompt } : {}),
|
|
96
|
+
...(str(x.accept) ? { accept: str(x.accept) } : {}),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
export function normalizeScan(raw, root) {
|
|
100
|
+
if (!isObj(raw))
|
|
101
|
+
return undefined;
|
|
102
|
+
const fw = isObj(raw.framework) ? raw.framework : {};
|
|
103
|
+
const rend = isObj(raw.rendering) ? raw.rendering : {};
|
|
104
|
+
const host = isObj(raw.hosting) ? raw.hosting : {};
|
|
105
|
+
const files = isObj(raw.files) ? raw.files : {};
|
|
106
|
+
const i18n = isObj(raw.i18n) ? raw.i18n : {};
|
|
107
|
+
const an = isObj(raw.analytics) ? raw.analytics : {};
|
|
108
|
+
const verdicts = ["ssr", "ssg", "hybrid", "csr", "static-html", "unknown"];
|
|
109
|
+
const kinds = ["public", "app", "auth", "api", "legal", "checkout", "search", "unknown"];
|
|
110
|
+
return {
|
|
111
|
+
version: 1,
|
|
112
|
+
ts: str(raw.ts) ?? "",
|
|
113
|
+
root: str(raw.root) ?? root,
|
|
114
|
+
framework: {
|
|
115
|
+
name: str(fw.name) ?? "unknown",
|
|
116
|
+
...(str(fw.version) ? { version: str(fw.version) } : {}),
|
|
117
|
+
...(str(fw.router) ? { router: str(fw.router) } : {}),
|
|
118
|
+
...(str(fw.builder) ? { builder: str(fw.builder) } : {}),
|
|
119
|
+
},
|
|
120
|
+
rendering: {
|
|
121
|
+
verdict: (verdicts.includes(String(rend.verdict)) ? rend.verdict : "unknown"),
|
|
122
|
+
confidence: (["high", "medium", "low"].includes(String(rend.confidence)) ? rend.confidence : "low"),
|
|
123
|
+
evidence: arr(rend.evidence).map(String),
|
|
124
|
+
},
|
|
125
|
+
hosting: {
|
|
126
|
+
...(str(host.provider) ? { provider: str(host.provider) } : {}),
|
|
127
|
+
previewPatterns: arr(host.previewPatterns).map(String),
|
|
128
|
+
evidence: arr(host.evidence).map(String),
|
|
129
|
+
},
|
|
130
|
+
...(typeof raw.staticDir === "string" ? { staticDir: raw.staticDir } : {}),
|
|
131
|
+
routes: arr(raw.routes)
|
|
132
|
+
.filter((r) => isObj(r) && typeof r.path === "string")
|
|
133
|
+
.map((r) => ({
|
|
134
|
+
path: String(r.path),
|
|
135
|
+
...(str(r.file) ? { file: str(r.file) } : {}),
|
|
136
|
+
kind: (kinds.includes(String(r.kind)) ? r.kind : "unknown"),
|
|
137
|
+
dynamic: bool(r.dynamic) || /[[:*$]/.test(String(r.path)),
|
|
138
|
+
})),
|
|
139
|
+
files: {
|
|
140
|
+
robots: fileInfo(files.robots),
|
|
141
|
+
sitemap: fileInfo(files.sitemap),
|
|
142
|
+
llms: fileInfo(files.llms),
|
|
143
|
+
llmsFull: fileInfo(files.llmsFull),
|
|
144
|
+
notFound: fileInfo(files.notFound),
|
|
145
|
+
...(isObj(files.headSource) ? { headSource: fileInfo(files.headSource) } : {}),
|
|
146
|
+
},
|
|
147
|
+
jsonLdTypes: arr(raw.jsonLdTypes).map(String),
|
|
148
|
+
i18n: {
|
|
149
|
+
detected: bool(i18n.detected),
|
|
150
|
+
...(str(i18n.library) ? { library: str(i18n.library) } : {}),
|
|
151
|
+
locales: arr(i18n.locales).map(String),
|
|
152
|
+
...(str(i18n.htmlLang) ? { htmlLang: str(i18n.htmlLang) } : {}),
|
|
153
|
+
},
|
|
154
|
+
analytics: { providers: arr(an.providers).map(String), tracksAiChannel: bool(an.tracksAiChannel) },
|
|
155
|
+
findings: arr(raw.findings).map(normalizeFinding).filter((f) => !!f),
|
|
156
|
+
...(Array.isArray(raw.domains) ? { domains: arr(raw.domains).filter((d) => typeof d === "string" && d.includes(".")) } : {}),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
export async function loadScan(root) {
|
|
160
|
+
return normalizeScan(await readJson(root, "scan.json"), root);
|
|
161
|
+
}
|
|
162
|
+
// ---------------------------------------------------------------------------
|
|
163
|
+
// audits/<stamp>.json
|
|
164
|
+
function normalizePage(x) {
|
|
165
|
+
if (!isObj(x) || typeof x.url !== "string")
|
|
166
|
+
return undefined;
|
|
167
|
+
return {
|
|
168
|
+
url: x.url,
|
|
169
|
+
status: num(x.status),
|
|
170
|
+
...(str(x.contentType) ? { contentType: str(x.contentType) } : {}),
|
|
171
|
+
noindex: bool(x.noindex),
|
|
172
|
+
...(str(x.title) !== undefined ? { title: str(x.title) } : {}),
|
|
173
|
+
...(str(x.description) !== undefined ? { description: str(x.description) } : {}),
|
|
174
|
+
...(str(x.canonical) ? { canonical: str(x.canonical) } : {}),
|
|
175
|
+
...(typeof x.canonicalSelf === "boolean" ? { canonicalSelf: x.canonicalSelf } : {}),
|
|
176
|
+
h1Count: num(x.h1Count),
|
|
177
|
+
headingSkips: num(x.headingSkips),
|
|
178
|
+
wordsJsOff: num(x.wordsJsOff),
|
|
179
|
+
...(str(x.firstParagraph) ? { firstParagraph: str(x.firstParagraph) } : {}),
|
|
180
|
+
jsonLdTypes: arr(x.jsonLdTypes).map(String),
|
|
181
|
+
jsonLdErrors: num(x.jsonLdErrors),
|
|
182
|
+
...(str(x.ogImage) ? { ogImage: str(x.ogImage) } : {}),
|
|
183
|
+
...(str(x.lang) ? { lang: str(x.lang) } : {}),
|
|
184
|
+
hreflang: arr(x.hreflang).filter(isObj).map((h) => ({ lang: String(h.lang ?? ""), href: String(h.href ?? "") })),
|
|
185
|
+
imagesMissingAlt: num(x.imagesMissingAlt),
|
|
186
|
+
internalLinksOut: num(x.internalLinksOut),
|
|
187
|
+
internalLinksIn: num(x.internalLinksIn),
|
|
188
|
+
inSitemap: bool(x.inSitemap),
|
|
189
|
+
...(str(x.dateModified) ? { dateModified: str(x.dateModified) } : {}),
|
|
190
|
+
bytes: num(x.bytes),
|
|
191
|
+
ms: num(x.ms),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
function gate(x) {
|
|
195
|
+
if (!isObj(x))
|
|
196
|
+
return { pass: 0, total: 0, notes: [] };
|
|
197
|
+
return { pass: num(x.pass), total: num(x.total), notes: arr(x.notes).map(String) };
|
|
198
|
+
}
|
|
199
|
+
export function normalizeAudit(raw) {
|
|
200
|
+
if (!isObj(raw))
|
|
201
|
+
return undefined;
|
|
202
|
+
const site = isObj(raw.site) ? raw.site : {};
|
|
203
|
+
const robots = isObj(site.robots) ? site.robots : {};
|
|
204
|
+
const llms = isObj(site.llmsTxt) ? site.llmsTxt : {};
|
|
205
|
+
const gates = isObj(raw.gates) ? raw.gates : {};
|
|
206
|
+
return {
|
|
207
|
+
version: 1,
|
|
208
|
+
ts: str(raw.ts) ?? "",
|
|
209
|
+
siteUrl: str(raw.siteUrl) ?? "",
|
|
210
|
+
pages: arr(raw.pages).map(normalizePage).filter((p) => !!p),
|
|
211
|
+
site: {
|
|
212
|
+
...(typeof site.httpsRedirect === "boolean" ? { httpsRedirect: site.httpsRedirect } : {}),
|
|
213
|
+
...(typeof site.hostRedirect === "boolean" ? { hostRedirect: site.hostRedirect } : {}),
|
|
214
|
+
...(typeof site.hsts === "boolean" ? { hsts: site.hsts } : {}),
|
|
215
|
+
...(site.compression !== undefined ? { compression: (str(site.compression) ?? null) } : {}),
|
|
216
|
+
...(typeof site.soft404 === "boolean" ? { soft404: site.soft404 } : {}),
|
|
217
|
+
llmsTxt: { exists: bool(llms.exists), ...(typeof llms.bytes === "number" ? { bytes: llms.bytes } : {}) },
|
|
218
|
+
robots: {
|
|
219
|
+
exists: bool(robots.exists),
|
|
220
|
+
sitemaps: arr(robots.sitemaps).map(String),
|
|
221
|
+
blocksAll: bool(robots.blocksAll),
|
|
222
|
+
blockedAiTokens: arr(robots.blockedAiTokens).map(String),
|
|
223
|
+
invalidLines: arr(robots.invalidLines).map(String),
|
|
224
|
+
},
|
|
225
|
+
sitemapUrls: num(site.sitemapUrls),
|
|
226
|
+
...(str(site.cdn) ? { cdn: str(site.cdn) } : {}),
|
|
227
|
+
bots: arr(site.bots)
|
|
228
|
+
.filter(isObj)
|
|
229
|
+
.map((b) => ({
|
|
230
|
+
token: String(b.token ?? ""),
|
|
231
|
+
purpose: String(b.purpose ?? ""),
|
|
232
|
+
status: num(b.status),
|
|
233
|
+
blocked: bool(b.blocked),
|
|
234
|
+
...(str(b.reason) ? { reason: str(b.reason) } : {}),
|
|
235
|
+
})),
|
|
236
|
+
},
|
|
237
|
+
gates: { read: gate(gates.read), quote: gate(gates.quote), fit: gate(gates.fit) },
|
|
238
|
+
findings: arr(raw.findings).map(normalizeFinding).filter((f) => !!f),
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
/** Latest audit by file name (stamps sort chronologically). Skips files that don't parse. */
|
|
242
|
+
export async function loadLatestAudit(root) {
|
|
243
|
+
const names = (await listFiles(root, "audits")).filter((n) => n.endsWith(".json"));
|
|
244
|
+
for (let i = names.length - 1; i >= 0; i--) {
|
|
245
|
+
const a = normalizeAudit(await readJson(root, path.join("audits", names[i])));
|
|
246
|
+
if (a)
|
|
247
|
+
return { audit: a, file: names[i], count: names.length };
|
|
248
|
+
}
|
|
249
|
+
return undefined;
|
|
250
|
+
}
|
|
251
|
+
// ---------------------------------------------------------------------------
|
|
252
|
+
// runs/<stamp>.json
|
|
253
|
+
function normalizeAnswer(x) {
|
|
254
|
+
if (!isObj(x))
|
|
255
|
+
return undefined;
|
|
256
|
+
return {
|
|
257
|
+
questionId: String(x.questionId ?? ""),
|
|
258
|
+
question: String(x.question ?? ""),
|
|
259
|
+
questionType: (str(x.questionType) ?? "situation"),
|
|
260
|
+
model: String(x.model ?? ""),
|
|
261
|
+
family: String(x.family ?? x.model ?? ""),
|
|
262
|
+
source: x.source === "manual" ? "manual" : "api",
|
|
263
|
+
...(str(x.surface) ? { surface: str(x.surface) } : {}),
|
|
264
|
+
ts: String(x.ts ?? ""),
|
|
265
|
+
...(typeof x.temperature === "number" ? { temperature: x.temperature } : {}),
|
|
266
|
+
mentioned: bool(x.mentioned),
|
|
267
|
+
recommendedFirst: bool(x.recommendedFirst),
|
|
268
|
+
rank: typeof x.rank === "number" ? x.rank : null,
|
|
269
|
+
cited: bool(x.cited),
|
|
270
|
+
namedProducts: arr(x.namedProducts).map(String),
|
|
271
|
+
competitorsNamed: arr(x.competitorsNamed).map(String),
|
|
272
|
+
citations: arr(x.citations)
|
|
273
|
+
.filter(isObj)
|
|
274
|
+
.map((c) => ({ url: String(c.url ?? ""), ...(str(c.title) ? { title: str(c.title) } : {}), domain: String(c.domain ?? domainOf(String(c.url ?? ""))) })),
|
|
275
|
+
answerExcerpt: String(x.answerExcerpt ?? ""),
|
|
276
|
+
...(typeof x.identityCorrect === "boolean" || x.identityCorrect === null ? { identityCorrect: x.identityCorrect } : {}),
|
|
277
|
+
...(typeof x.costUsd === "number" ? { costUsd: x.costUsd } : {}),
|
|
278
|
+
...(str(x.error) ? { error: str(x.error) } : {}),
|
|
279
|
+
...(typeof x.searched === "boolean" ? { searched: x.searched } : {}),
|
|
280
|
+
...(typeof x.sample === "number" ? { sample: x.sample } : {}),
|
|
281
|
+
...(str(x.lang) ? { lang: str(x.lang) } : {}),
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
/** Recompute the summary from answers (used when a run file has none or a partial one). */
|
|
285
|
+
export function summarizeAnswers(answers) {
|
|
286
|
+
const ok = answers.filter((a) => !a.error);
|
|
287
|
+
const byFamily = {};
|
|
288
|
+
const byLanguage = {};
|
|
289
|
+
const comp = new Map();
|
|
290
|
+
const doms = new Map();
|
|
291
|
+
for (const a of ok) {
|
|
292
|
+
const buckets = [(byFamily[a.family || a.model] ??= { answered: 0, mentioned: 0, recommendedFirst: 0, cited: 0 })];
|
|
293
|
+
if (a.lang)
|
|
294
|
+
buckets.push((byLanguage[a.lang] ??= { answered: 0, mentioned: 0, recommendedFirst: 0, cited: 0 }));
|
|
295
|
+
for (const f of buckets) {
|
|
296
|
+
f.answered++;
|
|
297
|
+
if (a.mentioned)
|
|
298
|
+
f.mentioned++;
|
|
299
|
+
if (a.recommendedFirst)
|
|
300
|
+
f.recommendedFirst++;
|
|
301
|
+
if (a.cited)
|
|
302
|
+
f.cited++;
|
|
303
|
+
}
|
|
304
|
+
for (const c of a.competitorsNamed)
|
|
305
|
+
comp.set(c, (comp.get(c) ?? 0) + 1);
|
|
306
|
+
for (const c of a.citations)
|
|
307
|
+
if (c.domain)
|
|
308
|
+
doms.set(c.domain, (doms.get(c.domain) ?? 0) + 1);
|
|
309
|
+
}
|
|
310
|
+
const n = ok.length || 1;
|
|
311
|
+
const top = (m) => [...m.entries()].sort((a, b) => b[1] - a[1]).slice(0, 15);
|
|
312
|
+
return {
|
|
313
|
+
answered: ok.length,
|
|
314
|
+
mentionRate: ok.filter((a) => a.mentioned).length / n,
|
|
315
|
+
recommendedFirstRate: ok.filter((a) => a.recommendedFirst).length / n,
|
|
316
|
+
citationRate: ok.filter((a) => a.cited).length / n,
|
|
317
|
+
byFamily,
|
|
318
|
+
topCompetitors: top(comp).map(([name, count]) => ({ name, count })),
|
|
319
|
+
topCitedDomains: top(doms).map(([domain, count]) => ({ domain, count })),
|
|
320
|
+
...(Object.keys(byLanguage).length ? { byLanguage } : {}),
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
export function normalizeRun(raw, fallbackId) {
|
|
324
|
+
if (!isObj(raw))
|
|
325
|
+
return undefined;
|
|
326
|
+
const answers = arr(raw.answers).map(normalizeAnswer).filter((a) => !!a);
|
|
327
|
+
const s = isObj(raw.summary) ? raw.summary : undefined;
|
|
328
|
+
const computed = summarizeAnswers(answers);
|
|
329
|
+
const summary = s
|
|
330
|
+
? {
|
|
331
|
+
answered: typeof s.answered === "number" ? s.answered : computed.answered,
|
|
332
|
+
mentionRate: typeof s.mentionRate === "number" ? s.mentionRate : computed.mentionRate,
|
|
333
|
+
recommendedFirstRate: typeof s.recommendedFirstRate === "number" ? s.recommendedFirstRate : computed.recommendedFirstRate,
|
|
334
|
+
citationRate: typeof s.citationRate === "number" ? s.citationRate : computed.citationRate,
|
|
335
|
+
byFamily: isObj(s.byFamily) ? s.byFamily : computed.byFamily,
|
|
336
|
+
topCompetitors: Array.isArray(s.topCompetitors) ? s.topCompetitors : computed.topCompetitors,
|
|
337
|
+
topCitedDomains: Array.isArray(s.topCitedDomains) ? s.topCitedDomains : computed.topCitedDomains,
|
|
338
|
+
...(isObj(s.byLanguage)
|
|
339
|
+
? { byLanguage: s.byLanguage }
|
|
340
|
+
: computed.byLanguage
|
|
341
|
+
? { byLanguage: computed.byLanguage }
|
|
342
|
+
: {}),
|
|
343
|
+
}
|
|
344
|
+
: computed;
|
|
345
|
+
return {
|
|
346
|
+
id: str(raw.id) ?? fallbackId,
|
|
347
|
+
ts: str(raw.ts) ?? "",
|
|
348
|
+
brand: String(raw.brand ?? ""),
|
|
349
|
+
domain: String(raw.domain ?? ""),
|
|
350
|
+
models: arr(raw.models)
|
|
351
|
+
.filter(isObj)
|
|
352
|
+
.map((m) => ({ id: String(m.id ?? ""), family: String(m.family ?? m.id ?? ""), search: (["native", "plugin", "none"].includes(String(m.search)) ? m.search : "none") })),
|
|
353
|
+
temperature: typeof raw.temperature === "number" ? raw.temperature : null,
|
|
354
|
+
...(raw.status === "complete" || raw.status === "partial" ? { status: raw.status } : {}),
|
|
355
|
+
answers,
|
|
356
|
+
summary,
|
|
357
|
+
...(typeof raw.totalCostUsd === "number" ? { totalCostUsd: raw.totalCostUsd } : {}),
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
/** All runs, oldest first. */
|
|
361
|
+
export async function loadRuns(root) {
|
|
362
|
+
const names = (await listFiles(root, "runs")).filter((n) => n.endsWith(".json"));
|
|
363
|
+
const out = [];
|
|
364
|
+
for (const n of names) {
|
|
365
|
+
const r = normalizeRun(await readJson(root, path.join("runs", n)), n.replace(/\.json$/, ""));
|
|
366
|
+
if (r)
|
|
367
|
+
out.push(r);
|
|
368
|
+
}
|
|
369
|
+
return out.sort((a, b) => (a.ts || a.id).localeCompare(b.ts || b.id));
|
|
370
|
+
}
|
|
371
|
+
export function runDate(r) {
|
|
372
|
+
const d = new Date(r.ts);
|
|
373
|
+
if (!Number.isNaN(d.getTime()))
|
|
374
|
+
return d;
|
|
375
|
+
const m = /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})Z/.exec(r.id);
|
|
376
|
+
if (m)
|
|
377
|
+
return new Date(`${m[1]}T${m[2]}:${m[3]}:${m[4]}Z`);
|
|
378
|
+
return undefined;
|
|
379
|
+
}
|
|
380
|
+
// ---------------------------------------------------------------------------
|
|
381
|
+
// project files (read-only, never secrets)
|
|
382
|
+
const SECRET_RE = /(^|[\\/])(\.env[^\\/]*|\.dev\.vars[^\\/]*)$|\.(pem|key|p12|pfx)$|secret|credential/i;
|
|
383
|
+
export function isSecretPath(p) {
|
|
384
|
+
return SECRET_RE.test(p);
|
|
385
|
+
}
|
|
386
|
+
/** Resolve a path relative to the project root; undefined when it escapes the root. */
|
|
387
|
+
export function insideRoot(root, rel) {
|
|
388
|
+
const base = path.resolve(root);
|
|
389
|
+
const p = path.resolve(base, rel);
|
|
390
|
+
if (p !== base && !p.startsWith(base + path.sep))
|
|
391
|
+
return undefined;
|
|
392
|
+
return p;
|
|
393
|
+
}
|
|
394
|
+
export async function readProjectFile(root, rel, maxBytes = 2_000_000) {
|
|
395
|
+
const abs = insideRoot(root, rel);
|
|
396
|
+
if (!abs)
|
|
397
|
+
return { error: `${rel} is outside the project root ${root}.` };
|
|
398
|
+
if (isSecretPath(abs))
|
|
399
|
+
return { error: `Katman never reads secret files (${path.basename(abs)}).` };
|
|
400
|
+
try {
|
|
401
|
+
const st = await fs.stat(abs);
|
|
402
|
+
if (!st.isFile())
|
|
403
|
+
return { abs, error: `${rel} is not a file.` };
|
|
404
|
+
if (st.size > maxBytes)
|
|
405
|
+
return { abs, error: `${rel} is larger than ${Math.round(maxBytes / 1e6)} MB.` };
|
|
406
|
+
return { abs, text: await fs.readFile(abs, "utf8") };
|
|
407
|
+
}
|
|
408
|
+
catch {
|
|
409
|
+
return { abs, error: `${rel} not found.` };
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
export async function mtimeOf(p) {
|
|
413
|
+
try {
|
|
414
|
+
return (await fs.stat(p)).mtime;
|
|
415
|
+
}
|
|
416
|
+
catch {
|
|
417
|
+
return undefined;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
export async function katmanMtime(root, rel) {
|
|
421
|
+
return mtimeOf(katmanPath(root, rel));
|
|
422
|
+
}
|
|
423
|
+
// ---------------------------------------------------------------------------
|
|
424
|
+
// static dir (folder served as-is at the site root)
|
|
425
|
+
const STATIC_BY_FRAMEWORK = {
|
|
426
|
+
sveltekit: "static", hugo: "static", gatsby: "static", jekyll: "", eleventy: "", "static-html": "",
|
|
427
|
+
};
|
|
428
|
+
export function cleanRel(p) {
|
|
429
|
+
const s = p.replace(/\\/g, "/").replace(/^\.\/?/, "").replace(/\/+$/, "");
|
|
430
|
+
return s === "." ? "" : s;
|
|
431
|
+
}
|
|
432
|
+
/** scan.staticDir → existing public/ or static/ → framework convention → project root. */
|
|
433
|
+
export function staticDirFor(root, scan) {
|
|
434
|
+
if (scan && typeof scan.staticDir === "string")
|
|
435
|
+
return { dir: cleanRel(scan.staticDir), source: "scan" };
|
|
436
|
+
for (const d of ["public", "static"]) {
|
|
437
|
+
try {
|
|
438
|
+
if (statSync(path.join(root, d)).isDirectory())
|
|
439
|
+
return { dir: d, source: "detected" };
|
|
440
|
+
}
|
|
441
|
+
catch {
|
|
442
|
+
// not there
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const name = scan?.framework.name ?? "";
|
|
446
|
+
if (name in STATIC_BY_FRAMEWORK)
|
|
447
|
+
return { dir: STATIC_BY_FRAMEWORK[name], source: "convention" };
|
|
448
|
+
if (name && name !== "unknown")
|
|
449
|
+
return { dir: "public", source: "convention" };
|
|
450
|
+
return { dir: "", source: "root" };
|
|
451
|
+
}
|
|
452
|
+
export function joinRel(...parts) {
|
|
453
|
+
return parts.filter((p) => p !== "").join("/").replace(/\/+/g, "/");
|
|
454
|
+
}
|