katman-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +13 -0
- package/README.md +160 -0
- package/dist/cli.js +71 -0
- package/dist/context.js +43 -0
- package/dist/crawlers.js +86 -0
- package/dist/index.js +25 -0
- package/dist/license.js +448 -0
- package/dist/meta.js +28 -0
- package/dist/net.js +128 -0
- package/dist/server.js +17 -0
- package/dist/state.js +81 -0
- package/dist/tools/account/account.js +64 -0
- package/dist/tools/account/cli.js +21 -0
- package/dist/tools/account/index.js +48 -0
- package/dist/tools/account/sync.js +142 -0
- package/dist/tools/account/upsell.js +290 -0
- package/dist/tools/crawl/audit/analyze.js +719 -0
- package/dist/tools/crawl/audit/bots.js +114 -0
- package/dist/tools/crawl/audit/checks.js +126 -0
- package/dist/tools/crawl/audit/html.js +382 -0
- package/dist/tools/crawl/audit/index.js +364 -0
- package/dist/tools/crawl/audit/model.js +1 -0
- package/dist/tools/crawl/audit/report.js +199 -0
- package/dist/tools/crawl/audit/robots.js +199 -0
- package/dist/tools/crawl/audit/sitemap.js +147 -0
- package/dist/tools/crawl/cli.js +106 -0
- package/dist/tools/crawl/common/advice.js +61 -0
- package/dist/tools/crawl/common/findings.js +57 -0
- package/dist/tools/crawl/common/routes.js +80 -0
- package/dist/tools/crawl/common/text.js +89 -0
- package/dist/tools/crawl/common/urls.js +64 -0
- package/dist/tools/crawl/index.js +226 -0
- package/dist/tools/crawl/indexnow.js +271 -0
- package/dist/tools/crawl/keywords.js +178 -0
- package/dist/tools/crawl/scan/framework.js +645 -0
- package/dist/tools/crawl/scan/index.js +175 -0
- package/dist/tools/crawl/scan/project.js +180 -0
- package/dist/tools/crawl/scan/routes.js +354 -0
- package/dist/tools/crawl/scan/signals.js +414 -0
- package/dist/tools/method/cli.js +42 -0
- package/dist/tools/method/core/brief.js +302 -0
- package/dist/tools/method/core/extract.js +435 -0
- package/dist/tools/method/core/generate.js +504 -0
- package/dist/tools/method/core/lint.js +556 -0
- package/dist/tools/method/core/plan.js +405 -0
- package/dist/tools/method/core/prompt.js +150 -0
- package/dist/tools/method/core/questions.js +196 -0
- package/dist/tools/method/core/report.js +182 -0
- package/dist/tools/method/core/research-run.js +148 -0
- package/dist/tools/method/core/research.js +448 -0
- package/dist/tools/method/core/snippets.js +292 -0
- package/dist/tools/method/core/start.js +277 -0
- package/dist/tools/method/data/anatomy.js +141 -0
- package/dist/tools/method/data/appendices.js +202 -0
- package/dist/tools/method/data/cloudflare.js +192 -0
- package/dist/tools/method/data/evidence.js +137 -0
- package/dist/tools/method/data/geo-jobs.js +128 -0
- package/dist/tools/method/data/index.js +19 -0
- package/dist/tools/method/data/landing.js +67 -0
- package/dist/tools/method/data/lessons.js +127 -0
- package/dist/tools/method/data/measurement.js +142 -0
- package/dist/tools/method/data/method.js +141 -0
- package/dist/tools/method/data/offsite.js +148 -0
- package/dist/tools/method/data/pages.js +167 -0
- package/dist/tools/method/data/prompts.js +633 -0
- package/dist/tools/method/data/publish.js +113 -0
- package/dist/tools/method/data/render.js +117 -0
- package/dist/tools/method/data/research.js +131 -0
- package/dist/tools/method/data/roadmap.js +105 -0
- package/dist/tools/method/data/traps.js +74 -0
- package/dist/tools/method/data/verify.js +88 -0
- package/dist/tools/method/data/viral-list.js +39 -0
- package/dist/tools/method/index.js +209 -0
- package/dist/tools/method/lib/calls.js +53 -0
- package/dist/tools/method/lib/framework.js +525 -0
- package/dist/tools/method/lib/l10n.js +17 -0
- package/dist/tools/method/lib/routes.js +118 -0
- package/dist/tools/method/lib/state.js +454 -0
- package/dist/tools/method/lib/text.js +165 -0
- package/dist/tools/method/mcp/prompts.js +79 -0
- package/dist/tools/method/mcp/resources.js +50 -0
- package/dist/tools/visibility/analyze.js +463 -0
- package/dist/tools/visibility/cli.js +86 -0
- package/dist/tools/visibility/estimate.js +105 -0
- package/dist/tools/visibility/index.js +108 -0
- package/dist/tools/visibility/judge.js +156 -0
- package/dist/tools/visibility/manual.js +275 -0
- package/dist/tools/visibility/models.js +277 -0
- package/dist/tools/visibility/monitoring.js +165 -0
- package/dist/tools/visibility/openrouter.js +449 -0
- package/dist/tools/visibility/render.js +295 -0
- package/dist/tools/visibility/run.js +684 -0
- package/dist/tools/visibility/summary.js +164 -0
- package/dist/types.js +3 -0
- package/dist/util.js +92 -0
- package/package.json +54 -0
- package/server.json +34 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
// Code generators for katman_generate: the browser AI-channel snippet (book ch. 15 human filter + channel rule)
|
|
2
|
+
// and the server-side bot counter (per day, per bot, per path) for 5 platforms.
|
|
3
|
+
import { CRAWLERS } from "../../../crawlers.js";
|
|
4
|
+
import { AI_REFERRER_HOSTS, AI_UTM_MARKERS, SOCIAL_REFERRER_HOSTS } from "../data/measurement.js";
|
|
5
|
+
export const BOT_PLATFORMS = ["cloudflare-worker", "nextjs-middleware", "express", "astro-middleware", "sveltekit-hooks"];
|
|
6
|
+
/** Bots worth counting beyond Katman's crawler list (seen on the sample site). */
|
|
7
|
+
const EXTRA_BOTS = [
|
|
8
|
+
["YandexBot", "Yandex · search"],
|
|
9
|
+
["DuckDuckBot", "DuckDuckGo · search"],
|
|
10
|
+
];
|
|
11
|
+
export function botTable() {
|
|
12
|
+
const rows = CRAWLERS.map((c) => [c.token, `${c.org} · ${c.purpose}`]);
|
|
13
|
+
for (const e of EXTRA_BOTS)
|
|
14
|
+
if (!rows.some((r) => r[0] === e[0]))
|
|
15
|
+
rows.push(e);
|
|
16
|
+
return rows;
|
|
17
|
+
}
|
|
18
|
+
export function analyticsChannelSnippet(target = "auto") {
|
|
19
|
+
const js = (xs) => JSON.stringify(xs);
|
|
20
|
+
return `/*! Katman AI channel + human filter (Katman book, ch. 15) — https://katman.pro
|
|
21
|
+
* channel: "ai" (assistant referrer OR utm_source such as chatgpt.com), "search", "social", "referral", "direct".
|
|
22
|
+
* human: the session scrolled, viewed more than 2 pages, converted (katmanChannel.convert("signup")), or came
|
|
23
|
+
* from an in-app browser (TikTok / Instagram). Bots that run JavaScript rarely do any of these.
|
|
24
|
+
* No cookies; sessionStorage only. Include it once on every public page:
|
|
25
|
+
* <script src="/katman-channel.js" defer></script>
|
|
26
|
+
*/
|
|
27
|
+
(function () {
|
|
28
|
+
"use strict";
|
|
29
|
+
var TARGET = ${JSON.stringify(target)}; // auto | gtag | gtm | plausible | posthog | none
|
|
30
|
+
var AI_HOSTS = ${js(AI_REFERRER_HOSTS)};
|
|
31
|
+
var AI_UTM = ${js(AI_UTM_MARKERS)};
|
|
32
|
+
var SEARCH = [/(^|\\.)google\\.[a-z.]+$/, /(^|\\.)bing\\.com$/, /(^|\\.)yandex\\.[a-z.]+$/, /^search\\.brave\\.com$/, /(^|\\.)duckduckgo\\.com$/,
|
|
33
|
+
/(^|\\.)ecosia\\.org$/, /(^|\\.)search\\.yahoo\\.com$/, /(^|\\.)baidu\\.com$/, /(^|\\.)naver\\.com$/, /(^|\\.)seznam\\.cz$/, /(^|\\.)yep\\.com$/];
|
|
34
|
+
var SOCIAL = ${js(SOCIAL_REFERRER_HOSTS)};
|
|
35
|
+
var IN_APP = /Instagram|FBAN|FBAV|TikTok|Bytedance/i;
|
|
36
|
+
var KEY = "katman_session_v1";
|
|
37
|
+
|
|
38
|
+
function hostOf(u) {
|
|
39
|
+
try { return new URL(u).hostname.replace(/^www\\./, "").toLowerCase(); } catch (e) { return ""; }
|
|
40
|
+
}
|
|
41
|
+
function matchHost(h, list) {
|
|
42
|
+
for (var i = 0; i < list.length; i++) {
|
|
43
|
+
var d = list[i];
|
|
44
|
+
if (h === d || h.slice(-(d.length + 1)) === "." + d) return d;
|
|
45
|
+
}
|
|
46
|
+
return "";
|
|
47
|
+
}
|
|
48
|
+
function classify(referrer, utmSource) {
|
|
49
|
+
var utm = (utmSource || "").toLowerCase();
|
|
50
|
+
for (var i = 0; i < AI_UTM.length; i++) if (utm.indexOf(AI_UTM[i]) !== -1) return { channel: "ai", source: utm };
|
|
51
|
+
var h = hostOf(referrer);
|
|
52
|
+
var self = location.hostname.replace(/^www\\./, "").toLowerCase();
|
|
53
|
+
if (!h || h === self) return { channel: "direct", source: utm };
|
|
54
|
+
var ai = matchHost(h, AI_HOSTS);
|
|
55
|
+
if (ai) return { channel: "ai", source: ai };
|
|
56
|
+
for (var j = 0; j < SEARCH.length; j++) if (SEARCH[j].test(h)) return { channel: "search", source: h };
|
|
57
|
+
var social = matchHost(h, SOCIAL);
|
|
58
|
+
if (social) return { channel: "social", source: social };
|
|
59
|
+
return { channel: "referral", source: h };
|
|
60
|
+
}
|
|
61
|
+
function load() {
|
|
62
|
+
try { var raw = sessionStorage.getItem(KEY); return raw ? JSON.parse(raw) : null; } catch (e) { return null; }
|
|
63
|
+
}
|
|
64
|
+
function save() {
|
|
65
|
+
try { sessionStorage.setItem(KEY, JSON.stringify(s)); } catch (e) { /* private mode */ }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
var s = load();
|
|
69
|
+
var isNew = !s;
|
|
70
|
+
if (!s) {
|
|
71
|
+
var c = classify(document.referrer, new URLSearchParams(location.search).get("utm_source"));
|
|
72
|
+
s = { channel: c.channel, source: c.source, landing: location.pathname, pages: 0, scrolled: false, conversion: "", inApp: IN_APP.test(navigator.userAgent), human: false };
|
|
73
|
+
}
|
|
74
|
+
s.pages += 1;
|
|
75
|
+
|
|
76
|
+
function props() {
|
|
77
|
+
return { katman_channel: s.channel, katman_source: s.source, katman_landing: s.landing, katman_human: s.human };
|
|
78
|
+
}
|
|
79
|
+
function push(event) {
|
|
80
|
+
var w = window, p = props();
|
|
81
|
+
try {
|
|
82
|
+
if (TARGET === "gtm") w.dataLayer = w.dataLayer || [];
|
|
83
|
+
if ((TARGET === "auto" || TARGET === "gtm") && Array.isArray(w.dataLayer)) w.dataLayer.push(Object.assign({ event: event }, p));
|
|
84
|
+
if ((TARGET === "auto" || TARGET === "gtag") && typeof w.gtag === "function") w.gtag("event", event, p);
|
|
85
|
+
if ((TARGET === "auto" || TARGET === "plausible") && typeof w.plausible === "function") w.plausible(event, { props: p });
|
|
86
|
+
if ((TARGET === "auto" || TARGET === "posthog") && w.posthog && typeof w.posthog.capture === "function") {
|
|
87
|
+
w.posthog.register({ katman_channel: s.channel, katman_source: s.source });
|
|
88
|
+
w.posthog.capture(event, p);
|
|
89
|
+
}
|
|
90
|
+
} catch (e) { /* never break the page */ }
|
|
91
|
+
}
|
|
92
|
+
function whenReady(fn) {
|
|
93
|
+
if (document.readyState === "complete") setTimeout(fn, 0);
|
|
94
|
+
else window.addEventListener("load", function () { setTimeout(fn, 0); });
|
|
95
|
+
}
|
|
96
|
+
function evaluate() {
|
|
97
|
+
var human = s.scrolled || s.pages > 2 || !!s.conversion || s.inApp;
|
|
98
|
+
if (human && !s.human) {
|
|
99
|
+
s.human = true;
|
|
100
|
+
whenReady(function () { push("katman_human"); });
|
|
101
|
+
}
|
|
102
|
+
api.channel = s.channel; api.source = s.source; api.landing = s.landing; api.pages = s.pages; api.human = s.human;
|
|
103
|
+
save();
|
|
104
|
+
}
|
|
105
|
+
var api = {
|
|
106
|
+
channel: s.channel, source: s.source, landing: s.landing, pages: s.pages, human: s.human,
|
|
107
|
+
/** Call on a real conversion (quiz finished, sign-up, payment): marks the session human. */
|
|
108
|
+
convert: function (name) { s.conversion = String(name || "conversion"); evaluate(); },
|
|
109
|
+
classify: classify
|
|
110
|
+
};
|
|
111
|
+
window.katmanChannel = api;
|
|
112
|
+
if (isNew) whenReady(function () { push("katman_session"); });
|
|
113
|
+
evaluate();
|
|
114
|
+
if (!s.scrolled) {
|
|
115
|
+
var onScroll = function () {
|
|
116
|
+
if ((window.scrollY || document.documentElement.scrollTop) > 0) {
|
|
117
|
+
s.scrolled = true;
|
|
118
|
+
window.removeEventListener("scroll", onScroll);
|
|
119
|
+
evaluate();
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
window.addEventListener("scroll", onScroll, { passive: true });
|
|
123
|
+
}
|
|
124
|
+
})();
|
|
125
|
+
`;
|
|
126
|
+
}
|
|
127
|
+
export function botBlock(ts) {
|
|
128
|
+
const rows = botTable()
|
|
129
|
+
.map(([tok, who]) => ` [${JSON.stringify(tok)}, ${JSON.stringify(who)}],`)
|
|
130
|
+
.join("\n");
|
|
131
|
+
return `// Katman bot counter (Katman book, ch. 15): bots don't run browser analytics, so count them on the server.
|
|
132
|
+
// Counts requests per day, per bot, per path. Bot list generated from Katman's crawler list.
|
|
133
|
+
const KATMAN_BOTS${ts ? ": [string, string][]" : ""} = [
|
|
134
|
+
${rows}
|
|
135
|
+
];
|
|
136
|
+
|
|
137
|
+
/** The bot's token (e.g. "OAI-SearchBot") or null for anything else. */
|
|
138
|
+
export function katmanBot(userAgent${ts ? ": string | null | undefined" : ""})${ts ? ": string | null" : ""} {
|
|
139
|
+
if (!userAgent) return null;
|
|
140
|
+
const ua = userAgent.toLowerCase();
|
|
141
|
+
for (const [token] of KATMAN_BOTS) if (ua.includes(token.toLowerCase())) return token;
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
`;
|
|
145
|
+
}
|
|
146
|
+
export function botCounterSnippet(platform) {
|
|
147
|
+
const common = botBlock(true);
|
|
148
|
+
switch (platform) {
|
|
149
|
+
case "cloudflare-worker":
|
|
150
|
+
return {
|
|
151
|
+
fileName: "bot-counter.cloudflare-worker.ts",
|
|
152
|
+
placeAt: "your Worker entry (or a pass-through Worker on your site's route)",
|
|
153
|
+
notes: [
|
|
154
|
+
"Add to wrangler.toml: [[analytics_engine_datasets]] binding = \"KATMAN_BOTS\", dataset = \"katman_bots\" (Workers Analytics Engine; free tier included). Types: @cloudflare/workers-types.",
|
|
155
|
+
"Don't count into KV: its write limits are too low for per-request counters. D1 works (INSERT … ON CONFLICT(day, bot, path) DO UPDATE SET hits = hits + 1).",
|
|
156
|
+
"Requests blocked by the WAF/bot settings never reach the Worker: if a bot never shows up here, check Security → Events (book ch. 6).",
|
|
157
|
+
],
|
|
158
|
+
code: `${common}
|
|
159
|
+
export interface Env {
|
|
160
|
+
/** Workers Analytics Engine dataset (wrangler.toml: [[analytics_engine_datasets]] binding = "KATMAN_BOTS", dataset = "katman_bots"). */
|
|
161
|
+
KATMAN_BOTS?: AnalyticsEngineDataset;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export default {
|
|
165
|
+
async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
|
|
166
|
+
const bot = katmanBot(request.headers.get("user-agent"));
|
|
167
|
+
// Pass through to your origin — or call your own handler here if this Worker serves the site.
|
|
168
|
+
const response = await fetch(request);
|
|
169
|
+
if (bot) {
|
|
170
|
+
const url = new URL(request.url);
|
|
171
|
+
// Storage adapter: Analytics Engine (day comes from the event timestamp).
|
|
172
|
+
env.KATMAN_BOTS?.writeDataPoint({ blobs: [bot, url.pathname, String(response.status)], doubles: [1], indexes: [bot] });
|
|
173
|
+
}
|
|
174
|
+
void ctx;
|
|
175
|
+
return response;
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
// Read it (Cloudflare SQL API for Analytics Engine):
|
|
180
|
+
// SELECT toStartOfInterval(timestamp, INTERVAL '1' DAY) AS day, blob1 AS bot, blob2 AS path,
|
|
181
|
+
// SUM(_sample_interval) AS hits
|
|
182
|
+
// FROM katman_bots WHERE timestamp > NOW() - INTERVAL '7' DAY
|
|
183
|
+
// GROUP BY day, bot, path ORDER BY day DESC, hits DESC
|
|
184
|
+
`,
|
|
185
|
+
};
|
|
186
|
+
case "nextjs-middleware":
|
|
187
|
+
return {
|
|
188
|
+
fileName: "bot-counter.nextjs-middleware.ts",
|
|
189
|
+
placeAt: "middleware.ts at the project root (or src/middleware.ts); on Next.js 16+ the file is proxy.ts and the export is `proxy`",
|
|
190
|
+
notes: [
|
|
191
|
+
"If the project already has a middleware/proxy file, merge the katmanBot() call into it instead of replacing it.",
|
|
192
|
+
"The default storage adapter logs one JSON line per bot hit (visible in your host's logs / log drain). Swap record() for Upstash Redis HINCRBY or a database upsert to keep daily counts.",
|
|
193
|
+
],
|
|
194
|
+
code: `${common}
|
|
195
|
+
import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server";
|
|
196
|
+
|
|
197
|
+
export function middleware(request: NextRequest, event: NextFetchEvent) {
|
|
198
|
+
const bot = katmanBot(request.headers.get("user-agent"));
|
|
199
|
+
if (bot) event.waitUntil(record({ day: new Date().toISOString().slice(0, 10), bot, path: request.nextUrl.pathname }));
|
|
200
|
+
return NextResponse.next();
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export const config = { matcher: ["/((?!_next/static|_next/image|favicon.ico).*)"] };
|
|
204
|
+
|
|
205
|
+
/** Storage adapter: replace with your store (Upstash Redis HINCRBY katman:bots:<day> "<bot> <path>" 1, a DB upsert, …). */
|
|
206
|
+
async function record(hit: { day: string; bot: string; path: string }): Promise<void> {
|
|
207
|
+
console.log(JSON.stringify({ katman_bot: hit }));
|
|
208
|
+
}
|
|
209
|
+
`,
|
|
210
|
+
};
|
|
211
|
+
case "express":
|
|
212
|
+
return {
|
|
213
|
+
fileName: "bot-counter.express.ts",
|
|
214
|
+
placeAt: "your Express server: app.use(katmanBotCounter) before the routes",
|
|
215
|
+
notes: [
|
|
216
|
+
"Counts are kept in memory and appended to katman-bots.tsv every minute (day, bot, path, hits); sum rows with the same day/bot/path when you read it. Swap flush() for your database.",
|
|
217
|
+
],
|
|
218
|
+
code: `${common}
|
|
219
|
+
import type { NextFunction, Request, Response } from "express";
|
|
220
|
+
import { appendFile } from "node:fs/promises";
|
|
221
|
+
|
|
222
|
+
const counts = new Map<string, number>();
|
|
223
|
+
|
|
224
|
+
export function katmanBotCounter(req: Request, _res: Response, next: NextFunction): void {
|
|
225
|
+
const bot = katmanBot(req.get("user-agent"));
|
|
226
|
+
if (bot) {
|
|
227
|
+
const key = \`\${new Date().toISOString().slice(0, 10)}\\t\${bot}\\t\${req.path}\`;
|
|
228
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
229
|
+
}
|
|
230
|
+
next();
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Storage adapter: a TSV file (day, bot, path, hits). Replace with a database upsert if you have one. */
|
|
234
|
+
async function flush(): Promise<void> {
|
|
235
|
+
if (!counts.size) return;
|
|
236
|
+
const lines = [...counts].map(([k, n]) => \`\${k}\\t\${n}\\n\`).join("");
|
|
237
|
+
counts.clear();
|
|
238
|
+
await appendFile("katman-bots.tsv", lines).catch(() => undefined);
|
|
239
|
+
}
|
|
240
|
+
setInterval(() => void flush(), 60_000).unref();
|
|
241
|
+
`,
|
|
242
|
+
};
|
|
243
|
+
case "astro-middleware":
|
|
244
|
+
return {
|
|
245
|
+
fileName: "bot-counter.astro-middleware.ts",
|
|
246
|
+
placeAt: "src/middleware.ts",
|
|
247
|
+
notes: [
|
|
248
|
+
"Astro middleware runs only for on-demand (SSR) routes. Prerendered pages are served as static files and never reach it: for a static Astro site count bots at the edge (platform cloudflare-worker) or in your host's logs.",
|
|
249
|
+
"Swap record() for your store; the default logs one JSON line per bot hit.",
|
|
250
|
+
],
|
|
251
|
+
code: `${common}
|
|
252
|
+
import { defineMiddleware } from "astro:middleware";
|
|
253
|
+
|
|
254
|
+
export const onRequest = defineMiddleware(async (context, next) => {
|
|
255
|
+
const bot = katmanBot(context.request.headers.get("user-agent"));
|
|
256
|
+
const response = await next();
|
|
257
|
+
if (bot) await record({ day: new Date().toISOString().slice(0, 10), bot, path: context.url.pathname, status: response.status });
|
|
258
|
+
return response;
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
/** Storage adapter: replace with your store (a DB upsert, Redis HINCRBY, …). */
|
|
262
|
+
async function record(hit: { day: string; bot: string; path: string; status: number }): Promise<void> {
|
|
263
|
+
console.log(JSON.stringify({ katman_bot: hit }));
|
|
264
|
+
}
|
|
265
|
+
`,
|
|
266
|
+
};
|
|
267
|
+
case "sveltekit-hooks":
|
|
268
|
+
return {
|
|
269
|
+
fileName: "bot-counter.sveltekit-hooks.ts",
|
|
270
|
+
placeAt: "src/hooks.server.ts (merge with an existing handle using sequence())",
|
|
271
|
+
notes: [
|
|
272
|
+
"Prerendered pages don't pass through hooks at request time: count those at the edge (platform cloudflare-worker) or in your host's logs.",
|
|
273
|
+
"Swap record() for your store; the default logs one JSON line per bot hit.",
|
|
274
|
+
],
|
|
275
|
+
code: `${common}
|
|
276
|
+
import type { Handle } from "@sveltejs/kit";
|
|
277
|
+
|
|
278
|
+
export const handle: Handle = async ({ event, resolve }) => {
|
|
279
|
+
const bot = katmanBot(event.request.headers.get("user-agent"));
|
|
280
|
+
const response = await resolve(event);
|
|
281
|
+
if (bot) await record({ day: new Date().toISOString().slice(0, 10), bot, path: event.url.pathname, status: response.status });
|
|
282
|
+
return response;
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
/** Storage adapter: replace with your store (a DB upsert, Redis HINCRBY, …). */
|
|
286
|
+
async function record(hit: { day: string; bot: string; path: string; status: number }): Promise<void> {
|
|
287
|
+
console.log(JSON.stringify({ katman_bot: hit }));
|
|
288
|
+
}
|
|
289
|
+
`,
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
}
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import { logEvent, saveConfig, saveResearch } from "../../../state.js";
|
|
2
|
+
import { domainOf, mdTable, normalizeSiteUrl } from "../../../util.js";
|
|
3
|
+
import { PANEL_COST_TEXT, calls, fmtCallBlock } from "../lib/calls.js";
|
|
4
|
+
import { keySituationQuestionIds } from "./questions.js";
|
|
5
|
+
import { fwLabel } from "../lib/framework.js";
|
|
6
|
+
import { hasKatmanDir, katmanMtime, loadConfigSafe, loadLatestAudit, loadResearchSafe, loadRuns, loadScan, runDate } from "../lib/state.js";
|
|
7
|
+
import { addDays, isoDay } from "../lib/text.js";
|
|
8
|
+
import { methodSummaryMd } from "../data/render.js";
|
|
9
|
+
import { API_PROXY_NOTE } from "../data/measurement.js";
|
|
10
|
+
import { t } from "../lib/l10n.js";
|
|
11
|
+
import { researchCompleteness } from "./research.js";
|
|
12
|
+
import { loadPlanState, openTasks, taskCall } from "./plan.js";
|
|
13
|
+
import { siteUrlOf } from "./prompt.js";
|
|
14
|
+
const REMEASURE_DAYS = 14;
|
|
15
|
+
export function firstRunText(root) {
|
|
16
|
+
return [
|
|
17
|
+
"**First run in this project. What happens next:**",
|
|
18
|
+
`1. Katman keeps its notes in \`${root}/.katman/\` (safe to commit, never secrets). It writes nowhere else unless you call katman_generate or katman_setup_monitoring with write: true, and it never overwrites without overwrite: true.`,
|
|
19
|
+
"2. It reads your code (skipping node_modules, .git and build output; never .env*, keys or secrets) and, when you run the audit tools, your own live site.",
|
|
20
|
+
"3. Network only when a tool needs it: your site, openrouter.ai with YOUR key after a cost estimate (dry run first), api.indexnow.org on request, and katman.pro only to check KATMAN_KEY (at most once a day, only when it is set) or when you call katman_sync. No telemetry.",
|
|
21
|
+
"4. The order: research (who you are, your customers' own sentences) → plan → build with prompts P0–P14 → verify, then re-measure every 14 days.",
|
|
22
|
+
"5. It never invents facts: unknowns stay [FILL] and you'll be asked.",
|
|
23
|
+
].join("\n");
|
|
24
|
+
}
|
|
25
|
+
export async function runStart(root, opts) {
|
|
26
|
+
const now = opts.now ?? new Date();
|
|
27
|
+
const firstRun = !hasKatmanDir(root);
|
|
28
|
+
const notes = [];
|
|
29
|
+
let config = await loadConfigSafe(root);
|
|
30
|
+
let research = await loadResearchSafe(root);
|
|
31
|
+
const lang = opts.lang ?? config.lang ?? "en";
|
|
32
|
+
if (opts.siteUrl) {
|
|
33
|
+
let url;
|
|
34
|
+
try {
|
|
35
|
+
url = normalizeSiteUrl(opts.siteUrl);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return { text: `"${opts.siteUrl}" isn't a valid site URL. Pass the live address, e.g. https://example.com.`, isError: true };
|
|
39
|
+
}
|
|
40
|
+
config = { ...config, siteUrl: url };
|
|
41
|
+
await saveConfig(root, config);
|
|
42
|
+
if (research) {
|
|
43
|
+
research.site = { ...research.site, url, domain: domainOf(url) };
|
|
44
|
+
await saveResearch(root, research);
|
|
45
|
+
}
|
|
46
|
+
await logEvent(root, `katman_start: site URL set to ${url}`);
|
|
47
|
+
notes.push(`Saved the site URL ${url} to .katman/config.json${research ? " and research.json" : ""}.`);
|
|
48
|
+
}
|
|
49
|
+
const [scan, loaded, runs, planState] = await Promise.all([loadScan(root), loadLatestAudit(root), loadRuns(root), loadPlanState(root)]);
|
|
50
|
+
const audit = loaded?.audit;
|
|
51
|
+
const siteUrl = siteUrlOf(research, config);
|
|
52
|
+
const completeness = researchCompleteness(research);
|
|
53
|
+
const plan = planState.plan;
|
|
54
|
+
const planMtime = await katmanMtime(root, "plan.md");
|
|
55
|
+
const open = plan ? openTasks(plan) : [];
|
|
56
|
+
const doneCount = plan ? plan.tasks.length - open.length : 0;
|
|
57
|
+
const lastRun = runs.length ? runs[runs.length - 1] : undefined;
|
|
58
|
+
const lastRunDate = lastRun ? runDate(lastRun) : undefined;
|
|
59
|
+
const dueDate = lastRunDate ? addDays(lastRunDate, REMEASURE_DAYS) : undefined;
|
|
60
|
+
const remeasureDue = !!dueDate && dueDate.getTime() <= now.getTime();
|
|
61
|
+
const auditDate = audit?.ts ? new Date(audit.ts) : undefined;
|
|
62
|
+
const auditStale = !!(auditDate && planMtime && doneCount > 0 && auditDate.getTime() < planMtime.getTime());
|
|
63
|
+
// ------------------------------------------------------------ status table
|
|
64
|
+
const steps = [];
|
|
65
|
+
steps.push({
|
|
66
|
+
step: "1 Research",
|
|
67
|
+
state: research ? `${completeness.percent}%` : "not started",
|
|
68
|
+
detail: research ? (completeness.gaps.length ? `missing: ${completeness.gaps.slice(0, 4).join("; ")}` : "complete") : "no .katman/research.json",
|
|
69
|
+
});
|
|
70
|
+
steps.push({
|
|
71
|
+
step: "2 Plan",
|
|
72
|
+
state: [`scan ${scan ? "yes" : "no"}`, `audit ${audit ? "yes" : "no"}`, `plan ${plan ? "yes" : "no"}`].join(" · "),
|
|
73
|
+
detail: [
|
|
74
|
+
scan ? `${fwLabel(scan)}` : "no scan",
|
|
75
|
+
audit ? `audit ${isoDay(audit.ts)} (${audit.pages.length} pages)` : siteUrl ? "no audit yet" : "no site URL → no audit",
|
|
76
|
+
plan ? `plan ${isoDay(plan.generatedAt)}` : "",
|
|
77
|
+
].filter(Boolean).join(" · "),
|
|
78
|
+
});
|
|
79
|
+
steps.push({
|
|
80
|
+
step: "3 Build",
|
|
81
|
+
state: plan ? `${doneCount}/${plan.tasks.length} tasks (${plan.tasks.length ? Math.round((doneCount / plan.tasks.length) * 100) : 0}%)` : "–",
|
|
82
|
+
detail: plan ? (open[0] ? `next: ${open[0].title}` : "all tasks done") : "no plan yet",
|
|
83
|
+
});
|
|
84
|
+
steps.push({
|
|
85
|
+
step: "4 Verify",
|
|
86
|
+
state: audit ? `read ${audit.gates.read.pass}/${audit.gates.read.total} · quote ${audit.gates.quote.pass}/${audit.gates.quote.total} · fit ${audit.gates.fit.pass}/${audit.gates.fit.total}` : "–",
|
|
87
|
+
detail: audit ? (auditStale ? "plan changed after the last audit → re-verify" : `last audit ${isoDay(audit.ts)}`) : "no audit yet",
|
|
88
|
+
});
|
|
89
|
+
steps.push({
|
|
90
|
+
step: "Measure",
|
|
91
|
+
state: runs.length ? `${runs.length} run(s), last ${isoDay(lastRunDate ?? lastRun.ts)}` : "no runs",
|
|
92
|
+
detail: runs.length
|
|
93
|
+
? `mentioned ${Math.round(lastRun.summary.mentionRate * 100)}% · recommended first ${Math.round(lastRun.summary.recommendedFirstRate * 100)}% · cited ${Math.round(lastRun.summary.citationRate * 100)}% · re-run ${remeasureDue ? "DUE now" : `due ${isoDay(dueDate)}`}`
|
|
94
|
+
: "baseline needed (question set × assistants)",
|
|
95
|
+
});
|
|
96
|
+
// ------------------------------------------------------------ next step
|
|
97
|
+
const also = [];
|
|
98
|
+
let next;
|
|
99
|
+
const orKey = opts.hasOpenRouterKey;
|
|
100
|
+
const measureCall = orKey ? calls.visibilityDryRun(root) : calls.manualCheck(root);
|
|
101
|
+
if (!research) {
|
|
102
|
+
next = {
|
|
103
|
+
call: calls.researchTemplate(root),
|
|
104
|
+
title: "Create the research file (step 1)",
|
|
105
|
+
why: "Research comes first: the customer's own sentences decide which pages get built (book ch. 7).",
|
|
106
|
+
then: [
|
|
107
|
+
"Fill it from the codebase: README, landing-page copy, the pricing page, legal pages/footer (brand name, category, pricing, company, a first draft of the one sentence).",
|
|
108
|
+
"Ask the user for 10–12 customer sentences (what customers type into ChatGPT, no brand name), what the brand gets confused with, and the competitors assistants name today.",
|
|
109
|
+
"Save it with katman_research mode \"update\"; never invent numbers, reviews or competitor facts.",
|
|
110
|
+
],
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
else if (completeness.blocking.length) {
|
|
114
|
+
const onlyQuestions = completeness.blocking.length === 1 && completeness.blocking[0] === "question set";
|
|
115
|
+
next = onlyQuestions
|
|
116
|
+
? {
|
|
117
|
+
call: calls.generateQuestions(root),
|
|
118
|
+
title: "Build the question set (12 unbranded + 2 diagnostic)",
|
|
119
|
+
why: "Measurement and the plan need a fixed question set (book ch. 15).",
|
|
120
|
+
then: ["Show the questions to the user once; after the first run keep them fixed."],
|
|
121
|
+
}
|
|
122
|
+
: {
|
|
123
|
+
call: calls.researchUpdate(root, gapSkeleton(completeness.blocking)),
|
|
124
|
+
title: `Finish research: ${completeness.blocking.join(", ")}`,
|
|
125
|
+
why: "The plan and every prompt are filled from research; missing parts would become [FILL].",
|
|
126
|
+
then: [
|
|
127
|
+
"Take what you can from the codebase; ask the user for the customer sentences and anything you'd otherwise have to guess.",
|
|
128
|
+
research.situations.length >= 3 ? "Add \"generate_questions\": true in the same call to build the question set." : "Once there are ≥3 situations, add \"generate_questions\": true.",
|
|
129
|
+
],
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
else if (!scan) {
|
|
133
|
+
next = {
|
|
134
|
+
call: calls.scan(root),
|
|
135
|
+
title: "Scan the codebase",
|
|
136
|
+
why: "The scan finds the framework, how pages render (text only in JavaScript is trap 1), the routes and head tags — it decides P2 and where every file goes.",
|
|
137
|
+
then: ["Then call katman_start again."],
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
else if (!audit && siteUrl) {
|
|
141
|
+
next = {
|
|
142
|
+
call: calls.audit(root, siteUrl),
|
|
143
|
+
title: "Audit the live site the way AI crawlers see it",
|
|
144
|
+
why: "What bots actually get: robots.txt, CDN/firewall blocks (403), text with JavaScript off, soft 404s, canonicals, sitemap.",
|
|
145
|
+
then: ["Report only (P1); don't fix anything yet. Then call katman_start again."],
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
else if (!runs.length) {
|
|
149
|
+
next = {
|
|
150
|
+
call: measureCall,
|
|
151
|
+
title: "Take the baseline measurement",
|
|
152
|
+
why: "Whether assistants name, recommend and cite the site today — the before picture to compare against in 14 days.",
|
|
153
|
+
then: orKey
|
|
154
|
+
? [`Show the user the dry-run estimate and wait for their OK before running for real (${PANEL_COST_TEXT}).`, t(API_PROXY_NOTE, lang)]
|
|
155
|
+
: ["No OpenRouter key is set (OPENROUTER_API_KEY), so use the manual check (no API cost): the user asks the questions in the consumer apps, logged out, web search on, and pastes the answers.", "To measure through APIs later, the user sets OPENROUTER_API_KEY in the MCP config."],
|
|
156
|
+
};
|
|
157
|
+
if (!siteUrl)
|
|
158
|
+
also.push(domainHint(scan?.domains));
|
|
159
|
+
}
|
|
160
|
+
else if (!plan) {
|
|
161
|
+
next = {
|
|
162
|
+
call: calls.plan(root),
|
|
163
|
+
title: "Build the plan",
|
|
164
|
+
why: "Turns research + scan + audit + the baseline into the page plan and a 30-day task list, each task with an acceptance criterion.",
|
|
165
|
+
then: ["Show the plan to the user and get approval before changing any code (P0)."],
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
else if (open.length) {
|
|
169
|
+
const task = open[0];
|
|
170
|
+
const call = taskCall(root, task, siteUrl);
|
|
171
|
+
next = {
|
|
172
|
+
...(call ? { call } : {}),
|
|
173
|
+
title: `Next task: ${task.title}`,
|
|
174
|
+
why: `Week ${task.week} of the 30-day plan. Accept: ${task.accept}`,
|
|
175
|
+
then: [call ? "Do the task, run its check, and tick it in .katman/plan.md only when the check passes." : "This task is done by hand; tick it in .katman/plan.md when it's done.", `Verify with: ${task.verifiedBy}`],
|
|
176
|
+
};
|
|
177
|
+
if (auditStale && siteUrl)
|
|
178
|
+
also.push(`Pages changed since the last audit: re-verify with katman_audit_site ${JSON.stringify({ urls: [`${siteUrl}/`] })} (quick mode) or a full audit.`);
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
next = {
|
|
182
|
+
call: siteUrl ? calls.audit(root, siteUrl) : calls.scan(root),
|
|
183
|
+
title: "Verify, notify, re-measure",
|
|
184
|
+
why: "Every task is ticked: prove it on the live site, tell the search engines about changed URLs, and re-measure on schedule.",
|
|
185
|
+
then: [
|
|
186
|
+
"katman_indexnow (dry run first) for new or changed URLs only; then Search Console, Bing URL Submission and Brave submit-url by hand.",
|
|
187
|
+
`Re-measure ${dueDate ? `on ${isoDay(dueDate)}` : "14 days after the last run"} with the same questions, models and temperature.`,
|
|
188
|
+
"katman_report for a shareable summary.",
|
|
189
|
+
],
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const keyIds = keySituationQuestionIds(research);
|
|
193
|
+
if (remeasureDue && runs.length) {
|
|
194
|
+
also.push(`The re-measurement is due (last run ${isoDay(lastRunDate)}), same questions as before: ${orKey
|
|
195
|
+
? `katman_visibility_run ${JSON.stringify({ dry_run: true })} for the full set, plus ${JSON.stringify({ dry_run: true, question_ids: keyIds, samples: 3 })} to read how often you're named on the key situation questions (${PANEL_COST_TEXT})`
|
|
196
|
+
: "katman_manual_check (the key situation questions, in the real apps)"}.`);
|
|
197
|
+
}
|
|
198
|
+
if (!siteUrl && scan && next.call?.tool !== "katman_visibility_run" && next.call?.tool !== "katman_manual_check" && !also.some((a) => a.includes("site_url")))
|
|
199
|
+
also.push(domainHint(scan.domains));
|
|
200
|
+
if (runs.length && orKey)
|
|
201
|
+
also.push(`Confirm the key questions in the real apps too: katman_manual_check (${t(API_PROXY_NOTE, lang).split(":")[0]}).`);
|
|
202
|
+
if (plan) {
|
|
203
|
+
const planAt = new Date(plan.generatedAt).getTime();
|
|
204
|
+
const newer = [
|
|
205
|
+
research?.updatedAt && new Date(research.updatedAt).getTime() > planAt ? "research" : "",
|
|
206
|
+
scan?.ts && new Date(scan.ts).getTime() > planAt ? "the scan" : "",
|
|
207
|
+
audit?.ts && new Date(audit.ts).getTime() > planAt ? "the audit" : "",
|
|
208
|
+
].filter(Boolean);
|
|
209
|
+
if (newer.length && Number.isFinite(planAt))
|
|
210
|
+
also.push(`${newer.join(", ")} changed after the plan was built: run katman_plan again to fold in new situations and findings (your ticks are kept).`);
|
|
211
|
+
}
|
|
212
|
+
if (research && completeness.percent < 100 && !completeness.blocking.length)
|
|
213
|
+
also.push(`Research is ${completeness.percent}%: close the gaps this week (${completeness.gaps.slice(0, 3).join("; ")}).`);
|
|
214
|
+
// ------------------------------------------------------------ output
|
|
215
|
+
const parts = [];
|
|
216
|
+
if (opts.warning)
|
|
217
|
+
parts.push(`**Warning:** ${opts.warning}`);
|
|
218
|
+
parts.push(`# Katman — ${research?.brand.name && !/\[FILL/i.test(research.brand.name) ? research.brand.name : "project"} (${root})`);
|
|
219
|
+
if (firstRun)
|
|
220
|
+
parts.push(firstRunText(root));
|
|
221
|
+
if (opts.license?.line)
|
|
222
|
+
parts.push(opts.license.line);
|
|
223
|
+
if (notes.length)
|
|
224
|
+
parts.push(notes.join("\n"));
|
|
225
|
+
// Full method overview only while research hasn't started; afterwards one line keeps the agent's context lean.
|
|
226
|
+
if (firstRun || !research)
|
|
227
|
+
parts.push(methodSummaryMd(lang));
|
|
228
|
+
else
|
|
229
|
+
parts.push(lang === "tr"
|
|
230
|
+
? "Yöntem: 3 kapı (Okunmak · Alıntılanmak · Soruya uymak) · 4 adım (Araştır → Planla → Yaptır → Doğrula) · tamamı: kaynak `katman://method`."
|
|
231
|
+
: "Method: 3 gates (Read · Quote · Fit) · 4 steps (Research → Plan → Build → Verify) · full overview: resource `katman://method`.");
|
|
232
|
+
parts.push(`## Status\n${mdTable(["Step", "State", "Detail"], steps.map((s) => [s.step, s.state, s.detail]))}`);
|
|
233
|
+
parts.push(`## Next step: ${next.title}\n${next.call ? fmtCallBlock(next.call) : "_(by hand)_"}\n**Why:** ${next.why}${next.then.length ? `\n\n${next.then.map((x) => `- ${x}`).join("\n")}` : ""}`);
|
|
234
|
+
const proNote = next.call ? opts.license?.notice?.(next.call.tool) : undefined;
|
|
235
|
+
if (proNote)
|
|
236
|
+
parts.push(proNote);
|
|
237
|
+
if (also.length)
|
|
238
|
+
parts.push(`## Also\n${also.map((x) => `- ${x}`).join("\n")}`);
|
|
239
|
+
parts.push("Call katman_start again after each step. Never invent facts; write [FILL] and ask the user.");
|
|
240
|
+
return {
|
|
241
|
+
text: parts.join("\n\n"),
|
|
242
|
+
structured: {
|
|
243
|
+
root,
|
|
244
|
+
firstRun,
|
|
245
|
+
siteUrl: siteUrl ?? null,
|
|
246
|
+
research: { exists: !!research, percent: completeness.percent, gaps: completeness.gaps, blocking: completeness.blocking },
|
|
247
|
+
scan: scan ? { framework: scan.framework.name, rendering: scan.rendering.verdict, ts: scan.ts } : null,
|
|
248
|
+
audit: audit ? { ts: audit.ts, gates: audit.gates } : null,
|
|
249
|
+
plan: plan ? { tasks: plan.tasks.length, done: doneCount, nextTask: open[0]?.id ?? null } : null,
|
|
250
|
+
measure: { runs: runs.length, lastRun: lastRunDate ? lastRunDate.toISOString() : null, due: dueDate ? dueDate.toISOString() : null, dueNow: remeasureDue, openRouterKey: orKey },
|
|
251
|
+
steps,
|
|
252
|
+
next: { tool: next.call?.tool ?? null, args: next.call?.args ?? null, title: next.title, why: next.why, ...(opts.license ? { needsPro: !!proNote } : {}) },
|
|
253
|
+
...(opts.license?.structured ? { license: opts.license.structured } : {}),
|
|
254
|
+
},
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
function domainHint(domains) {
|
|
258
|
+
return domains?.length
|
|
259
|
+
? `No site URL yet. The hosting config names ${domains.map((d) => `https://${d}`).join(", ")}: confirm the live address with the user, then call katman_start with site_url (and katman_audit_site).`
|
|
260
|
+
: "No site URL yet: ask the user for the live address and call katman_start with site_url, then katman_audit_site.";
|
|
261
|
+
}
|
|
262
|
+
/** A data skeleton for exactly the blocking gaps (the agent fills it). */
|
|
263
|
+
function gapSkeleton(blocking) {
|
|
264
|
+
const data = {};
|
|
265
|
+
const brand = {};
|
|
266
|
+
if (blocking.includes("brand name"))
|
|
267
|
+
brand.name = "<one distinctive name>";
|
|
268
|
+
if (blocking.includes("category"))
|
|
269
|
+
brand.category = "<category>";
|
|
270
|
+
if (blocking.includes("one sentence (4 parts)"))
|
|
271
|
+
brand.oneLiner = { brings: "<what the customer brings>", weDo: "<what we do>", gets: "<what they get>", fear: "<which fear is unnecessary>" };
|
|
272
|
+
if (Object.keys(brand).length)
|
|
273
|
+
data.brand = brand;
|
|
274
|
+
if (blocking.includes("at least 3 situations"))
|
|
275
|
+
data.situations = [{ sentence: "<the customer's own sentence, no brand name>" }];
|
|
276
|
+
return data;
|
|
277
|
+
}
|