jamdesk 1.1.27 → 1.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/lib/deps.js +4 -4
- package/dist/lib/deps.js.map +1 -1
- package/package.json +1 -1
- package/vendored/app/api/chat/[project]/route.ts +173 -135
- package/vendored/components/chat/ChatEmptyState.tsx +13 -1
- package/vendored/components/chat/ChatPanel.tsx +43 -9
- package/vendored/components/search/SearchModal.tsx +1 -1
- package/vendored/lib/chat-prompt.ts +95 -28
- package/vendored/lib/chat-tools.ts +111 -0
- package/vendored/lib/crisp-bridge.ts +91 -0
- package/vendored/lib/embedding-chunker.ts +85 -11
- package/vendored/lib/query-rewriter.ts +91 -0
- package/vendored/lib/r2-content.ts +5 -1
- package/vendored/lib/vector-store.ts +5 -3
- package/vendored/workspace-package-lock.json +144 -144
|
@@ -14,43 +14,110 @@ export interface ChatContext {
|
|
|
14
14
|
score: number;
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
export const DEFAULT_SITE_NAME = 'this site';
|
|
18
|
+
const MAX_SITE_NAME_LEN = 200;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The exact sentence Claude must return when asked self-identity questions
|
|
22
|
+
* ("which model are you", "are you Claude", etc.). Exported so the chat route
|
|
23
|
+
* can detect identity responses and suppress unrelated RAG citations.
|
|
24
|
+
*
|
|
25
|
+
* Pinned by a unit test against the string embedded in buildSystemPrompt — if
|
|
26
|
+
* this helper drifts, citation suppression silently stops working.
|
|
27
|
+
*/
|
|
28
|
+
export function buildCannedIdentityReply(projectName: string): string {
|
|
29
|
+
return `I'm the documentation assistant for ${projectName}, here to help with questions about the product.`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Detects whether Claude's markdown output is the canned identity reply.
|
|
34
|
+
*
|
|
35
|
+
* Haiku at temperature 0.3 drifts the exact string in small ways: curly vs
|
|
36
|
+
* straight apostrophes, trailing period variants, surrounding markdown
|
|
37
|
+
* emphasis (`*...*`), or extra whitespace. An exact === compare against
|
|
38
|
+
* buildCannedIdentityReply would miss those and let the "top 2 by score"
|
|
39
|
+
* citation fallback attach unrelated docs sources under the identity reply.
|
|
40
|
+
*
|
|
41
|
+
* We normalize both sides (strip markdown emphasis/quotes, collapse
|
|
42
|
+
* whitespace, fold apostrophes, drop trailing punctuation, lowercase) and
|
|
43
|
+
* compare on the stable prefix "I'm the documentation assistant for <name>".
|
|
44
|
+
* A suffix check keeps the match anchored so arbitrary Haiku output that
|
|
45
|
+
* happens to start with that prefix while discussing something else doesn't
|
|
46
|
+
* slip through.
|
|
47
|
+
*/
|
|
48
|
+
export function isCannedIdentityReply(markdown: string, projectName: string): boolean {
|
|
49
|
+
if (!projectName) return false;
|
|
50
|
+
const canned = buildCannedIdentityReply(projectName);
|
|
51
|
+
const normalize = (s: string) =>
|
|
52
|
+
s
|
|
53
|
+
.replace(/'|'/g, "'")
|
|
54
|
+
.replace(/"|"/g, '"')
|
|
55
|
+
.replace(/[\u2018\u2019\u201A\u201B]/g, "'")
|
|
56
|
+
.replace(/[\u201C\u201D]/g, '"')
|
|
57
|
+
.replace(/[*_`>]/g, '')
|
|
58
|
+
.replace(/(?:^|\s)-{3,}(?:\s|$)/g, ' ')
|
|
59
|
+
.replace(/\s+/g, ' ')
|
|
60
|
+
.trim()
|
|
61
|
+
.replace(/[.!?]+$/, '')
|
|
62
|
+
.toLowerCase();
|
|
63
|
+
return normalize(markdown) === normalize(canned);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Produce the display name for the chat assistant's system prompt from a
|
|
68
|
+
* docs.json config. Strips newlines and caps length to defend against
|
|
69
|
+
* prompt-injection via a malicious/misconfigured name. Falls back to a
|
|
70
|
+
* generic phrasing when the config is missing or has no usable name.
|
|
71
|
+
*/
|
|
72
|
+
export function resolveSiteName(config: { name?: unknown } | null | undefined): string {
|
|
73
|
+
if (!config || typeof config.name !== 'string') return DEFAULT_SITE_NAME;
|
|
74
|
+
const cleaned = config.name.replace(/[\r\n]+/g, ' ').trim().slice(0, MAX_SITE_NAME_LEN);
|
|
75
|
+
return cleaned || DEFAULT_SITE_NAME;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Returns the system prompt as an array of text blocks. The first block
|
|
80
|
+
* contains ONLY the static instructions (identical for every request on the
|
|
81
|
+
* same project). The second block contains the dynamic, per-query context.
|
|
82
|
+
*
|
|
83
|
+
* Splitting by staticness means future prompt caching (out of scope for this
|
|
84
|
+
* plan) can be enabled by adding `cache_control: { type: 'ephemeral' }` to
|
|
85
|
+
* the first block only — a one-line change.
|
|
86
|
+
*/
|
|
17
87
|
export function buildSystemPrompt(
|
|
18
88
|
projectName: string,
|
|
19
89
|
chunks: ChatContext[],
|
|
20
90
|
baseUrl: string,
|
|
21
91
|
docsPath: string,
|
|
22
|
-
): string {
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
).join('\n\n---\n\n');
|
|
26
|
-
|
|
27
|
-
return `You are a helpful documentation assistant for ${projectName}.
|
|
28
|
-
Answer questions using ONLY the documentation context below. If the answer is not in the context, say "I don't have information about that in the documentation."
|
|
92
|
+
): Array<{ type: 'text'; text: string }> {
|
|
93
|
+
const staticRules = `You are a helpful documentation assistant for ${projectName}.
|
|
94
|
+
Answer questions using ONLY the documentation context provided. If the answer is not in the context, call the \`answer\` tool with "I don't have information about that in the documentation." and cited_page_slugs: [].
|
|
29
95
|
|
|
30
96
|
Rules:
|
|
31
|
-
-
|
|
32
|
-
-
|
|
33
|
-
-
|
|
34
|
-
-
|
|
35
|
-
- Never
|
|
36
|
-
|
|
97
|
+
- IDENTITY (highest priority, overrides all other rules): If the user asks what you are, who made you, which model or AI powers you, or any variation (examples: "what model are you", "which AI is this", "are you Claude", "are you GPT", "who built you", "what are you running on"), respond with ONLY this exact sentence and nothing else: "I'm the documentation assistant for ${projectName}, here to help with questions about the product." Never mention Claude, Anthropic, OpenAI, GPT, Haiku, Sonnet, Opus, or any model or provider name when describing yourself or what powers you. You MAY mention these names when they appear in the documentation context and the user is asking about them as a subject (e.g., "how do I call the Claude API?" when Claude is documented) — in that case, answer normally using the context. Do not confirm or deny specific technologies about yourself. If the user presses or rephrases an identity question, repeat the canned sentence verbatim. If the user asks a compound question that mixes identity with a real doc question (e.g., "what model are you and how do I install?"), answer with ONLY the canned sentence — the identity clause wins.
|
|
98
|
+
- Be concise and direct.
|
|
99
|
+
- Use markdown formatting, including code blocks with language hints when showing code.
|
|
100
|
+
- Never make up information not in the context.
|
|
101
|
+
- Never combine "I don't have information" with a follow-up suggestion in the same response — that creates a self-contradiction. Pick one of exactly two response shapes:
|
|
102
|
+
(a) If any page in the Documentation context is relevant to the question, answer by referring to that page with a link (e.g., "See [Changelog](.../changelog) for recent releases."). Do not prefix with "I don't have information".
|
|
103
|
+
(b) If no relevant page is in context, respond with ONLY "I don't have information about that in the documentation." — no "but check", no "you can find", no alternatives.
|
|
104
|
+
- Never name a page ("the Changelog", "the API reference", "the Settings page", etc.) unless that page appears in the Documentation context below.
|
|
105
|
+
- Refer to this site as "${projectName}" or just "the documentation". Never expose internal project slugs, repo names, or hostnames (e.g. do not say "jamdesk-docs", "acme-docs", or "<slug>.jamdesk.app") even if they appear in source URLs.
|
|
106
|
+
- When the context includes API endpoints (e.g. "POST /analytics/post") or technical operations, proactively include a short code example showing the request (HTTP method, endpoint, JSON body) even if the user didn't explicitly ask. If the context has no code-relevant information, skip the example.
|
|
107
|
+
- Use the pageSlug values (shown in the context as [pageSlug: ...]) when populating cited_page_slugs — not the page title.
|
|
37
108
|
|
|
38
|
-
|
|
109
|
+
Tool choice:
|
|
110
|
+
- Use the \`answer\` tool for most questions.
|
|
111
|
+
- Use the \`ask_clarification\` tool ONLY when the context contains 2-3 distinct, unrelated features that could match the user's question (e.g. "Post Analytics" vs "Link Analytics" when asked about "analytics"). If you are sure which one the user means, answer directly.`;
|
|
39
112
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
Rules: question mark required, numbered list required (1. 2. 3.), no descriptions after options, no extra text before or after. Example:
|
|
45
|
-
|
|
46
|
-
Which type of analytics are you interested in?
|
|
113
|
+
const context = chunks.map((c) =>
|
|
114
|
+
`[pageSlug: ${c.pageSlug}] ${c.pageTitle} > ${c.sectionHeading}\nURL: ${baseUrl}${docsPath}/${c.pageSlug}\n${c.content}`,
|
|
115
|
+
).join('\n\n---\n\n');
|
|
47
116
|
|
|
48
|
-
|
|
49
|
-
2. Social Analytics
|
|
50
|
-
3. Link Analytics
|
|
51
|
-
- When the context includes API endpoints (e.g. "POST /analytics/post") or technical operations, proactively include a short code example showing the request (HTTP method, endpoint, JSON body) even if the user didn't explicitly ask for one. Technical users expect to see code. If the context has no code-relevant information, skip the example.
|
|
52
|
-
- When constructing code examples from API endpoints in the context, use the endpoint, HTTP method, and any parameters/body fields mentioned. Always note it's a basic example and link to the full API reference page.
|
|
117
|
+
const dynamicContext = `Documentation context:\n${context}`;
|
|
53
118
|
|
|
54
|
-
|
|
55
|
-
|
|
119
|
+
return [
|
|
120
|
+
{ type: 'text', text: staticRules },
|
|
121
|
+
{ type: 'text', text: dynamicContext },
|
|
122
|
+
];
|
|
56
123
|
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool schemas for the AI documentation chat.
|
|
3
|
+
*
|
|
4
|
+
* Forces Claude into structured output: it must call exactly one of
|
|
5
|
+
* `answer` (for direct answers) or `ask_clarification` (when the query
|
|
6
|
+
* matches 2-3 distinct topics in the retrieved context).
|
|
7
|
+
*
|
|
8
|
+
* Replaces the regex-based `extractCitations` and `extractClarificationOptions`
|
|
9
|
+
* parsers. Citations become the `cited_page_slugs[]` field on the answer tool;
|
|
10
|
+
* clarification options become the `options[]` field on the clarification tool.
|
|
11
|
+
*/
|
|
12
|
+
import type Anthropic from '@anthropic-ai/sdk';
|
|
13
|
+
|
|
14
|
+
// Stricter local shapes so tests + consumers get typed access to nested
|
|
15
|
+
// schema fields. The SDK types `input_schema.properties` as `unknown | null`,
|
|
16
|
+
// which would force `any` casts at every access site. These local types are
|
|
17
|
+
// structurally assignable to `Anthropic.Tool`, so they remain valid SDK input.
|
|
18
|
+
interface AnswerToolSchema {
|
|
19
|
+
name: 'answer';
|
|
20
|
+
description: string;
|
|
21
|
+
input_schema: {
|
|
22
|
+
type: 'object';
|
|
23
|
+
properties: {
|
|
24
|
+
markdown: { type: 'string'; description: string };
|
|
25
|
+
cited_page_slugs: {
|
|
26
|
+
type: 'array';
|
|
27
|
+
items: { type: 'string' };
|
|
28
|
+
description: string;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
required: ['markdown', 'cited_page_slugs'];
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface ClarificationToolSchema {
|
|
36
|
+
name: 'ask_clarification';
|
|
37
|
+
description: string;
|
|
38
|
+
input_schema: {
|
|
39
|
+
type: 'object';
|
|
40
|
+
properties: {
|
|
41
|
+
question: { type: 'string'; description: string };
|
|
42
|
+
options: {
|
|
43
|
+
type: 'array';
|
|
44
|
+
items: { type: 'string' };
|
|
45
|
+
minItems: 2;
|
|
46
|
+
maxItems: 3;
|
|
47
|
+
description: string;
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
required: ['question', 'options'];
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export const ANSWER_TOOL: AnswerToolSchema = {
|
|
55
|
+
name: 'answer',
|
|
56
|
+
description:
|
|
57
|
+
'Provide a direct answer to the user\'s question using the documentation context. ' +
|
|
58
|
+
'Always populate cited_page_slugs with the pageSlug of every source you referenced. ' +
|
|
59
|
+
'If the context does not contain an answer, still call this tool and say "I don\'t have information about that in the documentation." with cited_page_slugs: [].',
|
|
60
|
+
input_schema: {
|
|
61
|
+
type: 'object',
|
|
62
|
+
properties: {
|
|
63
|
+
markdown: {
|
|
64
|
+
type: 'string',
|
|
65
|
+
description:
|
|
66
|
+
'The answer in markdown. Use code blocks with language hints when showing code. ' +
|
|
67
|
+
'Do not embed citation text like "[Page Title]" — citations are listed in cited_page_slugs.',
|
|
68
|
+
},
|
|
69
|
+
cited_page_slugs: {
|
|
70
|
+
type: 'array',
|
|
71
|
+
items: { type: 'string' },
|
|
72
|
+
description:
|
|
73
|
+
'The pageSlug values (e.g. "getting-started", "api/auth") for every documentation source you referenced. ' +
|
|
74
|
+
'Empty array if the answer is "I don\'t have information about that".',
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
required: ['markdown', 'cited_page_slugs'],
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
export const CLARIFICATION_TOOL: ClarificationToolSchema = {
|
|
82
|
+
name: 'ask_clarification',
|
|
83
|
+
description:
|
|
84
|
+
'Ask the user to choose between 2-3 distinct topics when their question is ambiguous. ' +
|
|
85
|
+
'Only use when the documentation context contains multiple unrelated features that could match ' +
|
|
86
|
+
'(e.g. "Post Analytics" vs "Link Analytics" when asked about "analytics"). ' +
|
|
87
|
+
'The question must end with a question mark.',
|
|
88
|
+
input_schema: {
|
|
89
|
+
type: 'object',
|
|
90
|
+
properties: {
|
|
91
|
+
question: {
|
|
92
|
+
type: 'string',
|
|
93
|
+
description: 'A short disambiguation question ending with "?". E.g. "Which type of analytics are you asking about?"',
|
|
94
|
+
},
|
|
95
|
+
// minItems/maxItems are advisory to the model — Anthropic does NOT
|
|
96
|
+
// enforce them server-side. The chat route must defensively validate
|
|
97
|
+
// options.length before rendering a clarification UI.
|
|
98
|
+
options: {
|
|
99
|
+
type: 'array',
|
|
100
|
+
items: { type: 'string' },
|
|
101
|
+
minItems: 2,
|
|
102
|
+
maxItems: 3,
|
|
103
|
+
description: 'The distinct topics the user can choose between. Short labels like "Post Analytics", "Link Analytics".',
|
|
104
|
+
},
|
|
105
|
+
},
|
|
106
|
+
required: ['question', 'options'],
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
// Exported as Anthropic.Tool[] so it passes directly to client.messages.stream({ tools: CHAT_TOOLS }).
|
|
111
|
+
export const CHAT_TOOLS: Anthropic.Tool[] = [ANSWER_TOOL, CLARIFICATION_TOOL];
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bridge to the Crisp chat widget's command queue API.
|
|
3
|
+
*
|
|
4
|
+
* Coordinates visibility between our AI chat panel and Crisp's launcher
|
|
5
|
+
* so they don't visually collide in the bottom-right corner of docs sites.
|
|
6
|
+
*
|
|
7
|
+
* Crisp's docs: https://help.crisp.chat/en/article/how-to-use-crisp-from-javascript-api-1xhvaxt/
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
type CrispAction = 'chat:close' | 'chat:hide' | 'chat:show' | 'chat:open';
|
|
11
|
+
type CrispCommand = ['do', CrispAction];
|
|
12
|
+
|
|
13
|
+
interface CrispQueue {
|
|
14
|
+
push(command: CrispCommand): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function getCrisp(): CrispQueue | null {
|
|
18
|
+
if (typeof window === 'undefined') return null;
|
|
19
|
+
const q = (window as unknown as { $crisp?: unknown }).$crisp;
|
|
20
|
+
if (q && typeof (q as { push?: unknown }).push === 'function') {
|
|
21
|
+
return q as CrispQueue;
|
|
22
|
+
}
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function crispAvailable(): boolean {
|
|
27
|
+
return getCrisp() !== null;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const RETRY_INTERVAL_MS = 500;
|
|
31
|
+
const RETRY_MAX_ATTEMPTS = 30;
|
|
32
|
+
|
|
33
|
+
let pendingHideTimer: ReturnType<typeof setInterval> | null = null;
|
|
34
|
+
|
|
35
|
+
function clearPendingHideRetry(): void {
|
|
36
|
+
if (pendingHideTimer !== null) {
|
|
37
|
+
clearInterval(pendingHideTimer);
|
|
38
|
+
pendingHideTimer = null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function applyHide(crisp: CrispQueue): void {
|
|
43
|
+
// chat:close must come before chat:hide. Crisp can auto-reshow the
|
|
44
|
+
// launcher after a session reset, so closing any open session first
|
|
45
|
+
// reduces the race window.
|
|
46
|
+
crisp.push(['do', 'chat:close']);
|
|
47
|
+
crisp.push(['do', 'chat:hide']);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function hideCrispLauncher(): void {
|
|
51
|
+
if (typeof window === 'undefined') return;
|
|
52
|
+
clearPendingHideRetry();
|
|
53
|
+
const crisp = getCrisp();
|
|
54
|
+
if (crisp) {
|
|
55
|
+
applyHide(crisp);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
let attempts = 0;
|
|
59
|
+
pendingHideTimer = setInterval(() => {
|
|
60
|
+
attempts += 1;
|
|
61
|
+
const q = getCrisp();
|
|
62
|
+
if (q) {
|
|
63
|
+
applyHide(q);
|
|
64
|
+
clearPendingHideRetry();
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
if (attempts >= RETRY_MAX_ATTEMPTS) {
|
|
68
|
+
clearPendingHideRetry();
|
|
69
|
+
}
|
|
70
|
+
}, RETRY_INTERVAL_MS);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function showCrispLauncher(): void {
|
|
74
|
+
clearPendingHideRetry();
|
|
75
|
+
const crisp = getCrisp();
|
|
76
|
+
if (!crisp) return;
|
|
77
|
+
crisp.push(['do', 'chat:show']);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function openCrispChat(): void {
|
|
81
|
+
clearPendingHideRetry();
|
|
82
|
+
const crisp = getCrisp();
|
|
83
|
+
if (!crisp) return;
|
|
84
|
+
crisp.push(['do', 'chat:show']);
|
|
85
|
+
crisp.push(['do', 'chat:open']);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Test-only: clears module-level retry state between cases.
|
|
89
|
+
export function _resetForTests(): void {
|
|
90
|
+
clearPendingHideRetry();
|
|
91
|
+
}
|
|
@@ -14,8 +14,15 @@ export interface EmbeddingChunk {
|
|
|
14
14
|
pageSlug: string;
|
|
15
15
|
/** Heading of the section this chunk belongs to */
|
|
16
16
|
sectionHeading: string;
|
|
17
|
-
/** Plain
|
|
17
|
+
/** Plain-text body, stripped of markdown and JSX. Used for display (LLM context, search snippets). */
|
|
18
18
|
content: string;
|
|
19
|
+
/**
|
|
20
|
+
* `<pageTitle> > <sectionHeading>\n` breadcrumb (plus `API Reference — METHOD /path\n`
|
|
21
|
+
* for API pages) that gets prepended to `content` when embedding — so BM25 finds
|
|
22
|
+
* chunks by page-title terms even when the body never uses them. Kept as a
|
|
23
|
+
* separate field so snippet consumers don't have to strip it.
|
|
24
|
+
*/
|
|
25
|
+
prefix: string;
|
|
19
26
|
/** Page title from frontmatter, or slug-derived fallback */
|
|
20
27
|
pageTitle: string;
|
|
21
28
|
}
|
|
@@ -111,18 +118,82 @@ function titleFromSlug(slug: string): string {
|
|
|
111
118
|
}
|
|
112
119
|
|
|
113
120
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
121
|
+
* Build the per-chunk prefix that gets prepended to the cleaned content
|
|
122
|
+
* before embedding/upsert. Two purposes:
|
|
123
|
+
*
|
|
124
|
+
* 1. Title-breadcrumb: `<pageTitle> > <sectionHeading>` gives every chunk
|
|
125
|
+
* a literal occurrence of its page title in the indexed text — so BM25
|
|
126
|
+
* finds e.g. the `Changelog > April 2026` chunk for a "changelog" query
|
|
127
|
+
* even when the month's content (Password Protection, YouTube Shorts)
|
|
128
|
+
* never mentions the word "changelog" itself. Without this, pages that
|
|
129
|
+
* happen to discuss a concept frequently (like `components/update`
|
|
130
|
+
* documenting the `<Update>` MDX tag) outrank the actual answer chunks.
|
|
131
|
+
*
|
|
132
|
+
* 2. API-method tag: API-reference pages additionally get an
|
|
133
|
+
* `API Reference — POST /endpoint` line so HTTP-method-specific queries
|
|
134
|
+
* ("how do I POST to /analytics") cluster to the right endpoint page.
|
|
135
|
+
*
|
|
136
|
+
* Both lines end in `\n` so they stay visually separated from the content
|
|
137
|
+
* body when the chat model reads the full context block.
|
|
117
138
|
*/
|
|
118
139
|
function getEmbeddingPrefix(
|
|
140
|
+
pageTitle: string,
|
|
141
|
+
sectionHeading: string,
|
|
119
142
|
slug: string,
|
|
120
143
|
frontmatter: Record<string, unknown>,
|
|
121
144
|
): string {
|
|
122
145
|
const apiMethod = (frontmatter.api as string) || (frontmatter.openapi as string);
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
146
|
+
const apiLabel = apiMethod
|
|
147
|
+
? `API Reference — ${apiMethod}\n`
|
|
148
|
+
: slug.startsWith('apis/') ? 'API Reference\n' : '';
|
|
149
|
+
const titleLabel = sectionHeading ? `${pageTitle} > ${sectionHeading}` : pageTitle;
|
|
150
|
+
return `${apiLabel}${titleLabel}\n`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Without this, changelog-style pages that use `<Update>` wrappers (a
|
|
155
|
+
* Mintlify convention) collapse into a single heading-less blob — features
|
|
156
|
+
* in April get merged with features in March, and retrieval can't target
|
|
157
|
+
* a specific release. Promoting each label to a synthetic `### <label>`
|
|
158
|
+
* heading lets the existing heading splitter separate them.
|
|
159
|
+
*
|
|
160
|
+
* Matches `label` regardless of attribute order, handles self-closing tags,
|
|
161
|
+
* and leaves Update tags without a `label` attribute alone for
|
|
162
|
+
* `stripForEmbedding` to handle.
|
|
163
|
+
*
|
|
164
|
+
* Fenced code blocks are masked so Update tags inside ``` ... ``` examples
|
|
165
|
+
* (e.g. `components/update.mdx` pages that document the component itself)
|
|
166
|
+
* don't produce spurious chunks. `[^><]*?` (not `[^>]*?`) prevents a
|
|
167
|
+
* malformed opener (missing `>`) from greedily matching through to the
|
|
168
|
+
* next `</Update>` and silently eating body text.
|
|
169
|
+
*/
|
|
170
|
+
function preprocessUpdateBlocks(content: string): string {
|
|
171
|
+
const preserved: string[] = [];
|
|
172
|
+
const masked = content.replace(/```[\s\S]*?```/g, (m) => {
|
|
173
|
+
preserved.push(m);
|
|
174
|
+
return `\x00${preserved.length - 1}\x00`;
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
const transformed = masked
|
|
178
|
+
.replace(
|
|
179
|
+
/<Update\b[^><]*?\blabel="([^"]+)"[^><]*?\/?>/g,
|
|
180
|
+
(_, label: string) => {
|
|
181
|
+
const clean = sanitizeHeadingText(label);
|
|
182
|
+
return clean ? `\n### ${clean}\n` : '\n';
|
|
183
|
+
},
|
|
184
|
+
)
|
|
185
|
+
.replace(/<\/Update>/g, '\n');
|
|
186
|
+
|
|
187
|
+
return transformed.replace(/\x00(\d+)\x00/g, (_, i) => preserved[parseInt(i)]);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Collapse whitespace and drop leading `#` runs so a synthetic heading
|
|
192
|
+
* derived from a label (or a page title flowing into the breadcrumb) can't
|
|
193
|
+
* inject extra markdown heading depth or span multiple lines.
|
|
194
|
+
*/
|
|
195
|
+
function sanitizeHeadingText(raw: string): string {
|
|
196
|
+
return raw.replace(/^#+\s*/, '').replace(/\s+/g, ' ').trim();
|
|
126
197
|
}
|
|
127
198
|
|
|
128
199
|
/**
|
|
@@ -137,11 +208,11 @@ export function chunkPageForEmbedding(
|
|
|
137
208
|
maxChars = 2000,
|
|
138
209
|
): EmbeddingChunk[] {
|
|
139
210
|
const slug = page.path.replace(/\.mdx?$/, '').replace(/\\/g, '/');
|
|
140
|
-
const
|
|
141
|
-
const
|
|
211
|
+
const rawTitle = (page.frontmatter.title as string) || titleFromSlug(slug);
|
|
212
|
+
const pageTitle = sanitizeHeadingText(rawTitle) || titleFromSlug(slug);
|
|
142
213
|
|
|
143
214
|
// Normalize Windows line endings before extracting sections
|
|
144
|
-
const normalizedContent = page.content.replace(/\r\n/g, '\n');
|
|
215
|
+
const normalizedContent = preprocessUpdateBlocks(page.content.replace(/\r\n/g, '\n'));
|
|
145
216
|
|
|
146
217
|
const sections = extractSections(normalizedContent);
|
|
147
218
|
const chunks: EmbeddingChunk[] = [];
|
|
@@ -156,13 +227,16 @@ export function chunkPageForEmbedding(
|
|
|
156
227
|
? [cleanContent]
|
|
157
228
|
: splitAtSentenceBoundaries(cleanContent, maxChars);
|
|
158
229
|
|
|
230
|
+
const prefix = getEmbeddingPrefix(pageTitle, section.heading, slug, page.frontmatter);
|
|
231
|
+
|
|
159
232
|
for (const piece of pieces) {
|
|
160
233
|
if (!piece.trim()) continue;
|
|
161
234
|
chunks.push({
|
|
162
235
|
id: `${slug}#${chunkIndex}`,
|
|
163
236
|
pageSlug: slug,
|
|
164
237
|
sectionHeading: section.heading,
|
|
165
|
-
content:
|
|
238
|
+
content: piece,
|
|
239
|
+
prefix,
|
|
166
240
|
pageTitle,
|
|
167
241
|
});
|
|
168
242
|
chunkIndex++;
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight query rewriter for AI chat retrieval.
|
|
3
|
+
*
|
|
4
|
+
* Translates vague/natural-language user queries into doc-search vocabulary
|
|
5
|
+
* using Claude Haiku. Runs in parallel with the original query in the chat
|
|
6
|
+
* route, so latency is masked — if the rewrite is slow or fails, the caller
|
|
7
|
+
* falls back to the original query without penalty.
|
|
8
|
+
*
|
|
9
|
+
* Design notes:
|
|
10
|
+
* - Returns null on ANY failure path (no client, API error, empty response).
|
|
11
|
+
* Callers must handle null by using the original query.
|
|
12
|
+
* - max_tokens is capped at 80 — rewrites are 3-10 words typically.
|
|
13
|
+
* - Conversation history is included so follow-ups like "what about the other
|
|
14
|
+
* one" can be disambiguated against the prior user message.
|
|
15
|
+
*/
|
|
16
|
+
import { getAnthropicClient } from '@/lib/anthropic-client';
|
|
17
|
+
|
|
18
|
+
const REWRITE_MODEL = 'claude-haiku-4-5-20251001';
|
|
19
|
+
const MAX_REWRITE_CHARS = 200;
|
|
20
|
+
|
|
21
|
+
export const SYSTEM_PROMPT = `You rewrite a user's chat question into a short documentation search query.
|
|
22
|
+
|
|
23
|
+
Rules:
|
|
24
|
+
- Output ONLY the rewritten query — no quotes, no explanation, no prefix.
|
|
25
|
+
- Use terminology that would appear in technical documentation.
|
|
26
|
+
- Keep it short (3-10 words is ideal).
|
|
27
|
+
- If the user's question already uses technical vocabulary, output it unchanged.
|
|
28
|
+
- If conversation history is provided, use it to disambiguate references like "the other one" or "that".
|
|
29
|
+
|
|
30
|
+
Examples:
|
|
31
|
+
Q: how do I make my docs live → deploy documentation site
|
|
32
|
+
Q: change colors → theme customization
|
|
33
|
+
Q: can it do auth → authentication setup
|
|
34
|
+
Q: what about the other one (after discussing analytics) → link analytics
|
|
35
|
+
Q: what is the most recent feature → changelog latest updates new features
|
|
36
|
+
Q: what's new → changelog new features updates
|
|
37
|
+
Q: latest release → changelog release notes updates
|
|
38
|
+
Q: any updates recently → changelog recent updates new features`;
|
|
39
|
+
|
|
40
|
+
export interface HistoryMessage {
|
|
41
|
+
role: 'user' | 'assistant';
|
|
42
|
+
content: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Rewrite a user query into doc-search vocabulary.
|
|
47
|
+
* Returns null on any failure — caller should fall back to the original query.
|
|
48
|
+
*/
|
|
49
|
+
export async function rewriteQueryForSearch(
|
|
50
|
+
message: string,
|
|
51
|
+
history: HistoryMessage[],
|
|
52
|
+
): Promise<string | null> {
|
|
53
|
+
const anthropic = getAnthropicClient();
|
|
54
|
+
if (!anthropic) return null;
|
|
55
|
+
|
|
56
|
+
// Compose the user prompt: include the last user message from history (if any)
|
|
57
|
+
// so follow-up references resolve. Ignore assistant replies — they'd contaminate
|
|
58
|
+
// the rewrite with doc-flavored phrasing that might mask the user's real intent.
|
|
59
|
+
const priorUserMsg = [...history].reverse().find(
|
|
60
|
+
h => h.role === 'user' && h.content !== message,
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
const userPrompt = priorUserMsg
|
|
64
|
+
? `Previous question: ${priorUserMsg.content}\nCurrent question: ${message}\n\nRewrite the current question:`
|
|
65
|
+
: `Question: ${message}\n\nRewrite:`;
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
const response = await anthropic.messages.create({
|
|
69
|
+
model: REWRITE_MODEL,
|
|
70
|
+
max_tokens: 80,
|
|
71
|
+
temperature: 0.1,
|
|
72
|
+
system: SYSTEM_PROMPT,
|
|
73
|
+
messages: [{ role: 'user', content: userPrompt }],
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const textBlock = response.content.find(c => c.type === 'text');
|
|
77
|
+
if (!textBlock || textBlock.type !== 'text') return null;
|
|
78
|
+
|
|
79
|
+
let rewrite = textBlock.text.trim();
|
|
80
|
+
|
|
81
|
+
// Strip surrounding quotes if Claude wrapped the output
|
|
82
|
+
rewrite = rewrite.replace(/^["'`]+|["'`]+$/g, '').trim();
|
|
83
|
+
|
|
84
|
+
if (!rewrite) return null;
|
|
85
|
+
|
|
86
|
+
// Defensive cap — prevents a runaway response from bloating the search query
|
|
87
|
+
return rewrite.slice(0, MAX_REWRITE_CHARS);
|
|
88
|
+
} catch {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -23,8 +23,12 @@ export function evictOldest<K, V>(cache: Map<K, V>, maxSize: number): void {
|
|
|
23
23
|
export function clearConfigCache(_projectSlug?: string): void {}
|
|
24
24
|
export function getConfigCacheSize(): number { return 0; }
|
|
25
25
|
|
|
26
|
+
let fetchDocsConfigWarned = false;
|
|
26
27
|
export async function fetchDocsConfig(_projectSlug: string): Promise<DocsConfig | null> {
|
|
27
|
-
|
|
28
|
+
if (!fetchDocsConfigWarned) {
|
|
29
|
+
fetchDocsConfigWarned = true;
|
|
30
|
+
console.warn('[r2-content stub] fetchDocsConfig called in dev mode (suppressing further warnings)');
|
|
31
|
+
}
|
|
28
32
|
return null;
|
|
29
33
|
}
|
|
30
34
|
|
|
@@ -44,8 +44,8 @@ const HYBRID_QUERY_OPTS = {
|
|
|
44
44
|
*/
|
|
45
45
|
const MIN_SCORE = 0.3;
|
|
46
46
|
|
|
47
|
-
/** Max chunks per page —
|
|
48
|
-
const MAX_CHUNKS_PER_PAGE =
|
|
47
|
+
/** Max chunks per page — raised from 3 to 4 to match broader topK retrieval budget */
|
|
48
|
+
const MAX_CHUNKS_PER_PAGE = 4;
|
|
49
49
|
|
|
50
50
|
/** Create a namespaced Upstash Vector index for a project. */
|
|
51
51
|
function getNamespace(projectId: string) {
|
|
@@ -80,7 +80,9 @@ export async function upsertChunks(
|
|
|
80
80
|
await ns.upsert(
|
|
81
81
|
batch.map(c => ({
|
|
82
82
|
id: c.id,
|
|
83
|
-
|
|
83
|
+
// Prefix + body goes to Upstash for embedding/BM25; metadata.content
|
|
84
|
+
// stays prefix-free so consumers display clean body text.
|
|
85
|
+
data: c.prefix + c.content,
|
|
84
86
|
metadata: {
|
|
85
87
|
pageSlug: c.pageSlug,
|
|
86
88
|
sectionHeading: c.sectionHeading,
|