gitlab-docs-mcp 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/data/gitlab-docs/documents.json +2753 -2505
- package/data/gitlab-docs/index_meta.json +2 -2
- package/dist/content/chunker.d.ts +54 -0
- package/dist/content/chunker.d.ts.map +1 -0
- package/dist/content/chunker.js +163 -0
- package/dist/content/chunker.js.map +1 -0
- package/dist/content/chunker.test.d.ts +2 -0
- package/dist/content/chunker.test.d.ts.map +1 -0
- package/dist/content/chunker.test.js +92 -0
- package/dist/content/chunker.test.js.map +1 -0
- package/dist/content/index.d.ts +1 -0
- package/dist/content/index.d.ts.map +1 -1
- package/dist/content/index.js +1 -0
- package/dist/content/index.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document chunking utilities for splitting large documents into manageable pieces
|
|
3
|
+
* Specifically handles the massive GraphQL reference documentation
|
|
4
|
+
*/
|
|
5
|
+
interface ChunkConfig {
|
|
6
|
+
/**
|
|
7
|
+
* Maximum size in characters before chunking is applied
|
|
8
|
+
*/
|
|
9
|
+
maxChunkSize: number;
|
|
10
|
+
/**
|
|
11
|
+
* Section markers that indicate logical split points
|
|
12
|
+
*/
|
|
13
|
+
sectionMarkers: string[];
|
|
14
|
+
/**
|
|
15
|
+
* Overlap between chunks to preserve context
|
|
16
|
+
*/
|
|
17
|
+
overlapSize: number;
|
|
18
|
+
}
|
|
19
|
+
interface DocumentChunk {
|
|
20
|
+
/**
|
|
21
|
+
* Original document path with chunk suffix
|
|
22
|
+
*/
|
|
23
|
+
path: string;
|
|
24
|
+
/**
|
|
25
|
+
* Title of the chunk
|
|
26
|
+
*/
|
|
27
|
+
title: string;
|
|
28
|
+
/**
|
|
29
|
+
* Content of the chunk
|
|
30
|
+
*/
|
|
31
|
+
content: string;
|
|
32
|
+
/**
|
|
33
|
+
* Which chunk this is (1-indexed)
|
|
34
|
+
*/
|
|
35
|
+
chunkIndex: number;
|
|
36
|
+
/**
|
|
37
|
+
* Total number of chunks for this document
|
|
38
|
+
*/
|
|
39
|
+
totalChunks: number;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Checks if a document should be chunked based on size and path
|
|
43
|
+
*/
|
|
44
|
+
export declare function shouldChunkDocument(path: string, content: string): boolean;
|
|
45
|
+
/**
|
|
46
|
+
* Splits a large document into logical chunks based on section markers
|
|
47
|
+
*/
|
|
48
|
+
export declare function chunkDocument(path: string, title: string, content: string, config?: ChunkConfig): DocumentChunk[];
|
|
49
|
+
/**
|
|
50
|
+
* Creates a summary chunk with links to all other chunks
|
|
51
|
+
*/
|
|
52
|
+
export declare function createSummaryChunk(originalPath: string, originalTitle: string, chunks: DocumentChunk[]): string;
|
|
53
|
+
export {};
|
|
54
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/content/chunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,UAAU,WAAW;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,cAAc,EAAE,MAAM,EAAE,CAAC;IAEzB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,aAAa;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAqBD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAM1E;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,WAAkC,GACzC,aAAa,EAAE,CAkEjB;AA+DD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,aAAa,EAAE,GACtB,MAAM,CA8BR"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document chunking utilities for splitting large documents into manageable pieces
|
|
3
|
+
* Specifically handles the massive GraphQL reference documentation
|
|
4
|
+
*/
|
|
5
|
+
import { logger } from '../utils/logger.js';
|
|
6
|
+
/**
|
|
7
|
+
* Default configuration for GraphQL reference chunking
|
|
8
|
+
* Note: These markers are based on GitLab's auto-generated GraphQL documentation structure.
|
|
9
|
+
* If markers change, the system falls back to size-based chunking automatically.
|
|
10
|
+
*/
|
|
11
|
+
const GRAPHQL_CHUNK_CONFIG = {
|
|
12
|
+
maxChunkSize: 500000, // 500KB per chunk
|
|
13
|
+
sectionMarkers: [
|
|
14
|
+
'Query type',
|
|
15
|
+
'Mutation type',
|
|
16
|
+
'Object types',
|
|
17
|
+
'Enumeration types',
|
|
18
|
+
'Scalar types',
|
|
19
|
+
'Abstract types',
|
|
20
|
+
'Input types',
|
|
21
|
+
],
|
|
22
|
+
overlapSize: 1000, // 1KB overlap to preserve context
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Checks if a document should be chunked based on size and path
|
|
26
|
+
*/
|
|
27
|
+
export function shouldChunkDocument(path, content) {
|
|
28
|
+
// Only chunk the massive GraphQL reference file
|
|
29
|
+
if (path === 'api/graphql/reference/_index.md' && content.length > 500000) {
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Splits a large document into logical chunks based on section markers
|
|
36
|
+
*/
|
|
37
|
+
export function chunkDocument(path, title, content, config = GRAPHQL_CHUNK_CONFIG) {
|
|
38
|
+
const chunks = [];
|
|
39
|
+
// Find all section markers
|
|
40
|
+
const sections = [];
|
|
41
|
+
for (const marker of config.sectionMarkers) {
|
|
42
|
+
const regex = new RegExp(`^${marker}\\s*$`, 'gm');
|
|
43
|
+
let match;
|
|
44
|
+
while ((match = regex.exec(content)) !== null) {
|
|
45
|
+
sections.push({
|
|
46
|
+
marker,
|
|
47
|
+
position: match.index,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// Sort sections by position
|
|
52
|
+
sections.sort((a, b) => a.position - b.position);
|
|
53
|
+
if (sections.length === 0) {
|
|
54
|
+
// No sections found, fall back to size-based chunking
|
|
55
|
+
logger.warn(`No section markers found in ${path}, using size-based chunking`);
|
|
56
|
+
return chunkBySize(path, title, content, config);
|
|
57
|
+
}
|
|
58
|
+
logger.info(`Found ${sections.length} sections in ${path}, creating semantic chunks`);
|
|
59
|
+
// Create chunks based on sections
|
|
60
|
+
for (let i = 0; i < sections.length; i++) {
|
|
61
|
+
const section = sections[i];
|
|
62
|
+
const nextSection = sections[i + 1];
|
|
63
|
+
const start = section.position;
|
|
64
|
+
const end = nextSection ? nextSection.position : content.length;
|
|
65
|
+
const chunkContent = content.slice(start, end);
|
|
66
|
+
// If chunk is still too large, split it further
|
|
67
|
+
if (chunkContent.length > config.maxChunkSize * 1.5) {
|
|
68
|
+
const subChunks = chunkBySize(`${path}#${sanitizeMarker(section.marker)}`, `${title} - ${section.marker}`, chunkContent, config);
|
|
69
|
+
chunks.push(...subChunks);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
chunks.push({
|
|
73
|
+
path: `${path}#${sanitizeMarker(section.marker)}`,
|
|
74
|
+
title: `${title} - ${section.marker}`,
|
|
75
|
+
content: chunkContent,
|
|
76
|
+
chunkIndex: chunks.length + 1,
|
|
77
|
+
totalChunks: 0, // Will be set later
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Update total chunks count
|
|
82
|
+
const totalChunks = chunks.length;
|
|
83
|
+
chunks.forEach((chunk) => {
|
|
84
|
+
chunk.totalChunks = totalChunks;
|
|
85
|
+
});
|
|
86
|
+
return chunks;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Splits content by size when no logical sections are found
|
|
90
|
+
*/
|
|
91
|
+
function chunkBySize(path, title, content, config) {
|
|
92
|
+
const chunks = [];
|
|
93
|
+
let position = 0;
|
|
94
|
+
let chunkIndex = 1;
|
|
95
|
+
while (position < content.length) {
|
|
96
|
+
const end = Math.min(position + config.maxChunkSize, content.length);
|
|
97
|
+
let chunkEnd = end;
|
|
98
|
+
// Try to break at a paragraph boundary
|
|
99
|
+
if (end < content.length) {
|
|
100
|
+
const nextNewlines = content.indexOf('\n\n', end - 100);
|
|
101
|
+
if (nextNewlines !== -1 && nextNewlines < end + 100) {
|
|
102
|
+
chunkEnd = nextNewlines;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const chunkContent = content.slice(Math.max(0, position - config.overlapSize), chunkEnd);
|
|
106
|
+
chunks.push({
|
|
107
|
+
path: `${path}#chunk-${chunkIndex}`,
|
|
108
|
+
title: `${title} (Part ${chunkIndex})`,
|
|
109
|
+
content: chunkContent,
|
|
110
|
+
chunkIndex,
|
|
111
|
+
totalChunks: 0, // Will be set later
|
|
112
|
+
});
|
|
113
|
+
position = chunkEnd;
|
|
114
|
+
chunkIndex++;
|
|
115
|
+
}
|
|
116
|
+
// Update total chunks count
|
|
117
|
+
const totalChunks = chunks.length;
|
|
118
|
+
chunks.forEach((chunk) => {
|
|
119
|
+
chunk.totalChunks = totalChunks;
|
|
120
|
+
});
|
|
121
|
+
return chunks;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Sanitizes a section marker for use in a path fragment
|
|
125
|
+
*/
|
|
126
|
+
function sanitizeMarker(marker) {
|
|
127
|
+
return marker
|
|
128
|
+
.toLowerCase()
|
|
129
|
+
.replace(/\s+/g, '-')
|
|
130
|
+
.replace(/[^a-z0-9-]/g, '');
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Creates a summary chunk with links to all other chunks
|
|
134
|
+
*/
|
|
135
|
+
export function createSummaryChunk(originalPath, originalTitle, chunks) {
|
|
136
|
+
const summaryLines = [
|
|
137
|
+
`# ${originalTitle}`,
|
|
138
|
+
'',
|
|
139
|
+
'> **Note:** This document has been split into multiple sections for better performance.',
|
|
140
|
+
'',
|
|
141
|
+
'## Available Sections',
|
|
142
|
+
'',
|
|
143
|
+
];
|
|
144
|
+
for (const chunk of chunks) {
|
|
145
|
+
summaryLines.push(`- [${chunk.title}](${chunk.path})`);
|
|
146
|
+
}
|
|
147
|
+
summaryLines.push('');
|
|
148
|
+
summaryLines.push('## About This Reference');
|
|
149
|
+
summaryLines.push('');
|
|
150
|
+
summaryLines.push('This is the auto-generated GraphQL API reference for GitLab. Each section contains detailed information about:');
|
|
151
|
+
summaryLines.push('');
|
|
152
|
+
summaryLines.push('- **Query types**: Top-level entry points for read operations');
|
|
153
|
+
summaryLines.push('- **Mutation types**: Entry points for write operations');
|
|
154
|
+
summaryLines.push('- **Object types**: Resource representations in the API');
|
|
155
|
+
summaryLines.push('- **Enumeration types**: Predefined value sets');
|
|
156
|
+
summaryLines.push('- **Scalar types**: Basic data types');
|
|
157
|
+
summaryLines.push('- **Abstract types**: Unions and interfaces');
|
|
158
|
+
summaryLines.push('- **Input types**: Arguments for mutations and queries');
|
|
159
|
+
summaryLines.push('');
|
|
160
|
+
summaryLines.push('Use the interactive GraphQL explorer to test queries, or generate a machine-readable schema in IDL or JSON formats.');
|
|
161
|
+
return summaryLines.join('\n');
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/content/chunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AA8C5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAgB;IACxC,YAAY,EAAE,MAAM,EAAE,kBAAkB;IACxC,cAAc,EAAE;QACd,YAAY;QACZ,eAAe;QACf,cAAc;QACd,mBAAmB;QACnB,cAAc;QACd,gBAAgB;QAChB,aAAa;KACd;IACD,WAAW,EAAE,IAAI,EAAE,kCAAkC;CACtD,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,OAAe;IAC/D,gDAAgD;IAChD,IAAI,IAAI,KAAK,iCAAiC,IAAI,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QAC1E,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,KAAa,EACb,OAAe,EACf,SAAsB,oBAAoB;IAE1C,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,2BAA2B;IAC3B,MAAM,QAAQ,GAA2C,EAAE,CAAC;IAE5D,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,MAAM,OAAO,EAAE,IAAI,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC;QAEV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9C,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,QAAQ,EAAE,KAAK,CAAC,KAAK;aACtB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEjD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,sDAAsD;QACtD,MAAM,CAAC,IAAI,CAAC,+BAA+B,IAAI,6BAA6B,CAAC,CAAC;QAC9E,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,MAAM,gBAAgB,IAAI,4BAA4B,CAAC,CAAC;IAEtF,kCAAkC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEpC,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC/B,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAEhE,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAE/C,gDAAgD;QAChD,IAAI,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC;YACpD,MAAM,SAAS,GAAG,WAAW,CAC3B,GAAG,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC3C,GAAG,KAAK,MAAM,OAAO,CAAC,MAAM,EAAE,EAC9B,YAAY,EACZ,MAAM,CACP,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBACjD,KAAK,EAAE,GAAG,KAAK,MAAM,OAAO,CAAC,MAAM,EAAE;gBACrC,OAAO,EAAE,YAAY;gBACrB,UAAU,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;gBAC7B,WAAW,EAAE,CAAC,EAAE,oBAAoB;aACrC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACvB,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAClB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAmB;IAEnB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,QAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,QAAQ,GAAG,GAAG,CAAC;QAEnB,uCAAuC;QACvC,IAAI,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACxD,IAAI,YAAY,KAAK,CAAC,CAAC,IAAI,YAAY,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;gBACpD,QAAQ,GAAG,YAAY,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAChC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,EAC1C,QAAQ,CACT,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,GAAG,IAAI,UAAU,UAAU,EAAE;YACnC,KAAK,EAAE,GAAG,KAAK,UAAU,UAAU,GAAG;YACtC,OAAO,EAAE,YAAY;YACrB,UAAU;YACV,WAAW,EAAE,CAAC,EAAE,oBAAoB;SACrC,CAAC,CAAC;QAEH,QAAQ,GAAG,QAAQ,CAAC;QACpB,UAAU,EAAE,CAAC;IACf,CAAC;IAED,4BAA4B;IAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACvB,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAAc;IACpC,OAAO,MAAM;SACV,WAAW,EAAE;SACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,YAAoB,EACpB,aAAqB,EACrB,MAAuB;IAEvB,MAAM,YAAY,GAAG;QACnB,KAAK,aAAa,EAAE;QACpB,EAAE;QACF,yFAAyF;QACzF,EAAE;QACF,uBAAuB;QACvB,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,YAAY,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,YAAY,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IAC7C,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,YAAY,CAAC,IAAI,CAAC,gHAAgH,CAAC,CAAC;IACpI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,YAAY,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IACnF,YAAY,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;IAC7E,YAAY,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;IAC7E,YAAY,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IACpE,YAAY,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IAC1D,YAAY,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IACjE,YAAY,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;IAC5E,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,YAAY,CAAC,IAAI,CAAC,qHAAqH,CAAC,CAAC;IAEzI,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.test.d.ts","sourceRoot":"","sources":["../../src/content/chunker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { shouldChunkDocument, chunkDocument, createSummaryChunk } from './chunker.js';
|
|
3
|
+
describe('Document Chunker', () => {
|
|
4
|
+
describe('shouldChunkDocument', () => {
|
|
5
|
+
it('should return true for large GraphQL reference', () => {
|
|
6
|
+
const largePath = 'api/graphql/reference/_index.md';
|
|
7
|
+
const largeContent = 'x'.repeat(600000); // 600KB
|
|
8
|
+
expect(shouldChunkDocument(largePath, largeContent)).toBe(true);
|
|
9
|
+
});
|
|
10
|
+
it('should return false for small GraphQL reference', () => {
|
|
11
|
+
const path = 'api/graphql/reference/_index.md';
|
|
12
|
+
const smallContent = 'x'.repeat(100000); // 100KB
|
|
13
|
+
expect(shouldChunkDocument(path, smallContent)).toBe(false);
|
|
14
|
+
});
|
|
15
|
+
it('should return false for other large documents', () => {
|
|
16
|
+
const path = 'some/other/doc.md';
|
|
17
|
+
const largeContent = 'x'.repeat(600000);
|
|
18
|
+
expect(shouldChunkDocument(path, largeContent)).toBe(false);
|
|
19
|
+
});
|
|
20
|
+
});
|
|
21
|
+
describe('chunkDocument', () => {
|
|
22
|
+
it('should split document by section markers', () => {
|
|
23
|
+
const content = `Query type
|
|
24
|
+
Some query content here.
|
|
25
|
+
This is about queries.
|
|
26
|
+
|
|
27
|
+
Mutation type
|
|
28
|
+
Some mutation content here.
|
|
29
|
+
This is about mutations.
|
|
30
|
+
|
|
31
|
+
Object types
|
|
32
|
+
Object types represent resources.
|
|
33
|
+
More object type info.`;
|
|
34
|
+
const chunks = chunkDocument('api/graphql/reference/_index.md', 'GraphQL Reference', content);
|
|
35
|
+
expect(chunks.length).toBe(3);
|
|
36
|
+
expect(chunks[0].title).toBe('GraphQL Reference - Query type');
|
|
37
|
+
expect(chunks[0].path).toBe('api/graphql/reference/_index.md#query-type');
|
|
38
|
+
expect(chunks[1].title).toBe('GraphQL Reference - Mutation type');
|
|
39
|
+
expect(chunks[2].title).toBe('GraphQL Reference - Object types');
|
|
40
|
+
// Check that each chunk has correct metadata
|
|
41
|
+
chunks.forEach((chunk) => {
|
|
42
|
+
expect(chunk.totalChunks).toBe(3);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
it('should handle content without section markers', () => {
|
|
46
|
+
const content = 'x'.repeat(600000);
|
|
47
|
+
const chunks = chunkDocument('some/doc.md', 'Some Document', content);
|
|
48
|
+
expect(chunks.length).toBeGreaterThan(1);
|
|
49
|
+
expect(chunks[0].title).toContain('(Part 1)');
|
|
50
|
+
expect(chunks[1].title).toContain('(Part 2)');
|
|
51
|
+
});
|
|
52
|
+
it('should include chunk metadata', () => {
|
|
53
|
+
const content = `Query type
|
|
54
|
+
Content here.
|
|
55
|
+
|
|
56
|
+
Mutation type
|
|
57
|
+
More content.`;
|
|
58
|
+
const chunks = chunkDocument('api/graphql/reference/_index.md', 'GraphQL Reference', content);
|
|
59
|
+
expect(chunks[0].chunkIndex).toBe(1);
|
|
60
|
+
expect(chunks[1].chunkIndex).toBe(2);
|
|
61
|
+
expect(chunks[0].totalChunks).toBe(2);
|
|
62
|
+
expect(chunks[1].totalChunks).toBe(2);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
describe('createSummaryChunk', () => {
|
|
66
|
+
it('should create a summary with links to all chunks', () => {
|
|
67
|
+
const chunks = [
|
|
68
|
+
{
|
|
69
|
+
path: 'api/graphql/reference/_index.md#query-type',
|
|
70
|
+
title: 'GraphQL Reference - Query type',
|
|
71
|
+
content: 'content',
|
|
72
|
+
chunkIndex: 1,
|
|
73
|
+
totalChunks: 2,
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
path: 'api/graphql/reference/_index.md#mutation-type',
|
|
77
|
+
title: 'GraphQL Reference - Mutation type',
|
|
78
|
+
content: 'content',
|
|
79
|
+
chunkIndex: 2,
|
|
80
|
+
totalChunks: 2,
|
|
81
|
+
},
|
|
82
|
+
];
|
|
83
|
+
const summary = createSummaryChunk('api/graphql/reference/_index.md', 'GraphQL Reference', chunks);
|
|
84
|
+
expect(summary).toContain('# GraphQL Reference');
|
|
85
|
+
expect(summary).toContain('split into multiple sections');
|
|
86
|
+
expect(summary).toContain('[GraphQL Reference - Query type](api/graphql/reference/_index.md#query-type)');
|
|
87
|
+
expect(summary).toContain('[GraphQL Reference - Mutation type](api/graphql/reference/_index.md#mutation-type)');
|
|
88
|
+
expect(summary).toContain('About This Reference');
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
//# sourceMappingURL=chunker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../src/content/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEtF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,SAAS,GAAG,iCAAiC,CAAC;YACpD,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;YAEjD,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,IAAI,GAAG,iCAAiC,CAAC;YAC/C,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;YAEjD,MAAM,CAAC,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,IAAI,GAAG,mBAAmB,CAAC;YACjC,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAExC,MAAM,CAAC,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,OAAO,GAAG;;;;;;;;;;uBAUC,CAAC;YAElB,MAAM,MAAM,GAAG,aAAa,CAC1B,iCAAiC,EACjC,mBAAmB,EACnB,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;YAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAClE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAEjE,6CAA6C;YAC7C,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACvB,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnC,MAAM,MAAM,GAAG,aAAa,CAC1B,aAAa,EACb,eAAe,EACf,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,OAAO,GAAG;;;;cAIR,CAAC;YAET,MAAM,MAAM,GAAG,aAAa,CAC1B,iCAAiC,EACjC,mBAAmB,EACnB,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,MAAM,GAAG;gBACb;oBACE,IAAI,EAAE,4CAA4C;oBAClD,KAAK,EAAE,gCAAgC;oBACvC,OAAO,EAAE,SAAS;oBAClB,UAAU,EAAE,CAAC;oBACb,WAAW,EAAE,CAAC;iBACf;gBACD;oBACE,IAAI,EAAE,+CAA+C;oBACrD,KAAK,EAAE,mCAAmC;oBAC1C,OAAO,EAAE,SAAS;oBAClB,UAAU,EAAE,CAAC;oBACb,WAAW,EAAE,CAAC;iBACf;aACF,CAAC;YAEF,MAAM,OAAO,GAAG,kBAAkB,CAChC,iCAAiC,EACjC,mBAAmB,EACnB,MAAM,CACP,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;YACjD,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;YAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,8EAA8E,CAAC,CAAC;YAC1G,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,oFAAoF,CAAC,CAAC;YAChH,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/content/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/content/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/content/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC"}
|
package/dist/content/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/content/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/content/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gitlab-docs-mcp",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"mcpName": "io.github.ozanmutlu/gitlab-docs",
|
|
5
5
|
"author": "Ozan Mutlu",
|
|
6
6
|
"license": "MIT",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"gitlab-docs-mcp": "./dist/bin.js"
|
|
16
16
|
},
|
|
17
17
|
"engines": {
|
|
18
|
-
"node": ">=
|
|
18
|
+
"node": ">=22.0.0"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"build": "tsc",
|