ctxman 3.2.7 → 3.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +2 -2
- package/bin/cm-count.js +45 -6
- package/index.d.ts +9 -0
- package/lib/api/mcp/server.js +229 -17
- package/lib/count/index.js +74 -14
- package/package.json +1 -1
package/bin/cli.js
CHANGED
|
@@ -810,8 +810,8 @@ function printHelp() {
|
|
|
810
810
|
console.log(' who TARGET [options] Find code & domain experts across git blame, history,');
|
|
811
811
|
console.log(' and CODEOWNERS (--repo DIR for multi-repo workspaces)');
|
|
812
812
|
console.log(' search QUERY [path] Ranked (BM25) search over methods and code chunks');
|
|
813
|
-
console.log(' count [path|-] [--model ID] [--api]');
|
|
814
|
-
console.log(' Count tokens for a model
|
|
813
|
+
console.log(' count [path|-] [--model ID] [--repo DIR] [--api]');
|
|
814
|
+
console.log(' Count tokens for a model or multi-repo workspace');
|
|
815
815
|
console.log(' mcp [install|uninstall|config]');
|
|
816
816
|
console.log(' Start the MCP server or add it to an MCP client');
|
|
817
817
|
console.log(
|
package/bin/cm-count.js
CHANGED
|
@@ -20,11 +20,12 @@ getLogger().level = 'warn';
|
|
|
20
20
|
|
|
21
21
|
const HELP = `Usage: ctxman count [path | -] [options]
|
|
22
22
|
|
|
23
|
-
Count the tokens of a project
|
|
24
|
-
and check them against the model's context window.
|
|
23
|
+
Count the tokens of a project, multi-repository workspace, or text piped to stdin with "-"
|
|
24
|
+
for a target model, and check them against the model's context window.
|
|
25
25
|
|
|
26
26
|
Options:
|
|
27
27
|
--model, -m ID Target model (default: ${LLMDetector.getDefaultModelId()}; see ctxman --list-llms)
|
|
28
|
+
--repo DIR Additional repository root to count in a workspace (repeatable)
|
|
28
29
|
--api Claude models: count exactly with Anthropic's count_tokens API.
|
|
29
30
|
Sends the (secret-redacted) text to Anthropic; needs ANTHROPIC_API_KEY
|
|
30
31
|
and the @anthropic-ai/sdk package. Counting is free and generates nothing.
|
|
@@ -39,26 +40,46 @@ their counts are approximate.
|
|
|
39
40
|
Examples:
|
|
40
41
|
ctxman count --model gpt-6.1-sol
|
|
41
42
|
ctxman count --model claude-opus-5-5 --api
|
|
43
|
+
ctxman count ./api --repo ../shared --repo ../web --model claude-sonnet-5-5
|
|
42
44
|
ctxman pack | ctxman count - --model claude-sonnet-5-5 --api`;
|
|
43
45
|
|
|
44
46
|
function parseArgs(argv) {
|
|
45
|
-
const options = { top: 5 };
|
|
47
|
+
const options = { top: 5, repos: [] };
|
|
46
48
|
for (let i = 0; i < argv.length; i++) {
|
|
47
49
|
const arg = argv[i];
|
|
48
50
|
if (arg === '--model' || arg === '-m') options.model = argv[++i];
|
|
49
|
-
else if (arg
|
|
51
|
+
else if (arg.startsWith('--model=')) options.model = arg.slice('--model='.length);
|
|
52
|
+
else if (arg === '--repo') {
|
|
53
|
+
const repoDir = argv[++i];
|
|
54
|
+
if (!repoDir || repoDir.startsWith('-')) {
|
|
55
|
+
throw new Error('--repo needs a directory path');
|
|
56
|
+
}
|
|
57
|
+
options.repos.push(repoDir);
|
|
58
|
+
} else if (arg.startsWith('--repo=')) {
|
|
59
|
+
const repoDir = arg.slice('--repo='.length);
|
|
60
|
+
if (!repoDir) throw new Error('--repo needs a directory path');
|
|
61
|
+
options.repos.push(repoDir);
|
|
62
|
+
} else if (arg === '--api') options.api = true;
|
|
50
63
|
else if (arg === '--json') options.json = true;
|
|
51
64
|
else if (arg === '--top') options.top = Number.parseInt(argv[++i], 10);
|
|
65
|
+
else if (arg.startsWith('--top='))
|
|
66
|
+
options.top = Number.parseInt(arg.slice('--top='.length), 10);
|
|
52
67
|
else if (arg === '--help' || arg === '-h') options.help = true;
|
|
53
68
|
else if (arg === '-' || !arg.startsWith('-')) options.target = arg;
|
|
54
69
|
else throw new Error(`Unknown option: ${arg}`);
|
|
55
70
|
}
|
|
56
|
-
if (
|
|
71
|
+
if (
|
|
72
|
+
(options.model === undefined || options.model === '') &&
|
|
73
|
+
argv.some((a) => a === '--model' || a === '-m' || a === '--model=')
|
|
74
|
+
) {
|
|
57
75
|
throw new Error('--model needs a model id');
|
|
58
76
|
}
|
|
59
77
|
if (!Number.isInteger(options.top) || options.top < 0) {
|
|
60
78
|
throw new Error('--top needs a number of files');
|
|
61
79
|
}
|
|
80
|
+
if (options.target === '-' && options.repos.length > 0) {
|
|
81
|
+
throw new Error('Cannot combine --repo with stdin ("-")');
|
|
82
|
+
}
|
|
62
83
|
return options;
|
|
63
84
|
}
|
|
64
85
|
|
|
@@ -89,8 +110,17 @@ function printResult(result, top) {
|
|
|
89
110
|
print(` For an exact count: ctxman count --model ${result.model} --api`);
|
|
90
111
|
}
|
|
91
112
|
|
|
113
|
+
if (result.repos?.length > 1) {
|
|
114
|
+
print('\nRepositories:');
|
|
115
|
+
for (const repo of result.repos) {
|
|
116
|
+
print(
|
|
117
|
+
` ${n(repo.tokens).padStart(9)} ${repo.alias} (${n(repo.files)} file${repo.files === 1 ? '' : 's'}, ${repo.percentOfWindow.toFixed(1)}%)`
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
92
122
|
const largest = result.largestFiles.slice(0, top);
|
|
93
|
-
if (largest.length > 1) {
|
|
123
|
+
if (largest.length > 1 || (result.repos?.length > 1 && largest.length > 0)) {
|
|
94
124
|
const scale = result.method === 'anthropic-api' && result.ratio ? result.ratio : 1;
|
|
95
125
|
print(`\nLargest files${scale !== 1 ? ' (≈, scaled to the API count)' : ''}:`);
|
|
96
126
|
for (const file of largest) {
|
|
@@ -119,6 +149,14 @@ async function main() {
|
|
|
119
149
|
process.exit(1);
|
|
120
150
|
}
|
|
121
151
|
|
|
152
|
+
const repos = options.repos.map((r) => path.resolve(r));
|
|
153
|
+
for (const repoPath of repos) {
|
|
154
|
+
if (!fs.statSync(repoPath, { throwIfNoEntry: false })?.isDirectory()) {
|
|
155
|
+
console.error(`❌ Not a directory: ${repoPath}`);
|
|
156
|
+
process.exit(1);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
122
160
|
const model = options.model || LLMDetector.getDefaultModelId();
|
|
123
161
|
if (options.api && LLMDetector.getProfile(model).vendor === 'Anthropic') {
|
|
124
162
|
console.error('Sending the redacted text to the Anthropic API to count tokens...');
|
|
@@ -129,6 +167,7 @@ async function main() {
|
|
|
129
167
|
model,
|
|
130
168
|
api: options.api,
|
|
131
169
|
text,
|
|
170
|
+
repos,
|
|
132
171
|
onProgress: (done, total) => {
|
|
133
172
|
if (total > 1) process.stderr.write(`\r request ${done}/${total}`);
|
|
134
173
|
if (total > 1 && done === total) process.stderr.write('\n');
|
package/index.d.ts
CHANGED
|
@@ -1765,6 +1765,7 @@ export function getSymbol(root: string, query: string, options?: GetSymbolOption
|
|
|
1765
1765
|
export interface CountForModelOptions {
|
|
1766
1766
|
model?: string;
|
|
1767
1767
|
text?: string;
|
|
1768
|
+
repos?: WorkspaceRepoInput[];
|
|
1768
1769
|
api?: boolean;
|
|
1769
1770
|
client?: unknown;
|
|
1770
1771
|
onProgress?: (done: number, total: number) => void;
|
|
@@ -1785,6 +1786,14 @@ export interface CountForModelResult {
|
|
|
1785
1786
|
percentOfWindow: number;
|
|
1786
1787
|
requests?: number;
|
|
1787
1788
|
ratio?: number | null;
|
|
1789
|
+
repos?: Array<{
|
|
1790
|
+
alias: string;
|
|
1791
|
+
root: string;
|
|
1792
|
+
files: number;
|
|
1793
|
+
localTokens: number;
|
|
1794
|
+
tokens: number;
|
|
1795
|
+
percentOfWindow: number;
|
|
1796
|
+
}>;
|
|
1788
1797
|
largestFiles: Array<{ path: string; tokens: number }>;
|
|
1789
1798
|
}
|
|
1790
1799
|
|
package/lib/api/mcp/server.js
CHANGED
|
@@ -25,7 +25,12 @@ import { countForModel } from '../../count/index.js';
|
|
|
25
25
|
import { explainHistory } from '../../why/index.js';
|
|
26
26
|
import { packContext, describePack, TOOL_DEFAULT_BUDGET } from '../../pack/index.js';
|
|
27
27
|
import { focusContext, describeFocus, DEFAULT_FOCUS_DEPTH } from '../../focus/index.js';
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
resolveWorkspaceRepos,
|
|
30
|
+
mapWorkspace,
|
|
31
|
+
searchWorkspace,
|
|
32
|
+
packWorkspace,
|
|
33
|
+
} from '../../workspace/index.js';
|
|
29
34
|
import {
|
|
30
35
|
loadPolicy,
|
|
31
36
|
guardExport,
|
|
@@ -118,7 +123,8 @@ class MCPServer {
|
|
|
118
123
|
tools: [
|
|
119
124
|
{
|
|
120
125
|
name: 'analyze_codebase',
|
|
121
|
-
description:
|
|
126
|
+
description:
|
|
127
|
+
'Analyze the codebase (single-repo or multi-repo workspace) and return statistics and token counts',
|
|
122
128
|
inputSchema: {
|
|
123
129
|
type: 'object',
|
|
124
130
|
properties: {
|
|
@@ -126,13 +132,20 @@ class MCPServer {
|
|
|
126
132
|
type: 'string',
|
|
127
133
|
description: 'Absolute path to the project root',
|
|
128
134
|
},
|
|
135
|
+
repos: {
|
|
136
|
+
type: 'array',
|
|
137
|
+
items: { type: 'string' },
|
|
138
|
+
description:
|
|
139
|
+
'Additional repository roots to analyze together in a multi-repository workspace',
|
|
140
|
+
},
|
|
129
141
|
},
|
|
130
142
|
required: ['path'],
|
|
131
143
|
},
|
|
132
144
|
},
|
|
133
145
|
{
|
|
134
146
|
name: 'generate_context',
|
|
135
|
-
description:
|
|
147
|
+
description:
|
|
148
|
+
'Generate optimized context for LLMs from the codebase (single-repo or multi-repo workspace)',
|
|
136
149
|
inputSchema: {
|
|
137
150
|
type: 'object',
|
|
138
151
|
properties: {
|
|
@@ -149,13 +162,20 @@ class MCPServer {
|
|
|
149
162
|
type: 'number',
|
|
150
163
|
description: 'Maximum tokens for the context',
|
|
151
164
|
},
|
|
165
|
+
repos: {
|
|
166
|
+
type: 'array',
|
|
167
|
+
items: { type: 'string' },
|
|
168
|
+
description:
|
|
169
|
+
'Additional repository roots to include in multi-repository context generation',
|
|
170
|
+
},
|
|
152
171
|
},
|
|
153
172
|
required: ['path'],
|
|
154
173
|
},
|
|
155
174
|
},
|
|
156
175
|
{
|
|
157
176
|
name: 'git_diff',
|
|
158
|
-
description:
|
|
177
|
+
description:
|
|
178
|
+
'Analyze git changes (single-repo or multi-repo workspace) and return diff context',
|
|
159
179
|
inputSchema: {
|
|
160
180
|
type: 'object',
|
|
161
181
|
properties: {
|
|
@@ -164,6 +184,12 @@ class MCPServer {
|
|
|
164
184
|
type: 'string',
|
|
165
185
|
description: 'Branch to compare against (default: main)',
|
|
166
186
|
},
|
|
187
|
+
repos: {
|
|
188
|
+
type: 'array',
|
|
189
|
+
items: { type: 'string' },
|
|
190
|
+
description:
|
|
191
|
+
'Additional repository roots to include in multi-repository git diff',
|
|
192
|
+
},
|
|
167
193
|
},
|
|
168
194
|
required: ['path'],
|
|
169
195
|
},
|
|
@@ -171,7 +197,7 @@ class MCPServer {
|
|
|
171
197
|
{
|
|
172
198
|
name: 'search_code',
|
|
173
199
|
description:
|
|
174
|
-
'Search source lines by regular expression (default) or plain substring. "ranked" returns BM25-ranked methods/chunks; "semantic" runs vector embedding search projected onto chunks (with BM25 fallback); "hybrid" fuses BM25 and semantic embeddings via Reciprocal Rank Fusion (RRF).',
|
|
200
|
+
'Search source lines by regular expression (default) or plain substring (single-repo or multi-repo workspace). "ranked" returns BM25-ranked methods/chunks; "semantic" runs vector embedding search projected onto chunks (with BM25 fallback); "hybrid" fuses BM25 and semantic embeddings via Reciprocal Rank Fusion (RRF).',
|
|
175
201
|
inputSchema: {
|
|
176
202
|
type: 'object',
|
|
177
203
|
properties: {
|
|
@@ -197,8 +223,7 @@ class MCPServer {
|
|
|
197
223
|
repos: {
|
|
198
224
|
type: 'array',
|
|
199
225
|
items: { type: 'string' },
|
|
200
|
-
description:
|
|
201
|
-
'ranked/hybrid: additional repository roots to search across under one shared token budget',
|
|
226
|
+
description: 'Additional repository roots to search across in the workspace',
|
|
202
227
|
},
|
|
203
228
|
},
|
|
204
229
|
required: ['path', 'query'],
|
|
@@ -206,7 +231,8 @@ class MCPServer {
|
|
|
206
231
|
},
|
|
207
232
|
{
|
|
208
233
|
name: 'list_methods',
|
|
209
|
-
description:
|
|
234
|
+
description:
|
|
235
|
+
'List all methods/functions in a given file, project, or multi-repository workspace',
|
|
210
236
|
inputSchema: {
|
|
211
237
|
type: 'object',
|
|
212
238
|
properties: {
|
|
@@ -216,7 +242,14 @@ class MCPServer {
|
|
|
216
242
|
},
|
|
217
243
|
file: {
|
|
218
244
|
type: 'string',
|
|
219
|
-
description:
|
|
245
|
+
description:
|
|
246
|
+
'Optional: Relative path (or <alias>/<path>) to a specific file within the project',
|
|
247
|
+
},
|
|
248
|
+
repos: {
|
|
249
|
+
type: 'array',
|
|
250
|
+
items: { type: 'string' },
|
|
251
|
+
description:
|
|
252
|
+
'Additional repository roots to list methods across in a multi-repository workspace',
|
|
220
253
|
},
|
|
221
254
|
},
|
|
222
255
|
required: ['path'],
|
|
@@ -307,7 +340,7 @@ class MCPServer {
|
|
|
307
340
|
{
|
|
308
341
|
name: 'count_tokens',
|
|
309
342
|
description:
|
|
310
|
-
"Token count of a project (or of a text) for a model, with the model's context window and whether it fits. Exact for OpenAI models; approximate for models with their own tokenizer (Claude, Gemini, ...). Nothing leaves the machine; for Claude's exact count run `ctxman count --model <id> --api`",
|
|
343
|
+
"Token count of a project or multi-repository workspace (or of a text) for a model, with the model's context window and whether it fits. Exact for OpenAI models; approximate for models with their own tokenizer (Claude, Gemini, ...). Nothing leaves the machine; for Claude's exact count run `ctxman count --model <id> --api`",
|
|
311
344
|
inputSchema: {
|
|
312
345
|
type: 'object',
|
|
313
346
|
properties: {
|
|
@@ -324,6 +357,12 @@ class MCPServer {
|
|
|
324
357
|
type: 'number',
|
|
325
358
|
description: 'How many of the largest files to list (default 10)',
|
|
326
359
|
},
|
|
360
|
+
repos: {
|
|
361
|
+
type: 'array',
|
|
362
|
+
items: { type: 'string' },
|
|
363
|
+
description:
|
|
364
|
+
'Additional repository roots to count across in a multi-repository workspace (ignored when text is given)',
|
|
365
|
+
},
|
|
327
366
|
},
|
|
328
367
|
},
|
|
329
368
|
},
|
|
@@ -813,6 +852,28 @@ class MCPServer {
|
|
|
813
852
|
// Update resource provider project path
|
|
814
853
|
this.resourceProvider.projectPath = projectPath;
|
|
815
854
|
|
|
855
|
+
if (args.repos?.length > 0) {
|
|
856
|
+
const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
|
|
857
|
+
const allFiles = [];
|
|
858
|
+
for (const repo of resolvedRepos) {
|
|
859
|
+
const scanner = new Scanner(repo.root, { policy: repo.policy });
|
|
860
|
+
for (const file of scanner.scan()) {
|
|
861
|
+
const relPosix = `${repo.alias}/${file.relativePath.split(path.sep).join('/')}`;
|
|
862
|
+
allFiles.push({
|
|
863
|
+
...file,
|
|
864
|
+
relativePath: relPosix,
|
|
865
|
+
});
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
const analysis = await this.analyzer.analyze(allFiles);
|
|
869
|
+
analysis.repos = resolvedRepos.map((r) => ({ alias: r.alias, root: r.root }));
|
|
870
|
+
const analysisId = Date.now().toString();
|
|
871
|
+
this.resourceProvider.cacheAnalysis(analysisId, analysis);
|
|
872
|
+
return {
|
|
873
|
+
content: [{ type: 'text', text: JSON.stringify(analysis, null, 2) }],
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
|
|
816
877
|
const scanner = new Scanner(projectPath);
|
|
817
878
|
const files = scanner.scan();
|
|
818
879
|
const analysis = await this.analyzer.analyze(files);
|
|
@@ -836,20 +897,57 @@ class MCPServer {
|
|
|
836
897
|
// Update resource provider project path
|
|
837
898
|
this.resourceProvider.projectPath = projectPath;
|
|
838
899
|
|
|
839
|
-
const scanner = new Scanner(projectPath);
|
|
840
|
-
const files = scanner.scan();
|
|
841
|
-
const analysis = await this.analyzer.analyze(files);
|
|
842
|
-
|
|
843
|
-
// Configure ContextBuilder based on args
|
|
844
900
|
const builderOptions = {
|
|
845
901
|
targetTokens: args.maxTokens,
|
|
846
902
|
useCase: args.template || 'custom',
|
|
847
903
|
};
|
|
848
904
|
const contextBuilder = new ContextBuilder(builderOptions);
|
|
905
|
+
const templateName = args.template || 'custom';
|
|
906
|
+
|
|
907
|
+
if (args.repos?.length > 0) {
|
|
908
|
+
const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
|
|
909
|
+
const allFiles = [];
|
|
910
|
+
for (const repo of resolvedRepos) {
|
|
911
|
+
const scanner = new Scanner(repo.root, { policy: repo.policy });
|
|
912
|
+
for (const file of scanner.scan()) {
|
|
913
|
+
const relPosix = `${repo.alias}/${file.relativePath.split(path.sep).join('/')}`;
|
|
914
|
+
allFiles.push({
|
|
915
|
+
...file,
|
|
916
|
+
relativePath: relPosix,
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
const analysis = await this.analyzer.analyze(allFiles);
|
|
921
|
+
const context = contextBuilder.build(analysis);
|
|
922
|
+
context.repos = resolvedRepos.map((r) => ({ alias: r.alias, root: r.root }));
|
|
923
|
+
const contextText = JSON.stringify(context, null, 2);
|
|
924
|
+
const exportedFiles = Object.values(context.files || {}).flat();
|
|
925
|
+
for (const repo of resolvedRepos) {
|
|
926
|
+
const prefix = `${repo.alias}/`;
|
|
927
|
+
const repoFiles = exportedFiles
|
|
928
|
+
.filter((f) => typeof f.path === 'string' && f.path.startsWith(prefix))
|
|
929
|
+
.map((f) => ({ ...f, path: f.path.slice(prefix.length) }));
|
|
930
|
+
guardExport(repo.policy, {
|
|
931
|
+
command: 'mcp generate_context',
|
|
932
|
+
root: repo.root,
|
|
933
|
+
destination: 'mcp',
|
|
934
|
+
format: 'json',
|
|
935
|
+
files: repoFiles,
|
|
936
|
+
text: contextText,
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
this.resourceProvider.cacheContext(templateName, contextText);
|
|
940
|
+
return {
|
|
941
|
+
content: [{ type: 'text', text: contextText }],
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
const scanner = new Scanner(projectPath);
|
|
946
|
+
const files = scanner.scan();
|
|
947
|
+
const analysis = await this.analyzer.analyze(files);
|
|
849
948
|
const context = contextBuilder.build(analysis);
|
|
850
949
|
|
|
851
950
|
// Cache context as a resource
|
|
852
|
-
const templateName = args.template || 'custom';
|
|
853
951
|
const contextText = JSON.stringify(context, null, 2);
|
|
854
952
|
// Export policy: throws PolicyViolationError (an isError result) and logs the export
|
|
855
953
|
guardExport(loadPolicy(projectPath), {
|
|
@@ -873,7 +971,6 @@ class MCPServer {
|
|
|
873
971
|
}
|
|
874
972
|
case 'git_diff': {
|
|
875
973
|
const projectPath = args.path;
|
|
876
|
-
const gitClient = new GitClient(projectPath);
|
|
877
974
|
const branch = args.branch || 'main';
|
|
878
975
|
|
|
879
976
|
if (!isValidGitRef(branch)) {
|
|
@@ -885,6 +982,47 @@ class MCPServer {
|
|
|
885
982
|
};
|
|
886
983
|
}
|
|
887
984
|
|
|
985
|
+
if (args.repos?.length > 0) {
|
|
986
|
+
const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
|
|
987
|
+
const diffs = [];
|
|
988
|
+
const errors = [];
|
|
989
|
+
for (const repo of resolvedRepos) {
|
|
990
|
+
try {
|
|
991
|
+
const gitClient = new GitClient(repo.root);
|
|
992
|
+
const rawDiff = gitClient.exec([
|
|
993
|
+
'diff',
|
|
994
|
+
branch,
|
|
995
|
+
'--',
|
|
996
|
+
...(this.root ? ['.'] : []),
|
|
997
|
+
]);
|
|
998
|
+
if (rawDiff && rawDiff.trim()) {
|
|
999
|
+
const prefixed = rawDiff
|
|
1000
|
+
.replace(
|
|
1001
|
+
/^diff --git a\/(.+) b\/(.+)$/gm,
|
|
1002
|
+
`diff --git a/${repo.alias}/$1 b/${repo.alias}/$2`
|
|
1003
|
+
)
|
|
1004
|
+
.replace(/^--- a\/(.+)$/gm, `--- a/${repo.alias}/$1`)
|
|
1005
|
+
.replace(/^\+\+\+ b\/(.+)$/gm, `+++ b/${repo.alias}/$1`);
|
|
1006
|
+
diffs.push(prefixed.trimEnd());
|
|
1007
|
+
}
|
|
1008
|
+
} catch (e) {
|
|
1009
|
+
errors.push(`${repo.alias}: ${e.message}`);
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
if (errors.length === resolvedRepos.length) {
|
|
1013
|
+
return {
|
|
1014
|
+
content: [{ type: 'text', text: `Error executing git diff: ${errors[0]}` }],
|
|
1015
|
+
isError: true,
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
const notes = errors.map((err) => `# Note: ${err}`).join('\n');
|
|
1019
|
+
const combined = [diffs.join('\n'), notes].filter(Boolean).join('\n\n');
|
|
1020
|
+
return {
|
|
1021
|
+
content: [{ type: 'text', text: combined || 'No changes detected.' }],
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
const gitClient = new GitClient(projectPath);
|
|
888
1026
|
let diff;
|
|
889
1027
|
try {
|
|
890
1028
|
// With a root, keep the diff to that directory even if the repository is larger
|
|
@@ -942,6 +1080,37 @@ class MCPServer {
|
|
|
942
1080
|
};
|
|
943
1081
|
}
|
|
944
1082
|
}
|
|
1083
|
+
|
|
1084
|
+
if (args.repos?.length > 0) {
|
|
1085
|
+
const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
|
|
1086
|
+
const results = [];
|
|
1087
|
+
for (const repo of resolvedRepos) {
|
|
1088
|
+
const scanner = new Scanner(repo.root, { policy: repo.policy });
|
|
1089
|
+
for (const file of scanner.scan()) {
|
|
1090
|
+
try {
|
|
1091
|
+
const content = fs.readFileSync(file.path, 'utf-8');
|
|
1092
|
+
const lines = content.split('\n');
|
|
1093
|
+
const relPosix = `${repo.alias}/${file.relativePath.split(path.sep).join('/')}`;
|
|
1094
|
+
lines.forEach((line, index) => {
|
|
1095
|
+
const match = pattern ? pattern.test(line) : line.includes(query);
|
|
1096
|
+
if (match) {
|
|
1097
|
+
results.push({
|
|
1098
|
+
file: relPosix,
|
|
1099
|
+
line: index + 1,
|
|
1100
|
+
content: line.trim(),
|
|
1101
|
+
});
|
|
1102
|
+
}
|
|
1103
|
+
});
|
|
1104
|
+
} catch (_e) {
|
|
1105
|
+
// Skip read errors
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
return {
|
|
1110
|
+
content: [{ type: 'text', text: JSON.stringify(results, null, 2) }],
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
1113
|
+
|
|
945
1114
|
const scanner = new Scanner(projectPath);
|
|
946
1115
|
const files = scanner.scan();
|
|
947
1116
|
const results = [];
|
|
@@ -979,6 +1148,48 @@ class MCPServer {
|
|
|
979
1148
|
file = path.basename(projectPath);
|
|
980
1149
|
projectPath = path.dirname(projectPath);
|
|
981
1150
|
}
|
|
1151
|
+
|
|
1152
|
+
if (args.repos?.length > 0) {
|
|
1153
|
+
const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
|
|
1154
|
+
const targetPosix = file ? file.split(path.sep).join('/').replace(/^\.\//, '') : null;
|
|
1155
|
+
const allFiles = [];
|
|
1156
|
+
for (const repo of resolvedRepos) {
|
|
1157
|
+
const scanner = new Scanner(repo.root, { policy: repo.policy });
|
|
1158
|
+
for (const f of scanner.scan()) {
|
|
1159
|
+
const barePosix = f.relativePath.split(path.sep).join('/');
|
|
1160
|
+
const relPosix = `${repo.alias}/${barePosix}`;
|
|
1161
|
+
if (!targetPosix || relPosix === targetPosix || barePosix === targetPosix) {
|
|
1162
|
+
allFiles.push({
|
|
1163
|
+
...f,
|
|
1164
|
+
relativePath: relPosix,
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
if (targetPosix && allFiles.length === 0) {
|
|
1170
|
+
return {
|
|
1171
|
+
content: [{ type: 'text', text: `File not found in project: ${file}` }],
|
|
1172
|
+
isError: true,
|
|
1173
|
+
};
|
|
1174
|
+
}
|
|
1175
|
+
const methodAnalyzer = new Analyzer({ methodLevel: true, cache: true });
|
|
1176
|
+
const analysis = await methodAnalyzer.analyze(allFiles);
|
|
1177
|
+
const methods = [];
|
|
1178
|
+
if (analysis.files) {
|
|
1179
|
+
for (const analyzedFile of analysis.files) {
|
|
1180
|
+
if (analyzedFile.methods && analyzedFile.methods.length > 0) {
|
|
1181
|
+
methods.push({
|
|
1182
|
+
file: analyzedFile.relativePath,
|
|
1183
|
+
methods: analyzedFile.methods,
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
return {
|
|
1189
|
+
content: [{ type: 'text', text: JSON.stringify(methods, null, 2) }],
|
|
1190
|
+
};
|
|
1191
|
+
}
|
|
1192
|
+
|
|
982
1193
|
const scanner = new Scanner(projectPath);
|
|
983
1194
|
// Optional `file`: one project-relative file instead of the whole project
|
|
984
1195
|
const target = file ? path.normalize(file) : null;
|
|
@@ -1060,6 +1271,7 @@ class MCPServer {
|
|
|
1060
1271
|
const { largestFiles, ...summary } = await countForModel(args.path, {
|
|
1061
1272
|
model: args.model,
|
|
1062
1273
|
text: args.text,
|
|
1274
|
+
repos: args.repos,
|
|
1063
1275
|
});
|
|
1064
1276
|
const top = Number.isInteger(args.top) && args.top >= 0 ? args.top : 10;
|
|
1065
1277
|
const result = { ...summary, largestFiles: largestFiles.slice(0, top) };
|
package/lib/count/index.js
CHANGED
|
@@ -1,40 +1,73 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Token count for a target model
|
|
3
|
-
* Counts a project
|
|
4
|
-
* available offline (tiktoken for OpenAI models), and, on request,
|
|
5
|
-
* Anthropic's count_tokens API for Claude models.
|
|
3
|
+
* Counts a project, multi-repository workspace, or piped text with the target model's
|
|
4
|
+
* tokenizer where one is available offline (tiktoken for OpenAI models), and, on request,
|
|
5
|
+
* exactly through Anthropic's count_tokens API for Claude models.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import fs from 'fs';
|
|
9
|
+
import path from 'path';
|
|
9
10
|
import Scanner from '../core/Scanner.js';
|
|
10
11
|
import TokenUtils from '../utils/token-utils.js';
|
|
11
12
|
import { LLMDetector } from '../utils/llm-detector.js';
|
|
12
13
|
import { ClaudeTokenCounter } from '../utils/claude-token-counter.js';
|
|
13
14
|
import { redactSecrets } from '../utils/secret-redactor.js';
|
|
15
|
+
import { resolveWorkspaceRepos } from '../workspace/index.js';
|
|
14
16
|
|
|
15
17
|
/**
|
|
16
|
-
* Read the documents to count
|
|
18
|
+
* Read the documents to count from a single project or multi-repository workspace
|
|
17
19
|
* @param {string} root - Project root (ignored when `text` is given)
|
|
18
20
|
* @param {string} [text] - A single piped text instead of the project files
|
|
19
|
-
* @
|
|
21
|
+
* @param {Array<string|{root: string, alias?: string}>} [repos] - Additional repository roots
|
|
22
|
+
* @returns {{documents: {path: string, content: string, repo?: string}[], resolvedRepos: {root: string, alias: string}[] | null}}
|
|
20
23
|
*/
|
|
21
|
-
function readDocuments(root, text) {
|
|
22
|
-
if (text !== undefined)
|
|
23
|
-
|
|
24
|
+
function readDocuments(root, text, repos) {
|
|
25
|
+
if (text !== undefined) {
|
|
26
|
+
return {
|
|
27
|
+
documents: [{ path: '<stdin>', content: text }],
|
|
28
|
+
resolvedRepos: null,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const extraRepos = Array.isArray(repos) ? repos.filter(Boolean) : [];
|
|
33
|
+
if (extraRepos.length > 0) {
|
|
34
|
+
const resolvedRepos = resolveWorkspaceRepos([root, ...extraRepos]);
|
|
35
|
+
const documents = [];
|
|
36
|
+
for (const repo of resolvedRepos) {
|
|
37
|
+
const scanned = new Scanner(repo.root, { policy: repo.policy }).scan();
|
|
38
|
+
for (const file of scanned) {
|
|
39
|
+
try {
|
|
40
|
+
const relPosix = file.relativePath.split(path.sep).join('/');
|
|
41
|
+
documents.push({
|
|
42
|
+
path: `${repo.alias}/${relPosix}`,
|
|
43
|
+
repo: repo.alias,
|
|
44
|
+
content: fs.readFileSync(file.path, 'utf8'),
|
|
45
|
+
});
|
|
46
|
+
} catch (_error) {
|
|
47
|
+
// Skip unreadable files
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return { documents, resolvedRepos };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const documents = new Scanner(root).scan().flatMap((file) => {
|
|
24
55
|
try {
|
|
25
56
|
return [{ path: file.relativePath, content: fs.readFileSync(file.path, 'utf8') }];
|
|
26
57
|
} catch (_error) {
|
|
27
58
|
return [];
|
|
28
59
|
}
|
|
29
60
|
});
|
|
61
|
+
return { documents, resolvedRepos: null };
|
|
30
62
|
}
|
|
31
63
|
|
|
32
64
|
/**
|
|
33
|
-
* Count tokens for a target model
|
|
65
|
+
* Count tokens for a target model across a project, multi-repo workspace, or piped text
|
|
34
66
|
* @param {string} root - Project root
|
|
35
67
|
* @param {object} [options]
|
|
36
68
|
* @param {string} [options.model] - Target model id or alias (default: DEFAULT_TARGET_MODEL)
|
|
37
69
|
* @param {string} [options.text] - Count this text instead of the project
|
|
70
|
+
* @param {Array<string|{root: string, alias?: string}>} [options.repos] - Additional workspace repository roots
|
|
38
71
|
* @param {boolean} [options.api=false] - Count exactly with the Anthropic API (Claude only)
|
|
39
72
|
* @param {object} [options.client] - Anthropic client stand-in (tests)
|
|
40
73
|
* @param {(done: number, total: number) => void} [options.onProgress]
|
|
@@ -45,12 +78,13 @@ export async function countForModel(root, options = {}) {
|
|
|
45
78
|
const profile = LLMDetector.getProfile(requested);
|
|
46
79
|
const modelId = profile.id || requested;
|
|
47
80
|
const encoding = TokenUtils.encodingFor(modelId);
|
|
48
|
-
const documents = readDocuments(root, options.text);
|
|
81
|
+
const { documents, resolvedRepos } = readDocuments(root, options.text, options.repos);
|
|
49
82
|
|
|
50
83
|
const files = documents
|
|
51
84
|
.map((doc) => ({
|
|
52
85
|
path: doc.path,
|
|
53
86
|
tokens: TokenUtils.calculate(doc.content, doc.path, encoding),
|
|
87
|
+
...(doc.repo ? { repo: doc.repo } : {}),
|
|
54
88
|
}))
|
|
55
89
|
.sort((a, b) => b.tokens - a.tokens);
|
|
56
90
|
const localTokens = files.reduce((sum, file) => sum + file.tokens, 0);
|
|
@@ -67,7 +101,7 @@ export async function countForModel(root, options = {}) {
|
|
|
67
101
|
// tiktoken matches OpenAI models whose encoding is published; others are approximations
|
|
68
102
|
exact: TokenUtils.isExact() && profile.tokenizer?.exact === true,
|
|
69
103
|
contextWindow: profile.contextWindow,
|
|
70
|
-
largestFiles: files,
|
|
104
|
+
largestFiles: files.map(({ path: filePath, tokens }) => ({ path: filePath, tokens })),
|
|
71
105
|
};
|
|
72
106
|
|
|
73
107
|
if (options.api) {
|
|
@@ -89,11 +123,37 @@ export async function countForModel(root, options = {}) {
|
|
|
89
123
|
result.method = 'anthropic-api';
|
|
90
124
|
result.exact = true;
|
|
91
125
|
result.requests = requests;
|
|
92
|
-
result.ratio = localTokens > 0 ? total / localTokens :
|
|
126
|
+
result.ratio = localTokens > 0 ? total / localTokens : 1;
|
|
127
|
+
// The API counts batches, not single files; scale each file's local count by the
|
|
128
|
+
// observed ratio so the per-file breakdown matches the model too
|
|
129
|
+
result.largestFiles = files.map(({ path: filePath, tokens }) => ({
|
|
130
|
+
path: filePath,
|
|
131
|
+
tokens: Math.round(tokens * result.ratio),
|
|
132
|
+
}));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const window = result.contextWindow || 0;
|
|
136
|
+
result.fits = window > 0 ? result.tokens <= window : true;
|
|
137
|
+
result.percentOfWindow = window > 0 ? (result.tokens / window) * 100 : 0;
|
|
138
|
+
|
|
139
|
+
if (resolvedRepos) {
|
|
140
|
+
const ratio = result.method === 'anthropic-api' && result.ratio ? result.ratio : 1;
|
|
141
|
+
result.repos = resolvedRepos.map((repo) => {
|
|
142
|
+
const repoFiles = files.filter((f) => f.repo === repo.alias);
|
|
143
|
+
const repoLocalTokens = repoFiles.reduce((sum, f) => sum + f.tokens, 0);
|
|
144
|
+
const repoTokens =
|
|
145
|
+
result.method === 'anthropic-api' ? Math.round(repoLocalTokens * ratio) : repoLocalTokens;
|
|
146
|
+
return {
|
|
147
|
+
alias: repo.alias,
|
|
148
|
+
root: repo.root,
|
|
149
|
+
files: repoFiles.length,
|
|
150
|
+
localTokens: repoLocalTokens,
|
|
151
|
+
tokens: repoTokens,
|
|
152
|
+
percentOfWindow: window > 0 ? (repoTokens / window) * 100 : 0,
|
|
153
|
+
};
|
|
154
|
+
});
|
|
93
155
|
}
|
|
94
156
|
|
|
95
|
-
result.fits = result.tokens <= result.contextWindow;
|
|
96
|
-
result.percentOfWindow = (result.tokens / result.contextWindow) * 100;
|
|
97
157
|
return result;
|
|
98
158
|
}
|
|
99
159
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ctxman",
|
|
3
|
-
"version": "3.2.
|
|
3
|
+
"version": "3.2.8",
|
|
4
4
|
"description": "AI Development Platform with plugin architecture, Git integration, REST API, watch mode, LLM optimization, and support for 14+ languages. Features include TOON format (40-50% token reduction), real-time analysis, and comprehensive context generation.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|