ctxman 3.2.7 → 3.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/cli.js CHANGED
@@ -810,8 +810,8 @@ function printHelp() {
810
810
  console.log(' who TARGET [options] Find code & domain experts across git blame, history,');
811
811
  console.log(' and CODEOWNERS (--repo DIR for multi-repo workspaces)');
812
812
  console.log(' search QUERY [path] Ranked (BM25) search over methods and code chunks');
813
- console.log(' count [path|-] [--model ID] [--api]');
814
- console.log(' Count tokens for a model (exact for Claude with --api)');
813
+ console.log(' count [path|-] [--model ID] [--repo DIR] [--api]');
814
+ console.log(' Count tokens for a model or multi-repo workspace');
815
815
  console.log(' mcp [install|uninstall|config]');
816
816
  console.log(' Start the MCP server or add it to an MCP client');
817
817
  console.log(
package/bin/cm-count.js CHANGED
@@ -20,11 +20,12 @@ getLogger().level = 'warn';
20
20
 
21
21
  const HELP = `Usage: ctxman count [path | -] [options]
22
22
 
23
- Count the tokens of a project (or of text piped to stdin with "-") for a target model,
24
- and check them against the model's context window.
23
+ Count the tokens of a project, multi-repository workspace, or text piped to stdin with "-"
24
+ for a target model, and check them against the model's context window.
25
25
 
26
26
  Options:
27
27
  --model, -m ID Target model (default: ${LLMDetector.getDefaultModelId()}; see ctxman --list-llms)
28
+ --repo DIR Additional repository root to count in a workspace (repeatable)
28
29
  --api Claude models: count exactly with Anthropic's count_tokens API.
29
30
  Sends the (secret-redacted) text to Anthropic; needs ANTHROPIC_API_KEY
30
31
  and the @anthropic-ai/sdk package. Counting is free and generates nothing.
@@ -39,26 +40,46 @@ their counts are approximate.
39
40
  Examples:
40
41
  ctxman count --model gpt-6.1-sol
41
42
  ctxman count --model claude-opus-5-5 --api
43
+ ctxman count ./api --repo ../shared --repo ../web --model claude-sonnet-5-5
42
44
  ctxman pack | ctxman count - --model claude-sonnet-5-5 --api`;
43
45
 
44
46
  function parseArgs(argv) {
45
- const options = { top: 5 };
47
+ const options = { top: 5, repos: [] };
46
48
  for (let i = 0; i < argv.length; i++) {
47
49
  const arg = argv[i];
48
50
  if (arg === '--model' || arg === '-m') options.model = argv[++i];
49
- else if (arg === '--api') options.api = true;
51
+ else if (arg.startsWith('--model=')) options.model = arg.slice('--model='.length);
52
+ else if (arg === '--repo') {
53
+ const repoDir = argv[++i];
54
+ if (!repoDir || repoDir.startsWith('-')) {
55
+ throw new Error('--repo needs a directory path');
56
+ }
57
+ options.repos.push(repoDir);
58
+ } else if (arg.startsWith('--repo=')) {
59
+ const repoDir = arg.slice('--repo='.length);
60
+ if (!repoDir) throw new Error('--repo needs a directory path');
61
+ options.repos.push(repoDir);
62
+ } else if (arg === '--api') options.api = true;
50
63
  else if (arg === '--json') options.json = true;
51
64
  else if (arg === '--top') options.top = Number.parseInt(argv[++i], 10);
65
+ else if (arg.startsWith('--top='))
66
+ options.top = Number.parseInt(arg.slice('--top='.length), 10);
52
67
  else if (arg === '--help' || arg === '-h') options.help = true;
53
68
  else if (arg === '-' || !arg.startsWith('-')) options.target = arg;
54
69
  else throw new Error(`Unknown option: ${arg}`);
55
70
  }
56
- if (options.model === undefined && argv.includes('--model')) {
71
+ if (
72
+ (options.model === undefined || options.model === '') &&
73
+ argv.some((a) => a === '--model' || a === '-m' || a === '--model=')
74
+ ) {
57
75
  throw new Error('--model needs a model id');
58
76
  }
59
77
  if (!Number.isInteger(options.top) || options.top < 0) {
60
78
  throw new Error('--top needs a number of files');
61
79
  }
80
+ if (options.target === '-' && options.repos.length > 0) {
81
+ throw new Error('Cannot combine --repo with stdin ("-")');
82
+ }
62
83
  return options;
63
84
  }
64
85
 
@@ -89,8 +110,17 @@ function printResult(result, top) {
89
110
  print(` For an exact count: ctxman count --model ${result.model} --api`);
90
111
  }
91
112
 
113
+ if (result.repos?.length > 1) {
114
+ print('\nRepositories:');
115
+ for (const repo of result.repos) {
116
+ print(
117
+ ` ${n(repo.tokens).padStart(9)} ${repo.alias} (${n(repo.files)} file${repo.files === 1 ? '' : 's'}, ${repo.percentOfWindow.toFixed(1)}%)`
118
+ );
119
+ }
120
+ }
121
+
92
122
  const largest = result.largestFiles.slice(0, top);
93
- if (largest.length > 1) {
123
+ if (largest.length > 1 || (result.repos?.length > 1 && largest.length > 0)) {
94
124
  const scale = result.method === 'anthropic-api' && result.ratio ? result.ratio : 1;
95
125
  print(`\nLargest files${scale !== 1 ? ' (≈, scaled to the API count)' : ''}:`);
96
126
  for (const file of largest) {
@@ -119,6 +149,14 @@ async function main() {
119
149
  process.exit(1);
120
150
  }
121
151
 
152
+ const repos = options.repos.map((r) => path.resolve(r));
153
+ for (const repoPath of repos) {
154
+ if (!fs.statSync(repoPath, { throwIfNoEntry: false })?.isDirectory()) {
155
+ console.error(`❌ Not a directory: ${repoPath}`);
156
+ process.exit(1);
157
+ }
158
+ }
159
+
122
160
  const model = options.model || LLMDetector.getDefaultModelId();
123
161
  if (options.api && LLMDetector.getProfile(model).vendor === 'Anthropic') {
124
162
  console.error('Sending the redacted text to the Anthropic API to count tokens...');
@@ -129,6 +167,7 @@ async function main() {
129
167
  model,
130
168
  api: options.api,
131
169
  text,
170
+ repos,
132
171
  onProgress: (done, total) => {
133
172
  if (total > 1) process.stderr.write(`\r request ${done}/${total}`);
134
173
  if (total > 1 && done === total) process.stderr.write('\n');
package/index.d.ts CHANGED
@@ -1765,6 +1765,7 @@ export function getSymbol(root: string, query: string, options?: GetSymbolOption
1765
1765
  export interface CountForModelOptions {
1766
1766
  model?: string;
1767
1767
  text?: string;
1768
+ repos?: WorkspaceRepoInput[];
1768
1769
  api?: boolean;
1769
1770
  client?: unknown;
1770
1771
  onProgress?: (done: number, total: number) => void;
@@ -1785,6 +1786,14 @@ export interface CountForModelResult {
1785
1786
  percentOfWindow: number;
1786
1787
  requests?: number;
1787
1788
  ratio?: number | null;
1789
+ repos?: Array<{
1790
+ alias: string;
1791
+ root: string;
1792
+ files: number;
1793
+ localTokens: number;
1794
+ tokens: number;
1795
+ percentOfWindow: number;
1796
+ }>;
1788
1797
  largestFiles: Array<{ path: string; tokens: number }>;
1789
1798
  }
1790
1799
 
@@ -25,7 +25,12 @@ import { countForModel } from '../../count/index.js';
25
25
  import { explainHistory } from '../../why/index.js';
26
26
  import { packContext, describePack, TOOL_DEFAULT_BUDGET } from '../../pack/index.js';
27
27
  import { focusContext, describeFocus, DEFAULT_FOCUS_DEPTH } from '../../focus/index.js';
28
- import { mapWorkspace, searchWorkspace, packWorkspace } from '../../workspace/index.js';
28
+ import {
29
+ resolveWorkspaceRepos,
30
+ mapWorkspace,
31
+ searchWorkspace,
32
+ packWorkspace,
33
+ } from '../../workspace/index.js';
29
34
  import {
30
35
  loadPolicy,
31
36
  guardExport,
@@ -118,7 +123,8 @@ class MCPServer {
118
123
  tools: [
119
124
  {
120
125
  name: 'analyze_codebase',
121
- description: 'Analyze the codebase and return statistics and token counts',
126
+ description:
127
+ 'Analyze the codebase (single-repo or multi-repo workspace) and return statistics and token counts',
122
128
  inputSchema: {
123
129
  type: 'object',
124
130
  properties: {
@@ -126,13 +132,20 @@ class MCPServer {
126
132
  type: 'string',
127
133
  description: 'Absolute path to the project root',
128
134
  },
135
+ repos: {
136
+ type: 'array',
137
+ items: { type: 'string' },
138
+ description:
139
+ 'Additional repository roots to analyze together in a multi-repository workspace',
140
+ },
129
141
  },
130
142
  required: ['path'],
131
143
  },
132
144
  },
133
145
  {
134
146
  name: 'generate_context',
135
- description: 'Generate optimized context for LLMs from the codebase',
147
+ description:
148
+ 'Generate optimized context for LLMs from the codebase (single-repo or multi-repo workspace)',
136
149
  inputSchema: {
137
150
  type: 'object',
138
151
  properties: {
@@ -149,13 +162,20 @@ class MCPServer {
149
162
  type: 'number',
150
163
  description: 'Maximum tokens for the context',
151
164
  },
165
+ repos: {
166
+ type: 'array',
167
+ items: { type: 'string' },
168
+ description:
169
+ 'Additional repository roots to include in multi-repository context generation',
170
+ },
152
171
  },
153
172
  required: ['path'],
154
173
  },
155
174
  },
156
175
  {
157
176
  name: 'git_diff',
158
- description: 'Analyze git changes and return diff context',
177
+ description:
178
+ 'Analyze git changes (single-repo or multi-repo workspace) and return diff context',
159
179
  inputSchema: {
160
180
  type: 'object',
161
181
  properties: {
@@ -164,6 +184,12 @@ class MCPServer {
164
184
  type: 'string',
165
185
  description: 'Branch to compare against (default: main)',
166
186
  },
187
+ repos: {
188
+ type: 'array',
189
+ items: { type: 'string' },
190
+ description:
191
+ 'Additional repository roots to include in multi-repository git diff',
192
+ },
167
193
  },
168
194
  required: ['path'],
169
195
  },
@@ -171,7 +197,7 @@ class MCPServer {
171
197
  {
172
198
  name: 'search_code',
173
199
  description:
174
- 'Search source lines by regular expression (default) or plain substring. "ranked" returns BM25-ranked methods/chunks; "semantic" runs vector embedding search projected onto chunks (with BM25 fallback); "hybrid" fuses BM25 and semantic embeddings via Reciprocal Rank Fusion (RRF).',
200
+ 'Search source lines by regular expression (default) or plain substring (single-repo or multi-repo workspace). "ranked" returns BM25-ranked methods/chunks; "semantic" runs vector embedding search projected onto chunks (with BM25 fallback); "hybrid" fuses BM25 and semantic embeddings via Reciprocal Rank Fusion (RRF).',
175
201
  inputSchema: {
176
202
  type: 'object',
177
203
  properties: {
@@ -197,8 +223,7 @@ class MCPServer {
197
223
  repos: {
198
224
  type: 'array',
199
225
  items: { type: 'string' },
200
- description:
201
- 'ranked/hybrid: additional repository roots to search across under one shared token budget',
226
+ description: 'Additional repository roots to search across in the workspace',
202
227
  },
203
228
  },
204
229
  required: ['path', 'query'],
@@ -206,7 +231,8 @@ class MCPServer {
206
231
  },
207
232
  {
208
233
  name: 'list_methods',
209
- description: 'List all methods/functions in a given file or project',
234
+ description:
235
+ 'List all methods/functions in a given file, project, or multi-repository workspace',
210
236
  inputSchema: {
211
237
  type: 'object',
212
238
  properties: {
@@ -216,7 +242,14 @@ class MCPServer {
216
242
  },
217
243
  file: {
218
244
  type: 'string',
219
- description: 'Optional: Relative path to a specific file within the project',
245
+ description:
246
+ 'Optional: Relative path (or <alias>/<path>) to a specific file within the project',
247
+ },
248
+ repos: {
249
+ type: 'array',
250
+ items: { type: 'string' },
251
+ description:
252
+ 'Additional repository roots to list methods across in a multi-repository workspace',
220
253
  },
221
254
  },
222
255
  required: ['path'],
@@ -307,7 +340,7 @@ class MCPServer {
307
340
  {
308
341
  name: 'count_tokens',
309
342
  description:
310
- "Token count of a project (or of a text) for a model, with the model's context window and whether it fits. Exact for OpenAI models; approximate for models with their own tokenizer (Claude, Gemini, ...). Nothing leaves the machine; for Claude's exact count run `ctxman count --model <id> --api`",
343
+ "Token count of a project or multi-repository workspace (or of a text) for a model, with the model's context window and whether it fits. Exact for OpenAI models; approximate for models with their own tokenizer (Claude, Gemini, ...). Nothing leaves the machine; for Claude's exact count run `ctxman count --model <id> --api`",
311
344
  inputSchema: {
312
345
  type: 'object',
313
346
  properties: {
@@ -324,6 +357,12 @@ class MCPServer {
324
357
  type: 'number',
325
358
  description: 'How many of the largest files to list (default 10)',
326
359
  },
360
+ repos: {
361
+ type: 'array',
362
+ items: { type: 'string' },
363
+ description:
364
+ 'Additional repository roots to count across in a multi-repository workspace (ignored when text is given)',
365
+ },
327
366
  },
328
367
  },
329
368
  },
@@ -813,6 +852,28 @@ class MCPServer {
813
852
  // Update resource provider project path
814
853
  this.resourceProvider.projectPath = projectPath;
815
854
 
855
+ if (args.repos?.length > 0) {
856
+ const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
857
+ const allFiles = [];
858
+ for (const repo of resolvedRepos) {
859
+ const scanner = new Scanner(repo.root, { policy: repo.policy });
860
+ for (const file of scanner.scan()) {
861
+ const relPosix = `${repo.alias}/${file.relativePath.split(path.sep).join('/')}`;
862
+ allFiles.push({
863
+ ...file,
864
+ relativePath: relPosix,
865
+ });
866
+ }
867
+ }
868
+ const analysis = await this.analyzer.analyze(allFiles);
869
+ analysis.repos = resolvedRepos.map((r) => ({ alias: r.alias, root: r.root }));
870
+ const analysisId = Date.now().toString();
871
+ this.resourceProvider.cacheAnalysis(analysisId, analysis);
872
+ return {
873
+ content: [{ type: 'text', text: JSON.stringify(analysis, null, 2) }],
874
+ };
875
+ }
876
+
816
877
  const scanner = new Scanner(projectPath);
817
878
  const files = scanner.scan();
818
879
  const analysis = await this.analyzer.analyze(files);
@@ -836,20 +897,57 @@ class MCPServer {
836
897
  // Update resource provider project path
837
898
  this.resourceProvider.projectPath = projectPath;
838
899
 
839
- const scanner = new Scanner(projectPath);
840
- const files = scanner.scan();
841
- const analysis = await this.analyzer.analyze(files);
842
-
843
- // Configure ContextBuilder based on args
844
900
  const builderOptions = {
845
901
  targetTokens: args.maxTokens,
846
902
  useCase: args.template || 'custom',
847
903
  };
848
904
  const contextBuilder = new ContextBuilder(builderOptions);
905
+ const templateName = args.template || 'custom';
906
+
907
+ if (args.repos?.length > 0) {
908
+ const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
909
+ const allFiles = [];
910
+ for (const repo of resolvedRepos) {
911
+ const scanner = new Scanner(repo.root, { policy: repo.policy });
912
+ for (const file of scanner.scan()) {
913
+ const relPosix = `${repo.alias}/${file.relativePath.split(path.sep).join('/')}`;
914
+ allFiles.push({
915
+ ...file,
916
+ relativePath: relPosix,
917
+ });
918
+ }
919
+ }
920
+ const analysis = await this.analyzer.analyze(allFiles);
921
+ const context = contextBuilder.build(analysis);
922
+ context.repos = resolvedRepos.map((r) => ({ alias: r.alias, root: r.root }));
923
+ const contextText = JSON.stringify(context, null, 2);
924
+ const exportedFiles = Object.values(context.files || {}).flat();
925
+ for (const repo of resolvedRepos) {
926
+ const prefix = `${repo.alias}/`;
927
+ const repoFiles = exportedFiles
928
+ .filter((f) => typeof f.path === 'string' && f.path.startsWith(prefix))
929
+ .map((f) => ({ ...f, path: f.path.slice(prefix.length) }));
930
+ guardExport(repo.policy, {
931
+ command: 'mcp generate_context',
932
+ root: repo.root,
933
+ destination: 'mcp',
934
+ format: 'json',
935
+ files: repoFiles,
936
+ text: contextText,
937
+ });
938
+ }
939
+ this.resourceProvider.cacheContext(templateName, contextText);
940
+ return {
941
+ content: [{ type: 'text', text: contextText }],
942
+ };
943
+ }
944
+
945
+ const scanner = new Scanner(projectPath);
946
+ const files = scanner.scan();
947
+ const analysis = await this.analyzer.analyze(files);
849
948
  const context = contextBuilder.build(analysis);
850
949
 
851
950
  // Cache context as a resource
852
- const templateName = args.template || 'custom';
853
951
  const contextText = JSON.stringify(context, null, 2);
854
952
  // Export policy: throws PolicyViolationError (an isError result) and logs the export
855
953
  guardExport(loadPolicy(projectPath), {
@@ -873,7 +971,6 @@ class MCPServer {
873
971
  }
874
972
  case 'git_diff': {
875
973
  const projectPath = args.path;
876
- const gitClient = new GitClient(projectPath);
877
974
  const branch = args.branch || 'main';
878
975
 
879
976
  if (!isValidGitRef(branch)) {
@@ -885,6 +982,47 @@ class MCPServer {
885
982
  };
886
983
  }
887
984
 
985
+ if (args.repos?.length > 0) {
986
+ const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
987
+ const diffs = [];
988
+ const errors = [];
989
+ for (const repo of resolvedRepos) {
990
+ try {
991
+ const gitClient = new GitClient(repo.root);
992
+ const rawDiff = gitClient.exec([
993
+ 'diff',
994
+ branch,
995
+ '--',
996
+ ...(this.root ? ['.'] : []),
997
+ ]);
998
+ if (rawDiff && rawDiff.trim()) {
999
+ const prefixed = rawDiff
1000
+ .replace(
1001
+ /^diff --git a\/(.+) b\/(.+)$/gm,
1002
+ `diff --git a/${repo.alias}/$1 b/${repo.alias}/$2`
1003
+ )
1004
+ .replace(/^--- a\/(.+)$/gm, `--- a/${repo.alias}/$1`)
1005
+ .replace(/^\+\+\+ b\/(.+)$/gm, `+++ b/${repo.alias}/$1`);
1006
+ diffs.push(prefixed.trimEnd());
1007
+ }
1008
+ } catch (e) {
1009
+ errors.push(`${repo.alias}: ${e.message}`);
1010
+ }
1011
+ }
1012
+ if (errors.length === resolvedRepos.length) {
1013
+ return {
1014
+ content: [{ type: 'text', text: `Error executing git diff: ${errors[0]}` }],
1015
+ isError: true,
1016
+ };
1017
+ }
1018
+ const notes = errors.map((err) => `# Note: ${err}`).join('\n');
1019
+ const combined = [diffs.join('\n'), notes].filter(Boolean).join('\n\n');
1020
+ return {
1021
+ content: [{ type: 'text', text: combined || 'No changes detected.' }],
1022
+ };
1023
+ }
1024
+
1025
+ const gitClient = new GitClient(projectPath);
888
1026
  let diff;
889
1027
  try {
890
1028
  // With a root, keep the diff to that directory even if the repository is larger
@@ -942,6 +1080,37 @@ class MCPServer {
942
1080
  };
943
1081
  }
944
1082
  }
1083
+
1084
+ if (args.repos?.length > 0) {
1085
+ const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
1086
+ const results = [];
1087
+ for (const repo of resolvedRepos) {
1088
+ const scanner = new Scanner(repo.root, { policy: repo.policy });
1089
+ for (const file of scanner.scan()) {
1090
+ try {
1091
+ const content = fs.readFileSync(file.path, 'utf-8');
1092
+ const lines = content.split('\n');
1093
+ const relPosix = `${repo.alias}/${file.relativePath.split(path.sep).join('/')}`;
1094
+ lines.forEach((line, index) => {
1095
+ const match = pattern ? pattern.test(line) : line.includes(query);
1096
+ if (match) {
1097
+ results.push({
1098
+ file: relPosix,
1099
+ line: index + 1,
1100
+ content: line.trim(),
1101
+ });
1102
+ }
1103
+ });
1104
+ } catch (_e) {
1105
+ // Skip read errors
1106
+ }
1107
+ }
1108
+ }
1109
+ return {
1110
+ content: [{ type: 'text', text: JSON.stringify(results, null, 2) }],
1111
+ };
1112
+ }
1113
+
945
1114
  const scanner = new Scanner(projectPath);
946
1115
  const files = scanner.scan();
947
1116
  const results = [];
@@ -979,6 +1148,48 @@ class MCPServer {
979
1148
  file = path.basename(projectPath);
980
1149
  projectPath = path.dirname(projectPath);
981
1150
  }
1151
+
1152
+ if (args.repos?.length > 0) {
1153
+ const resolvedRepos = resolveWorkspaceRepos([projectPath, ...args.repos]);
1154
+ const targetPosix = file ? file.split(path.sep).join('/').replace(/^\.\//, '') : null;
1155
+ const allFiles = [];
1156
+ for (const repo of resolvedRepos) {
1157
+ const scanner = new Scanner(repo.root, { policy: repo.policy });
1158
+ for (const f of scanner.scan()) {
1159
+ const barePosix = f.relativePath.split(path.sep).join('/');
1160
+ const relPosix = `${repo.alias}/${barePosix}`;
1161
+ if (!targetPosix || relPosix === targetPosix || barePosix === targetPosix) {
1162
+ allFiles.push({
1163
+ ...f,
1164
+ relativePath: relPosix,
1165
+ });
1166
+ }
1167
+ }
1168
+ }
1169
+ if (targetPosix && allFiles.length === 0) {
1170
+ return {
1171
+ content: [{ type: 'text', text: `File not found in project: ${file}` }],
1172
+ isError: true,
1173
+ };
1174
+ }
1175
+ const methodAnalyzer = new Analyzer({ methodLevel: true, cache: true });
1176
+ const analysis = await methodAnalyzer.analyze(allFiles);
1177
+ const methods = [];
1178
+ if (analysis.files) {
1179
+ for (const analyzedFile of analysis.files) {
1180
+ if (analyzedFile.methods && analyzedFile.methods.length > 0) {
1181
+ methods.push({
1182
+ file: analyzedFile.relativePath,
1183
+ methods: analyzedFile.methods,
1184
+ });
1185
+ }
1186
+ }
1187
+ }
1188
+ return {
1189
+ content: [{ type: 'text', text: JSON.stringify(methods, null, 2) }],
1190
+ };
1191
+ }
1192
+
982
1193
  const scanner = new Scanner(projectPath);
983
1194
  // Optional `file`: one project-relative file instead of the whole project
984
1195
  const target = file ? path.normalize(file) : null;
@@ -1060,6 +1271,7 @@ class MCPServer {
1060
1271
  const { largestFiles, ...summary } = await countForModel(args.path, {
1061
1272
  model: args.model,
1062
1273
  text: args.text,
1274
+ repos: args.repos,
1063
1275
  });
1064
1276
  const top = Number.isInteger(args.top) && args.top >= 0 ? args.top : 10;
1065
1277
  const result = { ...summary, largestFiles: largestFiles.slice(0, top) };
@@ -1,40 +1,73 @@
1
1
  /**
2
2
  * Token count for a target model
3
- * Counts a project (or a piped text) with the target model's tokenizer where one is
4
- * available offline (tiktoken for OpenAI models), and, on request, exactly through
5
- * Anthropic's count_tokens API for Claude models.
3
+ * Counts a project, multi-repository workspace, or piped text with the target model's
4
+ * tokenizer where one is available offline (tiktoken for OpenAI models), and, on request,
5
+ * exactly through Anthropic's count_tokens API for Claude models.
6
6
  */
7
7
 
8
8
  import fs from 'fs';
9
+ import path from 'path';
9
10
  import Scanner from '../core/Scanner.js';
10
11
  import TokenUtils from '../utils/token-utils.js';
11
12
  import { LLMDetector } from '../utils/llm-detector.js';
12
13
  import { ClaudeTokenCounter } from '../utils/claude-token-counter.js';
13
14
  import { redactSecrets } from '../utils/secret-redactor.js';
15
+ import { resolveWorkspaceRepos } from '../workspace/index.js';
14
16
 
15
17
  /**
16
- * Read the documents to count
18
+ * Read the documents to count from a single project or multi-repository workspace
17
19
  * @param {string} root - Project root (ignored when `text` is given)
18
20
  * @param {string} [text] - A single piped text instead of the project files
19
- * @returns {{path: string, content: string}[]}
21
+ * @param {Array<string|{root: string, alias?: string}>} [repos] - Additional repository roots
22
+ * @returns {{documents: {path: string, content: string, repo?: string}[], resolvedRepos: {root: string, alias: string}[] | null}}
20
23
  */
21
- function readDocuments(root, text) {
22
- if (text !== undefined) return [{ path: '<stdin>', content: text }];
23
- return new Scanner(root).scan().flatMap((file) => {
24
+ function readDocuments(root, text, repos) {
25
+ if (text !== undefined) {
26
+ return {
27
+ documents: [{ path: '<stdin>', content: text }],
28
+ resolvedRepos: null,
29
+ };
30
+ }
31
+
32
+ const extraRepos = Array.isArray(repos) ? repos.filter(Boolean) : [];
33
+ if (extraRepos.length > 0) {
34
+ const resolvedRepos = resolveWorkspaceRepos([root, ...extraRepos]);
35
+ const documents = [];
36
+ for (const repo of resolvedRepos) {
37
+ const scanned = new Scanner(repo.root, { policy: repo.policy }).scan();
38
+ for (const file of scanned) {
39
+ try {
40
+ const relPosix = file.relativePath.split(path.sep).join('/');
41
+ documents.push({
42
+ path: `${repo.alias}/${relPosix}`,
43
+ repo: repo.alias,
44
+ content: fs.readFileSync(file.path, 'utf8'),
45
+ });
46
+ } catch (_error) {
47
+ // Skip unreadable files
48
+ }
49
+ }
50
+ }
51
+ return { documents, resolvedRepos };
52
+ }
53
+
54
+ const documents = new Scanner(root).scan().flatMap((file) => {
24
55
  try {
25
56
  return [{ path: file.relativePath, content: fs.readFileSync(file.path, 'utf8') }];
26
57
  } catch (_error) {
27
58
  return [];
28
59
  }
29
60
  });
61
+ return { documents, resolvedRepos: null };
30
62
  }
31
63
 
32
64
  /**
33
- * Count tokens for a target model
65
+ * Count tokens for a target model across a project, multi-repo workspace, or piped text
34
66
  * @param {string} root - Project root
35
67
  * @param {object} [options]
36
68
  * @param {string} [options.model] - Target model id or alias (default: DEFAULT_TARGET_MODEL)
37
69
  * @param {string} [options.text] - Count this text instead of the project
70
+ * @param {Array<string|{root: string, alias?: string}>} [options.repos] - Additional workspace repository roots
38
71
  * @param {boolean} [options.api=false] - Count exactly with the Anthropic API (Claude only)
39
72
  * @param {object} [options.client] - Anthropic client stand-in (tests)
40
73
  * @param {(done: number, total: number) => void} [options.onProgress]
@@ -45,12 +78,13 @@ export async function countForModel(root, options = {}) {
45
78
  const profile = LLMDetector.getProfile(requested);
46
79
  const modelId = profile.id || requested;
47
80
  const encoding = TokenUtils.encodingFor(modelId);
48
- const documents = readDocuments(root, options.text);
81
+ const { documents, resolvedRepos } = readDocuments(root, options.text, options.repos);
49
82
 
50
83
  const files = documents
51
84
  .map((doc) => ({
52
85
  path: doc.path,
53
86
  tokens: TokenUtils.calculate(doc.content, doc.path, encoding),
87
+ ...(doc.repo ? { repo: doc.repo } : {}),
54
88
  }))
55
89
  .sort((a, b) => b.tokens - a.tokens);
56
90
  const localTokens = files.reduce((sum, file) => sum + file.tokens, 0);
@@ -67,7 +101,7 @@ export async function countForModel(root, options = {}) {
67
101
  // tiktoken matches OpenAI models whose encoding is published; others are approximations
68
102
  exact: TokenUtils.isExact() && profile.tokenizer?.exact === true,
69
103
  contextWindow: profile.contextWindow,
70
- largestFiles: files,
104
+ largestFiles: files.map(({ path: filePath, tokens }) => ({ path: filePath, tokens })),
71
105
  };
72
106
 
73
107
  if (options.api) {
@@ -89,11 +123,37 @@ export async function countForModel(root, options = {}) {
89
123
  result.method = 'anthropic-api';
90
124
  result.exact = true;
91
125
  result.requests = requests;
92
- result.ratio = localTokens > 0 ? total / localTokens : null;
126
+ result.ratio = localTokens > 0 ? total / localTokens : 1;
127
+ // The API counts batches, not single files; scale each file's local count by the
128
+ // observed ratio so the per-file breakdown matches the model too
129
+ result.largestFiles = files.map(({ path: filePath, tokens }) => ({
130
+ path: filePath,
131
+ tokens: Math.round(tokens * result.ratio),
132
+ }));
133
+ }
134
+
135
+ const window = result.contextWindow || 0;
136
+ result.fits = window > 0 ? result.tokens <= window : true;
137
+ result.percentOfWindow = window > 0 ? (result.tokens / window) * 100 : 0;
138
+
139
+ if (resolvedRepos) {
140
+ const ratio = result.method === 'anthropic-api' && result.ratio ? result.ratio : 1;
141
+ result.repos = resolvedRepos.map((repo) => {
142
+ const repoFiles = files.filter((f) => f.repo === repo.alias);
143
+ const repoLocalTokens = repoFiles.reduce((sum, f) => sum + f.tokens, 0);
144
+ const repoTokens =
145
+ result.method === 'anthropic-api' ? Math.round(repoLocalTokens * ratio) : repoLocalTokens;
146
+ return {
147
+ alias: repo.alias,
148
+ root: repo.root,
149
+ files: repoFiles.length,
150
+ localTokens: repoLocalTokens,
151
+ tokens: repoTokens,
152
+ percentOfWindow: window > 0 ? (repoTokens / window) * 100 : 0,
153
+ };
154
+ });
93
155
  }
94
156
 
95
- result.fits = result.tokens <= result.contextWindow;
96
- result.percentOfWindow = (result.tokens / result.contextWindow) * 100;
97
157
  return result;
98
158
  }
99
159
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ctxman",
3
- "version": "3.2.7",
3
+ "version": "3.2.8",
4
4
  "description": "AI Development Platform with plugin architecture, Git integration, REST API, watch mode, LLM optimization, and support for 14+ languages. Features include TOON format (40-50% token reduction), real-time analysis, and comprehensive context generation.",
5
5
  "type": "module",
6
6
  "main": "index.js",