ctxman 3.0.0 ā 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ctxman/llm-profiles.json +1067 -39
- package/.ctxman/profiles/backend-team.json +13 -0
- package/.ctxman/profiles/default.json +12 -0
- package/.ctxman/profiles/devops-team.json +21 -0
- package/.ctxman/profiles/frontend-team.json +20 -0
- package/.ctxman/templates/bug-fix.json +17 -0
- package/.ctxman/templates/code-review.json +9 -0
- package/.ctxman/templates/documentation.json +9 -0
- package/.ctxman/templates/feature.json +9 -0
- package/.ctxman/templates/full-context.json +9 -0
- package/.ctxman/templates/refactoring.json +18 -0
- package/.ctxman/templates/testing.json +9 -0
- package/.ctxman/wizard-profiles/security-audit/profile.json +7 -1
- package/LICENSE +21 -0
- package/README.md +380 -136
- package/bin/cli.js +1880 -529
- package/bin/cm-ask.js +18 -13
- package/bin/cm-audit.js +124 -0
- package/bin/cm-count.js +151 -0
- package/bin/cm-gitingest.js +128 -115
- package/bin/cm-map.js +242 -0
- package/bin/cm-mcp.js +293 -0
- package/bin/cm-pack.js +405 -0
- package/bin/cm-policy.js +170 -0
- package/bin/cm-pr.js +248 -0
- package/bin/cm-search.js +178 -0
- package/bin/cm-update.js +128 -128
- package/bin/cm-why.js +197 -0
- package/bin/mcp-server.js +8 -24
- package/ctxman.js +237 -183
- package/examples/README.md +19 -4
- package/examples/github-actions/ctxman-check.yml +49 -0
- package/examples/wizard-profiles/security-audit/profile.json +7 -1
- package/index.d.ts +1422 -0
- package/index.js +55 -31
- package/lib/agents/pr-failure-agent.js +48 -45
- package/lib/analyzers/ai-suggester.js +451 -0
- package/lib/analyzers/dependency-scanner.js +505 -0
- package/lib/analyzers/duplicate-detector.js +264 -0
- package/lib/analyzers/go-method-analyzer.js +151 -110
- package/lib/analyzers/import-tracker.js +431 -0
- package/lib/analyzers/method-analyzer.js +1055 -434
- package/lib/analyzers/performance-dashboard.js +683 -0
- package/lib/analyzers/token-calculator.js +895 -504
- package/lib/analyzers/unused-exports-detector.js +336 -0
- package/lib/api/mcp/install.js +424 -0
- package/lib/api/mcp/prompts.js +281 -281
- package/lib/api/mcp/resources.js +300 -241
- package/lib/api/mcp/sandbox.js +89 -0
- package/lib/api/mcp/server.js +675 -312
- package/lib/api/mcp/start.js +125 -0
- package/lib/api/mcp/transports/HttpSseTransport.js +60 -52
- package/lib/api/rest/server.js +105 -28
- package/lib/cache/CacheManager.js +9 -17
- package/lib/core/Analyzer.js +16 -10
- package/lib/core/ContextBuilder.js +19 -28
- package/lib/core/Reporter.js +15 -14
- package/lib/core/Scanner.js +81 -12
- package/lib/count/index.js +100 -0
- package/lib/formatters/format-registry.js +394 -372
- package/lib/formatters/gitingest-formatter.js +656 -621
- package/lib/formatters/toon-formatter-v1.3.js +637 -632
- package/lib/formatters/toon-formatter.js +239 -240
- package/lib/integrations/git/BlameTracker.js +4 -4
- package/lib/integrations/git/DiffAnalyzer.js +38 -31
- package/lib/integrations/git/GitClient.js +71 -53
- package/lib/lsp/server.js +311 -0
- package/lib/map/graph.js +299 -0
- package/lib/map/imports.js +233 -0
- package/lib/map/index.js +33 -0
- package/lib/map/rank.js +253 -0
- package/lib/map/render.js +346 -0
- package/lib/map/resolver.js +574 -0
- package/lib/pack/budget.js +111 -0
- package/lib/pack/code-lexer.js +360 -0
- package/lib/pack/compress.js +414 -0
- package/lib/pack/index.js +354 -0
- package/lib/pack/renderers/common.js +134 -0
- package/lib/pack/renderers/index.js +24 -0
- package/lib/pack/renderers/json.js +47 -0
- package/lib/pack/renderers/markdown.js +54 -0
- package/lib/pack/renderers/plain.js +38 -0
- package/lib/pack/renderers/xml.js +85 -0
- package/lib/pack/select.js +177 -0
- package/lib/pack/split.js +182 -0
- package/lib/pack/tree.js +35 -0
- package/lib/parsers/gitignore-parser.js +113 -107
- package/lib/parsers/method-filter-parser.js +50 -33
- package/lib/plugins/ExporterPlugin.js +3 -3
- package/lib/plugins/LanguagePlugin.js +3 -3
- package/lib/plugins/PluginManager.js +7 -13
- package/lib/policy/audit.js +294 -0
- package/lib/policy/enforce.js +125 -0
- package/lib/policy/ignore-files.js +169 -0
- package/lib/policy/index.js +52 -0
- package/lib/policy/policy.js +489 -0
- package/lib/pr/diff-parser.js +267 -0
- package/lib/pr/method-mapper.js +148 -0
- package/lib/pr/pr-context.js +829 -0
- package/lib/pr/pr-git.js +149 -0
- package/lib/pr/pr-renderer.js +644 -0
- package/lib/pr/related-tests.js +296 -0
- package/lib/rag/DataSourcePlugin.js +5 -5
- package/lib/rag/EmbeddingProvider.js +19 -8
- package/lib/rag/Indexer.js +14 -5
- package/lib/rag/VectorStoreFactory.js +3 -3
- package/lib/rag/sources/FileDataSource.js +6 -6
- package/lib/rag/stores/BaseVectorStore.js +5 -5
- package/lib/rag/stores/ChromaDBStore.js +10 -10
- package/lib/rag/stores/LanceDBStore.js +29 -19
- package/lib/rag/stores/PineconeStore.js +11 -9
- package/lib/search/bm25.js +433 -0
- package/lib/search/code-units.js +201 -0
- package/lib/search/symbols.js +189 -0
- package/lib/ui/dashboard.js +120 -89
- package/lib/ui/index.js +1 -6
- package/lib/ui/init-wizard-ui.js +550 -0
- package/lib/ui/progress-bar.js +94 -88
- package/lib/ui/select-input.js +40 -32
- package/lib/ui/template-selector.js +99 -0
- package/lib/ui/wizard.js +374 -329
- package/lib/utils/claude-token-counter.js +151 -0
- package/lib/utils/cli-output.js +17 -0
- package/lib/utils/clipboard-utils.js +49 -50
- package/lib/utils/config-utils.js +81 -66
- package/lib/utils/context-versioning.js +532 -0
- package/lib/utils/error-handler.js +112 -112
- package/lib/utils/file-utils.js +153 -69
- package/lib/utils/format-converter.js +356 -283
- package/lib/utils/git-utils.js +366 -294
- package/lib/utils/llm-cost-estimator.js +281 -0
- package/lib/utils/llm-detector.js +242 -78
- package/lib/utils/logger.js +243 -237
- package/lib/utils/messages.js +355 -0
- package/lib/utils/multi-repo-manager.js +364 -0
- package/lib/utils/profile-manager.js +380 -0
- package/lib/utils/safe-read.js +61 -0
- package/lib/utils/secret-redactor.js +61 -0
- package/lib/utils/snapshot-manager.js +679 -0
- package/lib/utils/template-manager.js +378 -0
- package/lib/utils/token-utils.js +201 -170
- package/lib/utils/tokenizer-adapter.js +421 -377
- package/lib/utils/updater.js +539 -506
- package/lib/watch/FileWatcher.js +12 -8
- package/lib/watch/IncrementalAnalyzer.js +7 -7
- package/lib/why/coupling.js +81 -0
- package/lib/why/errors.js +23 -0
- package/lib/why/git.js +169 -0
- package/lib/why/history.js +108 -0
- package/lib/why/index.js +720 -0
- package/lib/why/ownership.js +79 -0
- package/lib/why/paths.js +207 -0
- package/lib/why/references.js +259 -0
- package/lib/why/render.js +722 -0
- package/lib/why/search.js +358 -0
- package/lib/why/target.js +389 -0
- package/lib/wizards/config-generator.js +255 -0
- package/lib/wizards/project-detector.js +571 -0
- package/package.json +44 -10
- package/.ctxman/logs/ctxman-2026-07-02.log +0 -156
- package/.ctxman/self-knowledge/documents.lance/_transactions/0-b3dd1558-f95b-44b9-a154-18e8c330ae7a.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/1-44bff6d4-e552-4010-a769-e09ad822698d.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/10-37260794-5947-4ecb-b4c8-d732f58f40f1.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/11-0d52a883-5518-463c-9b6d-986c3af89dfd.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/12-ae354646-52f2-49c9-90fb-2e6ed8932663.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/13-5d7d5e57-27da-498b-b734-091b9d38da13.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/14-558ce928-810b-4252-a364-fdd1ee6f8a25.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/15-27e3a58f-8b64-43b8-a9f4-e3369446968b.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/16-a110c390-e5c8-4464-b976-bc7f5195fa15.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/17-97513225-53de-4216-b61a-6337bd0cc4ad.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/18-5ba450f0-c527-459c-bf4f-6d79b5e4285f.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/19-600d3a63-5225-49e9-b8b7-30e037b564ab.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/2-48a61b4e-92cf-4342-8376-914c1b99a9f9.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/20-121dd912-5410-410e-84ff-82c7a23604a0.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/3-bc6844ab-7ea3-4bb3-9ee1-538ce5664a9c.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/4-d5e1754f-dbb5-454e-a9be-2a7da8d7bcf9.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/5-4fae93df-0568-4b6f-8acf-8bef4bba6047.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/6-4b1b8b87-068e-49a3-ba2b-d8dae628b252.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/7-63d040fb-d2fa-4c0f-bed0-53411918a7b0.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/8-c5628cf6-264c-420f-aab8-bc96a9083591.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/9-c94570aa-5e53-4fcb-971f-cecc1d1d5492.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/1.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/10.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/11.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/12.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/13.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/14.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/15.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/16.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/17.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/18.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/19.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/2.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/20.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/21.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/3.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/4.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/5.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/6.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/7.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/8.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/9.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/000010010011011101000010de8621405d9d5cb06540dbf02c.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/0000101001011100011000003fb62a4aa8978974075c144f1c.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/000011010000100111001011c6cc824d628f3bb1f4e04cd5d0.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/001000001000011001011100d8f74349ada090b332cd1df669.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/01000001111010001000010045ab44467bb1e7e4d8ae06bd58.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/01000101101110111110011131ef904241a89e1ffb6ddaf8e3.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/010010110000010000011010fbfcc145c6a8a827b28057a07e.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1000011000101010001110003fb4f045c3835982efa1405324.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/100111011101110111101011f4396b410fa06af3b028d204e6.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/100111100111101000001100596b174e00995d6cc2969cc8dc.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1010001100011000110010102ea16b484d887c7c9725feffd6.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/101101100010101111110111c5230944d9b3af7301b6392dda.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1011011100100111111110105892bf41b5a530fedd7cf516f1.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/10111110000010001011110164e8774742bade87b483c10f36.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1100000100010110001110103ffaca40a2a42c85067ebaf9ae.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/110000010001110110101110664bde4c76a78dfc14b05e83b8.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/11001100111000010100110050645c4281b1f35b2022234f7f.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1101111011001100110111017214924a368f5719eab2246619.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1110000111001010001000002150c24dcab653d39d07032e63.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1110100000000100000011014d20534967a18b5c87721659ff.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/11110101111010000101011135a0114ef2b35dc6f1297a3381.lance +0 -0
|
@@ -1,577 +1,968 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
-
import { execSync } from 'child_process';
|
|
4
3
|
import readline from 'readline';
|
|
5
4
|
import TokenUtils from '../utils/token-utils.js';
|
|
6
5
|
import FileUtils from '../utils/file-utils.js';
|
|
7
6
|
import ClipboardUtils from '../utils/clipboard-utils.js';
|
|
8
7
|
import ConfigUtils from '../utils/config-utils.js';
|
|
9
|
-
import GitIgnoreParser from '../parsers/gitignore-parser.js';
|
|
10
8
|
import MethodAnalyzer from './method-analyzer.js';
|
|
11
|
-
import MethodFilterParser from '../parsers/method-filter-parser.js';
|
|
12
9
|
import GitIngestFormatter from '../formatters/gitingest-formatter.js';
|
|
10
|
+
import FormatRegistry from '../formatters/format-registry.js';
|
|
11
|
+
import { formatRedactionSummary, redactSecrets } from '../utils/secret-redactor.js';
|
|
13
12
|
import { LLMDetector } from '../utils/llm-detector.js';
|
|
13
|
+
import { POLICY_FILE } from '../policy/policy.js';
|
|
14
|
+
import { enforceExport, PolicyViolationError } from '../policy/enforce.js';
|
|
15
|
+
import { recordExport } from '../policy/audit.js';
|
|
16
|
+
import { loadIgnoreFileRules, resolveIgnoreFileNames } from '../policy/ignore-files.js';
|
|
17
|
+
|
|
18
|
+
// What ctxman itself writes into a project: reports, context exports and digests in the root,
|
|
19
|
+
// and its state under .ctxman/. A later run must not count them as project files.
|
|
20
|
+
const OWN_OUTPUT_PATHS = [
|
|
21
|
+
/^token-analysis-report\.json$/,
|
|
22
|
+
/^llm-context\.\w+$/,
|
|
23
|
+
/^digest(-chunk-\d+-of-\d+)?\.txt$/,
|
|
24
|
+
/^(ai-suggestions-report|dependency-context|restored-context)\.json$/,
|
|
25
|
+
/^performance-report\.txt$/,
|
|
26
|
+
/^\.ctxman[/\\]audit\.log$/,
|
|
27
|
+
/^\.ctxman[/\\](versions|snapshots|metrics|logs|temp|self-knowledge|rag-store)$/,
|
|
28
|
+
];
|
|
14
29
|
|
|
15
30
|
class TokenCalculator {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
31
|
+
constructor(projectRoot, options = {}) {
|
|
32
|
+
this.projectRoot = projectRoot;
|
|
33
|
+
this.options = { verbose: false, compactContext: true, methodLevel: false, ...options };
|
|
34
|
+
this.stats = this.initStats();
|
|
35
|
+
this.gitIgnore = this.initGitIgnore();
|
|
36
|
+
this.patternFilters = this.initPatternFilters();
|
|
37
|
+
this.methodAnalyzer = new MethodAnalyzer();
|
|
38
|
+
this.methodFilter = this.options.methodLevel ? this.initMethodFilter() : null;
|
|
39
|
+
this.methodStats = { totalMethods: 0, includedMethods: 0, methodTokens: {} };
|
|
40
|
+
// o200k_base for OpenAI models since GPT-4o, cl100k_base otherwise
|
|
41
|
+
this.encoding = TokenUtils.encodingFor(this.options.targetModel);
|
|
42
|
+
|
|
43
|
+
// Export policy from loadPolicy() (lib/policy): deny rules, export limits, audit log.
|
|
44
|
+
// Without one, scanning and exporting behave as before.
|
|
45
|
+
this.policy = this.options.policy?.active ? this.options.policy : null;
|
|
46
|
+
this.ignoreFileRules = this.initIgnoreFileRules();
|
|
47
|
+
if (this.policy) this.stats.policyDeniedFiles = 0;
|
|
48
|
+
if (this.ignoreFileRules) this.stats.ignoreFileIgnoredFiles = 0;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* AI-tool ignore files (.cursorignore, .aiexclude, ...) from --respect-ignore-files and
|
|
53
|
+
* the policy's ignoreFiles
|
|
54
|
+
* @returns {import('../policy/ignore-files.js').IgnoreFileRules|null}
|
|
55
|
+
*/
|
|
56
|
+
initIgnoreFileRules() {
|
|
57
|
+
const names = [
|
|
58
|
+
...new Set([
|
|
59
|
+
...resolveIgnoreFileNames(this.options.respectIgnoreFiles),
|
|
60
|
+
...(this.policy?.ignoreFiles ?? []),
|
|
61
|
+
]),
|
|
62
|
+
];
|
|
63
|
+
return names.length > 0 ? loadIgnoreFileRules(this.projectRoot, names) : null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
initMethodFilter() {
|
|
67
|
+
return ConfigUtils.initMethodFilter(this.projectRoot);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
initStats() {
|
|
71
|
+
return {
|
|
72
|
+
totalFiles: 0,
|
|
73
|
+
totalTokens: 0,
|
|
74
|
+
totalBytes: 0,
|
|
75
|
+
totalLines: 0,
|
|
76
|
+
ignoredFiles: 0,
|
|
77
|
+
calculatorIgnoredFiles: 0,
|
|
78
|
+
byExtension: {},
|
|
79
|
+
byDirectory: {},
|
|
80
|
+
largestFiles: [],
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
initGitIgnore() {
|
|
85
|
+
return ConfigUtils.initGitIgnore(this.projectRoot);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Include/exclude patterns of --template and --profile, with .contextinclude /
|
|
90
|
+
* .contextignore syntax. A file must pass every active set.
|
|
91
|
+
*/
|
|
92
|
+
initPatternFilters() {
|
|
93
|
+
const toRegex = (pattern) => this.gitIgnore.convertToRegex(pattern);
|
|
94
|
+
return [this.options.templateConfig, this.options.profileConfig]
|
|
95
|
+
.filter(Boolean)
|
|
96
|
+
.map((config) => ({
|
|
97
|
+
include: (config.includePatterns || []).map(toRegex),
|
|
98
|
+
exclude: (config.ignorePatterns || []).map(toRegex),
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
passesPatternFilters(relativePath) {
|
|
103
|
+
return this.patternFilters.every(({ include, exclude }) => {
|
|
104
|
+
if (include.length > 0 && !this.gitIgnore.testPatternsWithNegation(include, relativePath)) {
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
// Last matching pattern wins, as in .contextignore
|
|
108
|
+
let excluded = false;
|
|
109
|
+
for (const pattern of exclude) {
|
|
110
|
+
if (pattern.regex.test(relativePath)) excluded = !pattern.isNegation;
|
|
111
|
+
}
|
|
112
|
+
return !excluded;
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
calculateTokens(content, filePath) {
|
|
117
|
+
return TokenUtils.calculate(content, filePath, this.encoding);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
isTextFile(filePath) {
|
|
121
|
+
return FileUtils.isText(filePath);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
analyzeFile(filePath) {
|
|
125
|
+
try {
|
|
126
|
+
const content = fs.readFileSync(filePath, 'utf8');
|
|
127
|
+
const stats = fs.statSync(filePath);
|
|
128
|
+
|
|
129
|
+
const fileInfo = {
|
|
130
|
+
path: filePath,
|
|
131
|
+
relativePath: path.relative(this.projectRoot, filePath),
|
|
132
|
+
sizeBytes: stats.size,
|
|
133
|
+
tokens: this.calculateTokens(content, filePath),
|
|
134
|
+
lines: content.split('\n').length,
|
|
135
|
+
extension: path.extname(filePath).toLowerCase() || 'no-extension',
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
// Method-level analysis if enabled
|
|
139
|
+
if (this.options.methodLevel && this.isCodeFile(filePath)) {
|
|
140
|
+
fileInfo.methods = this.analyzeFileMethods(content, filePath);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return fileInfo;
|
|
144
|
+
} catch (error) {
|
|
145
|
+
return {
|
|
146
|
+
path: filePath,
|
|
147
|
+
relativePath: path.relative(this.projectRoot, filePath),
|
|
148
|
+
sizeBytes: 0,
|
|
149
|
+
tokens: 0,
|
|
150
|
+
lines: 0,
|
|
151
|
+
extension: 'error',
|
|
152
|
+
error: error.message,
|
|
153
|
+
};
|
|
24
154
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
isCodeFile(filePath) {
|
|
158
|
+
return FileUtils.isCode(filePath);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
analyzeFileMethods(content, filePath) {
|
|
162
|
+
const methods = this.methodAnalyzer.extractMethods(content, filePath);
|
|
163
|
+
const sources = this.methodAnalyzer.extractMethodSources(content, methods, filePath);
|
|
164
|
+
const fileName = path.basename(filePath, path.extname(filePath));
|
|
165
|
+
const filteredMethods = [];
|
|
166
|
+
|
|
167
|
+
this.methodStats.totalMethods += methods.length;
|
|
168
|
+
|
|
169
|
+
methods.forEach((method, i) => {
|
|
170
|
+
if (!this.methodFilter || this.methodFilter.shouldIncludeMethod(method.name, fileName)) {
|
|
171
|
+
// `content` keeps the narrower legacy extraction that reports have always carried
|
|
172
|
+
const methodContent = this.methodAnalyzer.extractMethodContent(content, method.name);
|
|
173
|
+
const methodTokens = this.calculateTokens(sources[i], filePath);
|
|
174
|
+
|
|
175
|
+
const methodInfo = {
|
|
176
|
+
...method,
|
|
177
|
+
tokens: methodTokens,
|
|
178
|
+
content: methodContent,
|
|
35
179
|
};
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
initGitIgnore() {
|
|
39
|
-
return ConfigUtils.initGitIgnore(this.projectRoot);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
calculateTokens(content, filePath) {
|
|
43
|
-
const tokens = TokenUtils.calculate(content, filePath);
|
|
44
|
-
return tokens;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
isTextFile(filePath) {
|
|
48
|
-
return FileUtils.isText(filePath);
|
|
49
|
-
}
|
|
50
180
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
const fileInfo = {
|
|
57
|
-
path: filePath,
|
|
58
|
-
relativePath: path.relative(this.projectRoot, filePath),
|
|
59
|
-
sizeBytes: stats.size,
|
|
60
|
-
tokens: this.calculateTokens(content, filePath),
|
|
61
|
-
lines: content.split('\n').length,
|
|
62
|
-
extension: path.extname(filePath).toLowerCase() || 'no-extension'
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
// Method-level analysis if enabled
|
|
66
|
-
if (this.options.methodLevel && this.isCodeFile(filePath)) {
|
|
67
|
-
fileInfo.methods = this.analyzeFileMethods(content, filePath);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
return fileInfo;
|
|
71
|
-
} catch (error) {
|
|
72
|
-
return {
|
|
73
|
-
path: filePath,
|
|
74
|
-
relativePath: path.relative(this.projectRoot, filePath),
|
|
75
|
-
sizeBytes: 0, tokens: 0, lines: 0,
|
|
76
|
-
extension: 'error', error: error.message
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
isCodeFile(filePath) {
|
|
82
|
-
return FileUtils.isCode(filePath);
|
|
83
|
-
}
|
|
181
|
+
filteredMethods.push(methodInfo);
|
|
182
|
+
this.methodStats.includedMethods++;
|
|
183
|
+
this.methodStats.methodTokens[`${fileName}.${method.name}`] = methodTokens;
|
|
184
|
+
}
|
|
185
|
+
});
|
|
84
186
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const fileName = path.basename(filePath, path.extname(filePath));
|
|
88
|
-
const filteredMethods = [];
|
|
187
|
+
return filteredMethods;
|
|
188
|
+
}
|
|
89
189
|
|
|
90
|
-
|
|
190
|
+
scanDirectory(dir) {
|
|
191
|
+
const files = [];
|
|
91
192
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
193
|
+
try {
|
|
194
|
+
for (const item of fs.readdirSync(dir)) {
|
|
195
|
+
const fullPath = path.join(dir, item);
|
|
196
|
+
const relativePath = path.relative(this.projectRoot, fullPath);
|
|
96
197
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
content: methodContent
|
|
101
|
-
};
|
|
102
|
-
|
|
103
|
-
filteredMethods.push(methodInfo);
|
|
104
|
-
this.methodStats.includedMethods++;
|
|
105
|
-
this.methodStats.methodTokens[`${fileName}.${method.name}`] = methodTokens;
|
|
106
|
-
}
|
|
198
|
+
if (this.gitIgnore.isIgnored(fullPath, relativePath)) {
|
|
199
|
+
this.countIgnoredFiles(fullPath);
|
|
200
|
+
continue;
|
|
107
201
|
}
|
|
108
202
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
scanDirectory(dir) {
|
|
113
|
-
const files = [];
|
|
114
|
-
|
|
115
|
-
try {
|
|
116
|
-
for (const item of fs.readdirSync(dir)) {
|
|
117
|
-
const fullPath = path.join(dir, item);
|
|
118
|
-
const relativePath = path.relative(this.projectRoot, fullPath);
|
|
119
|
-
|
|
120
|
-
if (this.gitIgnore.isIgnored(fullPath, relativePath)) {
|
|
121
|
-
this.countIgnoredFiles(fullPath);
|
|
122
|
-
continue;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
const stat = fs.statSync(fullPath);
|
|
126
|
-
if (stat.isDirectory()) {
|
|
127
|
-
if (!['node_modules', '.git', '.svn', '.hg', 'coverage', 'dist', 'build'].includes(item)) {
|
|
128
|
-
files.push(...this.scanDirectory(fullPath));
|
|
129
|
-
}
|
|
130
|
-
} else if (this.isTextFile(fullPath)) {
|
|
131
|
-
files.push(fullPath);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
} catch (error) {
|
|
135
|
-
console.error(`Error scanning directory ${dir}:`, error.message);
|
|
203
|
+
if (this.isOwnOutput(relativePath)) {
|
|
204
|
+
continue;
|
|
136
205
|
}
|
|
137
206
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
const isCalculatorIgnored = ['calculator', 'calculator-include'].includes(
|
|
143
|
-
this.gitIgnore._lastIgnoreReason
|
|
144
|
-
);
|
|
145
|
-
|
|
146
|
-
try {
|
|
147
|
-
const stat = fs.statSync(filePath);
|
|
148
|
-
if (stat.isDirectory()) {
|
|
149
|
-
const count = this.countFilesInDirectory(filePath);
|
|
150
|
-
if (isCalculatorIgnored) {
|
|
151
|
-
this.stats.calculatorIgnoredFiles += count;
|
|
152
|
-
} else {
|
|
153
|
-
this.stats.ignoredFiles += count;
|
|
154
|
-
}
|
|
155
|
-
} else {
|
|
156
|
-
if (isCalculatorIgnored) {
|
|
157
|
-
this.stats.calculatorIgnoredFiles++;
|
|
158
|
-
} else {
|
|
159
|
-
this.stats.ignoredFiles++;
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
} catch { }
|
|
163
|
-
}
|
|
207
|
+
if (this.ignoreFileRules?.isIgnored(relativePath)) {
|
|
208
|
+
this.stats.ignoreFileIgnoredFiles += this.countMatchedFiles(fullPath);
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
164
211
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
212
|
+
const stat = fs.statSync(fullPath);
|
|
213
|
+
// Template/profile patterns select files; directories are always traversed
|
|
214
|
+
if (!stat.isDirectory() && !this.passesPatternFilters(relativePath)) {
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
if (stat.isDirectory()) {
|
|
218
|
+
if (!FileUtils.isSkippedDirectory(item)) {
|
|
219
|
+
files.push(...this.scanDirectory(fullPath));
|
|
220
|
+
}
|
|
221
|
+
} else if (this.isTextFile(fullPath) && this.passesFileFilter(relativePath)) {
|
|
222
|
+
// Denied by the policy: left out of the analysis and every export
|
|
223
|
+
if (this.policy?.isDenied(relativePath)) {
|
|
224
|
+
this.stats.policyDeniedFiles++;
|
|
225
|
+
} else {
|
|
226
|
+
files.push(fullPath);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
} catch (error) {
|
|
231
|
+
console.error(`Error scanning directory ${dir}:`, error.message);
|
|
180
232
|
}
|
|
181
233
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
234
|
+
return files;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Optional caller-supplied filter (e.g. --changed-only) on project-relative paths
|
|
239
|
+
* @param {string} relativePath
|
|
240
|
+
* @returns {boolean}
|
|
241
|
+
*/
|
|
242
|
+
passesFileFilter(relativePath) {
|
|
243
|
+
return typeof this.options.fileFilter !== 'function' || this.options.fileFilter(relativePath);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* @param {string} relativePath - Path relative to the project root
|
|
248
|
+
* @returns {boolean} True for a report, export, digest or state folder written by ctxman
|
|
249
|
+
*/
|
|
250
|
+
isOwnOutput(relativePath) {
|
|
251
|
+
return (
|
|
252
|
+
OWN_OUTPUT_PATHS.some((pattern) => pattern.test(relativePath)) ||
|
|
253
|
+
Boolean(this.policy?.isAuditLog(relativePath))
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* @param {string} fullPath - File or directory matched by an ignore rule
|
|
259
|
+
* @returns {number} Files it stands for
|
|
260
|
+
*/
|
|
261
|
+
countMatchedFiles(fullPath) {
|
|
262
|
+
try {
|
|
263
|
+
return fs.statSync(fullPath).isDirectory() ? this.countFilesInDirectory(fullPath) : 1;
|
|
264
|
+
} catch {
|
|
265
|
+
return 0;
|
|
207
266
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
includedMethods: this.methodStats.includedMethods,
|
|
223
|
-
totalMethodTokens: Object.values(this.methodStats.methodTokens).reduce((sum, tokens) => sum + tokens, 0)
|
|
224
|
-
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
countIgnoredFiles(filePath) {
|
|
270
|
+
// Reasons set by GitIgnoreParser for .contextignore / .contextinclude matches
|
|
271
|
+
const isCalculatorIgnored = ['context', 'context-include'].includes(
|
|
272
|
+
this.gitIgnore._lastIgnoreReason
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
try {
|
|
276
|
+
const stat = fs.statSync(filePath);
|
|
277
|
+
if (stat.isDirectory()) {
|
|
278
|
+
const count = this.countFilesInDirectory(filePath);
|
|
279
|
+
if (isCalculatorIgnored) {
|
|
280
|
+
this.stats.calculatorIgnoredFiles += count;
|
|
225
281
|
} else {
|
|
226
|
-
|
|
282
|
+
this.stats.ignoredFiles += count;
|
|
227
283
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
const methodContext = {};
|
|
234
|
-
|
|
235
|
-
for (const fileInfo of analysisResults) {
|
|
236
|
-
if (fileInfo.methods && fileInfo.methods.length > 0) {
|
|
237
|
-
methodContext[fileInfo.relativePath] = fileInfo.methods.map(method => ({
|
|
238
|
-
name: method.name,
|
|
239
|
-
line: method.line,
|
|
240
|
-
tokens: method.tokens
|
|
241
|
-
}));
|
|
242
|
-
}
|
|
284
|
+
} else {
|
|
285
|
+
if (isCalculatorIgnored) {
|
|
286
|
+
this.stats.calculatorIgnoredFiles++;
|
|
287
|
+
} else {
|
|
288
|
+
this.stats.ignoredFiles++;
|
|
243
289
|
}
|
|
244
|
-
|
|
245
|
-
|
|
290
|
+
}
|
|
291
|
+
} catch {}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
countFilesInDirectory(dirPath) {
|
|
295
|
+
let count = 0;
|
|
296
|
+
try {
|
|
297
|
+
for (const item of fs.readdirSync(dirPath)) {
|
|
298
|
+
const fullPath = path.join(dirPath, item);
|
|
299
|
+
const stat = fs.statSync(fullPath);
|
|
300
|
+
|
|
301
|
+
if (stat.isDirectory()) {
|
|
302
|
+
count += this.countFilesInDirectory(fullPath);
|
|
303
|
+
} else if (this.isTextFile(fullPath)) {
|
|
304
|
+
count++;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
} catch {}
|
|
308
|
+
return count;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
updateStats(fileInfo) {
|
|
312
|
+
if (fileInfo.error) return;
|
|
313
|
+
|
|
314
|
+
const { extension, tokens, sizeBytes, lines } = fileInfo;
|
|
315
|
+
|
|
316
|
+
this.stats.totalFiles++;
|
|
317
|
+
this.stats.totalTokens += tokens;
|
|
318
|
+
this.stats.totalBytes += sizeBytes;
|
|
319
|
+
this.stats.totalLines += lines;
|
|
320
|
+
this.stats.largestFiles.push(fileInfo);
|
|
321
|
+
|
|
322
|
+
// Update extension stats
|
|
323
|
+
this.stats.byExtension[extension] = this.stats.byExtension[extension] || {
|
|
324
|
+
count: 0,
|
|
325
|
+
tokens: 0,
|
|
326
|
+
bytes: 0,
|
|
327
|
+
lines: 0,
|
|
328
|
+
};
|
|
329
|
+
const extStats = this.stats.byExtension[extension];
|
|
330
|
+
extStats.count++;
|
|
331
|
+
extStats.tokens += tokens;
|
|
332
|
+
extStats.bytes += sizeBytes;
|
|
333
|
+
extStats.lines += lines;
|
|
334
|
+
|
|
335
|
+
// Update directory stats
|
|
336
|
+
// Files in the project root share '.', instead of each counting as a directory
|
|
337
|
+
const parts = fileInfo.relativePath.split(/[\\/]/);
|
|
338
|
+
const topDir = parts.length > 1 ? parts[0] : '.';
|
|
339
|
+
this.stats.byDirectory[topDir] = this.stats.byDirectory[topDir] || {
|
|
340
|
+
count: 0,
|
|
341
|
+
tokens: 0,
|
|
342
|
+
bytes: 0,
|
|
343
|
+
lines: 0,
|
|
344
|
+
};
|
|
345
|
+
const dirStats = this.stats.byDirectory[topDir];
|
|
346
|
+
dirStats.count++;
|
|
347
|
+
dirStats.tokens += tokens;
|
|
348
|
+
dirStats.bytes += sizeBytes;
|
|
349
|
+
dirStats.lines += lines;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
generateLLMContext(analysisResults) {
|
|
353
|
+
const context = {
|
|
354
|
+
project: {
|
|
355
|
+
root: path.basename(this.projectRoot),
|
|
356
|
+
totalFiles: this.stats.totalFiles,
|
|
357
|
+
totalTokens: this.stats.totalTokens,
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
if (this.options.methodLevel) {
|
|
362
|
+
context.methods = this.generateMethodContext(analysisResults.files || analysisResults);
|
|
363
|
+
context.methodStats = {
|
|
364
|
+
totalMethods: this.methodStats.totalMethods,
|
|
365
|
+
includedMethods: this.methodStats.includedMethods,
|
|
366
|
+
totalMethodTokens: Object.values(this.methodStats.methodTokens).reduce(
|
|
367
|
+
(sum, tokens) => sum + tokens,
|
|
368
|
+
0
|
|
369
|
+
),
|
|
370
|
+
};
|
|
371
|
+
} else {
|
|
372
|
+
context.paths = this.generateCompactPaths(analysisResults.files || analysisResults);
|
|
246
373
|
}
|
|
247
374
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
const pathGroups = {};
|
|
251
|
-
|
|
252
|
-
sortedFiles.forEach(file => {
|
|
253
|
-
const filePath = file.relativePath;
|
|
254
|
-
const parts = filePath.split('/');
|
|
255
|
-
|
|
256
|
-
// Group files by their directory path
|
|
257
|
-
// If file is in root, use '/' as key, otherwise use directory path with trailing slash
|
|
258
|
-
const categoryPath = parts.length > 1 ? parts.slice(0, -1).join('/') + '/' : '/';
|
|
375
|
+
return context;
|
|
376
|
+
}
|
|
259
377
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
}
|
|
378
|
+
generateMethodContext(analysisResults) {
|
|
379
|
+
const methodContext = {};
|
|
263
380
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
381
|
+
for (const fileInfo of analysisResults) {
|
|
382
|
+
if (fileInfo.methods && fileInfo.methods.length > 0) {
|
|
383
|
+
methodContext[fileInfo.relativePath] = fileInfo.methods.map((method) => ({
|
|
384
|
+
name: method.name,
|
|
385
|
+
line: method.line,
|
|
386
|
+
tokens: method.tokens,
|
|
387
|
+
}));
|
|
388
|
+
}
|
|
269
389
|
}
|
|
270
390
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
391
|
+
return methodContext;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
generateCompactPaths(analysisResults) {
|
|
395
|
+
const sortedFiles = analysisResults.sort((a, b) => b.tokens - a.tokens);
|
|
396
|
+
const pathGroups = {};
|
|
397
|
+
|
|
398
|
+
sortedFiles.forEach((file) => {
|
|
399
|
+
const filePath = file.relativePath;
|
|
400
|
+
const parts = filePath.split('/');
|
|
401
|
+
|
|
402
|
+
// Group files by their directory path
|
|
403
|
+
// If file is in root, use '/' as key, otherwise use directory path with trailing slash
|
|
404
|
+
const categoryPath = parts.length > 1 ? parts.slice(0, -1).join('/') + '/' : '/';
|
|
405
|
+
|
|
406
|
+
if (!pathGroups[categoryPath]) {
|
|
407
|
+
pathGroups[categoryPath] = [];
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const filename = parts[parts.length - 1];
|
|
411
|
+
pathGroups[categoryPath].push(filename);
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
return pathGroups;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Format chosen with -o/--output for context exports (JSON when not set)
|
|
419
|
+
* @returns {string}
|
|
420
|
+
*/
|
|
421
|
+
getContextFormat() {
|
|
422
|
+
const format = this.options.outputFormat;
|
|
423
|
+
return format && format !== 'gitingest' ? format : 'json';
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Serialize the LLM context in the selected output format
|
|
428
|
+
* @param {object} context
|
|
429
|
+
* @returns {string}
|
|
430
|
+
*/
|
|
431
|
+
formatContext(context) {
|
|
432
|
+
const format = this.getContextFormat();
|
|
433
|
+
if (format === 'json') return JSON.stringify(context, null, 2);
|
|
434
|
+
return new FormatRegistry().encode(format, context);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
exportContextToClipboard(context) {
|
|
438
|
+
const contextString = this.formatContext(context);
|
|
439
|
+
const success = ClipboardUtils.copy(contextString);
|
|
440
|
+
|
|
441
|
+
if (success) {
|
|
442
|
+
this.recordAudit('cli --context-clipboard', {
|
|
443
|
+
destination: 'clipboard',
|
|
444
|
+
format: this.getContextFormat(),
|
|
445
|
+
text: contextString,
|
|
446
|
+
});
|
|
447
|
+
} else {
|
|
448
|
+
console.log('š” Clipboard unavailable, saving to a file instead');
|
|
449
|
+
this.saveContextToFile(context);
|
|
279
450
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
saveContextToFile(context) {
|
|
454
|
+
const format = this.getContextFormat();
|
|
455
|
+
const extension = format === 'json' ? '.json' : new FormatRegistry().getInfo(format).extension;
|
|
456
|
+
const contextPath = path.join(this.projectRoot, `llm-context${extension}`);
|
|
457
|
+
const text = this.formatContext(context);
|
|
458
|
+
fs.writeFileSync(contextPath, text);
|
|
459
|
+
console.log(`š¾ Context saved to: ${contextPath}`);
|
|
460
|
+
this.recordAudit('cli --context-export', { destination: contextPath, format, text });
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
promptForExport(analysisResults) {
|
|
464
|
+
console.log('\nš¤ Export Options:');
|
|
465
|
+
console.log('1) Save detailed JSON report (token-analysis-report.json)');
|
|
466
|
+
console.log('2) Generate LLM context file (llm-context.json)');
|
|
467
|
+
console.log('3) Copy LLM context to clipboard');
|
|
468
|
+
console.log('4) Generate GitIngest digest (digest.txt)');
|
|
469
|
+
console.log('5) No export (skip)');
|
|
470
|
+
|
|
471
|
+
const rl = readline.createInterface({
|
|
472
|
+
input: process.stdin,
|
|
473
|
+
output: process.stdout,
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
rl.question('š¤ Which export option would you like? (1-5): ', (answer) => {
|
|
477
|
+
const trimmed = answer.trim();
|
|
478
|
+
|
|
479
|
+
// Handle empty input (just Enter pressed) - default to option 5 (skip)
|
|
480
|
+
if (trimmed === '') {
|
|
481
|
+
console.log('\nā
Analysis complete. No export selected.');
|
|
482
|
+
rl.close();
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
const choice = parseInt(trimmed);
|
|
487
|
+
|
|
488
|
+
// Validate input is a number between 1-5
|
|
489
|
+
if (isNaN(choice) || choice < 1 || choice > 5) {
|
|
490
|
+
console.log('\nā Invalid option. Please enter a number between 1-5.');
|
|
491
|
+
console.log(' Skipping export.');
|
|
492
|
+
rl.close();
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
const requested = ['--save-report', '--context-export', '--context-clipboard', '--gitingest'];
|
|
497
|
+
if (choice !== 5 && !this.checkExportPolicy(analysisResults, requested[choice - 1])) {
|
|
498
|
+
rl.close();
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
switch (choice) {
|
|
503
|
+
case 1:
|
|
504
|
+
console.log('\nš¾ Saving detailed JSON report...');
|
|
505
|
+
this.saveDetailedReport(analysisResults);
|
|
506
|
+
break;
|
|
507
|
+
case 2:
|
|
508
|
+
console.log('\nš¤ Generating LLM context file...');
|
|
509
|
+
const context = this.generateLLMContext(analysisResults);
|
|
510
|
+
this.saveContextToFile(context);
|
|
511
|
+
break;
|
|
512
|
+
case 3:
|
|
513
|
+
console.log('\nš¤ Generating LLM context for clipboard...');
|
|
514
|
+
const clipboardContext = this.generateLLMContext(analysisResults);
|
|
515
|
+
this.exportContextToClipboard(clipboardContext);
|
|
516
|
+
break;
|
|
517
|
+
case 4:
|
|
518
|
+
console.log('\nš Generating GitIngest digest...');
|
|
519
|
+
this.saveGitIngestDigest(analysisResults);
|
|
520
|
+
break;
|
|
521
|
+
case 5:
|
|
522
|
+
console.log('\nā
Analysis complete. No export selected.');
|
|
523
|
+
break;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
rl.close();
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
saveGitIngestDigest(analysisResults) {
|
|
531
|
+
const formatter = new GitIngestFormatter(this.projectRoot, this.stats, analysisResults, {
|
|
532
|
+
redactSecrets: this.options.redactSecrets !== false,
|
|
533
|
+
chunking: this.options.chunking,
|
|
534
|
+
});
|
|
535
|
+
const digestPath = path.join(this.projectRoot, 'digest.txt');
|
|
536
|
+
const digestSize = formatter.saveToFile(digestPath);
|
|
537
|
+
|
|
538
|
+
if (formatter.chunkFiles?.length) {
|
|
539
|
+
console.log(`š¾ GitIngest digest saved as ${formatter.chunkFiles.length} chunks:`);
|
|
540
|
+
formatter.chunkFiles.forEach((file) => console.log(` ${path.basename(file)}`));
|
|
541
|
+
} else {
|
|
542
|
+
console.log(`š¾ GitIngest digest saved to: digest.txt`);
|
|
285
543
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
output: process.stdout
|
|
298
|
-
});
|
|
299
|
-
|
|
300
|
-
rl.question('š¤ Which export option would you like? (1-5): ', (answer) => {
|
|
301
|
-
const trimmed = answer.trim();
|
|
302
|
-
|
|
303
|
-
// Handle empty input (just Enter pressed) - default to option 5 (skip)
|
|
304
|
-
if (trimmed === '') {
|
|
305
|
-
console.log('\nā
Analysis complete. No export selected.');
|
|
306
|
-
rl.close();
|
|
307
|
-
return;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
const choice = parseInt(trimmed);
|
|
311
|
-
|
|
312
|
-
// Validate input is a number between 1-5
|
|
313
|
-
if (isNaN(choice) || choice < 1 || choice > 5) {
|
|
314
|
-
console.log('\nā Invalid option. Please enter a number between 1-5.');
|
|
315
|
-
console.log(' Skipping export.');
|
|
316
|
-
rl.close();
|
|
317
|
-
return;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
switch (choice) {
|
|
321
|
-
case 1:
|
|
322
|
-
console.log('\nš¾ Saving detailed JSON report...');
|
|
323
|
-
this.saveDetailedReport(analysisResults);
|
|
324
|
-
break;
|
|
325
|
-
case 2:
|
|
326
|
-
console.log('\nš¤ Generating LLM context file...');
|
|
327
|
-
const context = this.generateLLMContext(analysisResults);
|
|
328
|
-
this.saveContextToFile(context);
|
|
329
|
-
console.log('š¾ LLM context saved to: llm-context.json');
|
|
330
|
-
break;
|
|
331
|
-
case 3:
|
|
332
|
-
console.log('\nš¤ Generating LLM context for clipboard...');
|
|
333
|
-
const clipboardContext = this.generateLLMContext(analysisResults);
|
|
334
|
-
this.exportContextToClipboard(clipboardContext);
|
|
335
|
-
break;
|
|
336
|
-
case 4:
|
|
337
|
-
console.log('\nš Generating GitIngest digest...');
|
|
338
|
-
this.saveGitIngestDigest(analysisResults);
|
|
339
|
-
break;
|
|
340
|
-
case 5:
|
|
341
|
-
console.log('\nā
Analysis complete. No export selected.');
|
|
342
|
-
break;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
rl.close();
|
|
346
|
-
});
|
|
544
|
+
console.log(`š Digest size: ${(digestSize / 1024).toFixed(1)} KB`);
|
|
545
|
+
if (this.policy?.audit.enabled) {
|
|
546
|
+
const written = formatter.chunkFiles?.length ? formatter.chunkFiles : [digestPath];
|
|
547
|
+
this.recordAudit('cli --gitingest', {
|
|
548
|
+
destination: digestPath,
|
|
549
|
+
outputFiles: written.length > 1 ? written : undefined,
|
|
550
|
+
format: 'gitingest',
|
|
551
|
+
redactions: formatter.redactions,
|
|
552
|
+
// Hash what was written: the chunks in order
|
|
553
|
+
text: written.map((file) => fs.readFileSync(file)),
|
|
554
|
+
});
|
|
347
555
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
556
|
+
const redactions = formatter.redactions ?? {};
|
|
557
|
+
if (Object.keys(redactions).length > 0) {
|
|
558
|
+
console.log(
|
|
559
|
+
`š Redacted secrets: ${formatRedactionSummary(redactions)} (disable with --no-redact)`
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Scan and analyze the project without printing or exporting
|
|
566
|
+
* @returns {Array<object>} Per-file analysis results
|
|
567
|
+
*/
|
|
568
|
+
analyze() {
|
|
569
|
+
return this.analyzeFiles(this.scanDirectory(this.projectRoot));
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
analyzeFiles(files) {
|
|
573
|
+
const analysisResults = [];
|
|
574
|
+
for (const file of files) {
|
|
575
|
+
const fileInfo = this.analyzeFile(file);
|
|
576
|
+
this.updateStats(fileInfo);
|
|
577
|
+
analysisResults.push(fileInfo);
|
|
356
578
|
}
|
|
357
579
|
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
this.printScanResults(allFiles);
|
|
363
|
-
|
|
364
|
-
// Analyze all files
|
|
365
|
-
const analysisResults = [];
|
|
366
|
-
for (const file of allFiles) {
|
|
367
|
-
const fileInfo = this.analyzeFile(file);
|
|
368
|
-
this.updateStats(fileInfo);
|
|
369
|
-
analysisResults.push(fileInfo);
|
|
370
|
-
}
|
|
580
|
+
this.stats.largestFiles.sort((a, b) => b.tokens - a.tokens);
|
|
581
|
+
this.lastAnalysisResults = analysisResults;
|
|
582
|
+
return analysisResults;
|
|
583
|
+
}
|
|
371
584
|
|
|
372
|
-
|
|
373
|
-
|
|
585
|
+
run() {
|
|
586
|
+
this.printHeader();
|
|
374
587
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
this.printContextFitAnalysis();
|
|
378
|
-
}
|
|
588
|
+
const allFiles = this.scanDirectory(this.projectRoot);
|
|
589
|
+
this.printScanResults(allFiles);
|
|
379
590
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
this.handleExports(analysisResults);
|
|
383
|
-
}
|
|
591
|
+
const analysisResults = this.analyzeFiles(allFiles);
|
|
592
|
+
this.printReport();
|
|
384
593
|
|
|
385
|
-
|
|
386
|
-
|
|
594
|
+
// Context Fit Analysis (v2.3.7)
|
|
595
|
+
if (this.options.targetModel && !this.options.dashboard) {
|
|
596
|
+
this.printContextFitAnalysis();
|
|
387
597
|
}
|
|
388
598
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
const configPath = this.gitIgnore.hasIncludeFile ?
|
|
394
|
-
ConfigUtils.findConfigFile(this.projectRoot, '.contextinclude') :
|
|
395
|
-
ConfigUtils.findConfigFile(this.projectRoot, '.contextignore');
|
|
599
|
+
// Handle exports (skip for dashboard mode)
|
|
600
|
+
if (!this.options.dashboard) {
|
|
601
|
+
this.handleExports(analysisResults);
|
|
602
|
+
}
|
|
396
603
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
console.log(' Location: ' + path.relative(this.projectRoot, configPath));
|
|
401
|
-
}
|
|
604
|
+
// Return stats for programmatic usage (e.g., Dashboard)
|
|
605
|
+
return this.stats;
|
|
606
|
+
}
|
|
402
607
|
|
|
403
|
-
|
|
608
|
+
printHeader() {
|
|
609
|
+
console.log('š Analyzing project: ' + this.projectRoot);
|
|
610
|
+
console.log('š Respecting .gitignore rules...');
|
|
404
611
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
const hasInclude = this.methodFilter.hasIncludeFile;
|
|
409
|
-
const mode = hasInclude ? 'INCLUDE' : 'EXCLUDE';
|
|
410
|
-
console.log(` Method filtering: ${mode} mode active`);
|
|
411
|
-
}
|
|
412
|
-
}
|
|
612
|
+
const configPath = this.gitIgnore.hasIncludeFile
|
|
613
|
+
? ConfigUtils.findConfigFile(this.projectRoot, '.contextinclude')
|
|
614
|
+
: ConfigUtils.findConfigFile(this.projectRoot, '.contextignore');
|
|
413
615
|
|
|
414
|
-
|
|
616
|
+
if (configPath) {
|
|
617
|
+
const mode = this.gitIgnore.hasIncludeFile ? 'INCLUDE' : 'EXCLUDE';
|
|
618
|
+
console.log(`š
Found calculator config - using ${mode} mode`);
|
|
619
|
+
console.log(' Location: ' + path.relative(this.projectRoot, configPath));
|
|
415
620
|
}
|
|
416
621
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
if (this.options.methodLevel) {
|
|
426
|
-
console.log(`š§ Total methods found: ${this.methodStats.totalMethods}`);
|
|
427
|
-
console.log(`ā
Methods included: ${this.methodStats.includedMethods}`);
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
console.log();
|
|
622
|
+
if (this.policy) {
|
|
623
|
+
const files = this.policy.sources.map(({ scope, file }) =>
|
|
624
|
+
scope === 'project' ? POLICY_FILE : file
|
|
625
|
+
);
|
|
626
|
+
console.log(`š”ļø Export policy: ${files.join(', ')}`);
|
|
627
|
+
}
|
|
628
|
+
if (this.ignoreFileRules) {
|
|
629
|
+
console.log(`š§© AI-tool ignore files: ${this.ignoreFileRules.describe()}`);
|
|
431
630
|
}
|
|
432
631
|
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
632
|
+
console.log(
|
|
633
|
+
'šÆ Token calculation: ' +
|
|
634
|
+
(TokenUtils.hasExactCounting()
|
|
635
|
+
? `ā
Exact (using tiktoken) Ā· ${this.encoding}`
|
|
636
|
+
: 'ā ļø Estimated')
|
|
637
|
+
);
|
|
638
|
+
const target = this.options.targetModel
|
|
639
|
+
? LLMDetector.getProfile(this.options.targetModel)
|
|
640
|
+
: null;
|
|
641
|
+
if (target?.tokenizer && !target.tokenizer.exact && TokenUtils.hasExactCounting()) {
|
|
642
|
+
const exactly =
|
|
643
|
+
target.tokenizer.exactVia === 'anthropic-api'
|
|
644
|
+
? ` (exact count: ctxman count --model ${target.id} --api)`
|
|
645
|
+
: '';
|
|
646
|
+
console.log(
|
|
647
|
+
` Counts for ${target.name} are approximate: it uses its own tokenizer${exactly}`
|
|
648
|
+
);
|
|
649
|
+
}
|
|
451
650
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
651
|
+
if (this.options.methodLevel) {
|
|
652
|
+
console.log('š§ Method-level analysis: ā
Enabled');
|
|
653
|
+
if (this.methodFilter) {
|
|
654
|
+
const hasInclude = this.methodFilter.hasIncludeFile;
|
|
655
|
+
const mode = hasInclude ? 'INCLUDE' : 'EXCLUDE';
|
|
656
|
+
console.log(` Method filtering: ${mode} mode active`);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
455
659
|
|
|
456
|
-
|
|
660
|
+
console.log();
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
printScanResults(allFiles) {
|
|
664
|
+
console.log(`š Found ${allFiles.length} text files to analyze`);
|
|
665
|
+
console.log(`š« Ignored ${this.stats.ignoredFiles} files due to .gitignore rules`);
|
|
666
|
+
if (this.stats.calculatorIgnoredFiles > 0) {
|
|
667
|
+
const rules = this.gitIgnore.hasIncludeFile ? '.contextinclude' : '.contextignore';
|
|
668
|
+
console.log(
|
|
669
|
+
`š Filtered ${this.stats.calculatorIgnoredFiles} additional files due to ${rules} rules`
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
if (this.stats.ignoreFileIgnoredFiles > 0) {
|
|
673
|
+
console.log(
|
|
674
|
+
`š§© Filtered ${this.stats.ignoreFileIgnoredFiles} files due to AI-tool ignore files`
|
|
675
|
+
);
|
|
676
|
+
}
|
|
677
|
+
if (this.stats.policyDeniedFiles > 0) {
|
|
678
|
+
const count = this.stats.policyDeniedFiles;
|
|
679
|
+
console.log(
|
|
680
|
+
`š”ļø Removed ${count} file${count === 1 ? '' : 's'} denied by the export policy (never exported)`
|
|
681
|
+
);
|
|
457
682
|
}
|
|
458
683
|
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
console.log('Extension'.padEnd(15) + 'Files'.padStart(8) + 'Tokens'.padStart(12) + 'Size (KB)'.padStart(12) + 'Lines'.padStart(10));
|
|
463
|
-
console.log('-'.repeat(80));
|
|
464
|
-
|
|
465
|
-
Object.entries(this.stats.byExtension)
|
|
466
|
-
.sort((a, b) => b[1].tokens - a[1].tokens)
|
|
467
|
-
.forEach(([ext, data]) => {
|
|
468
|
-
console.log(
|
|
469
|
-
ext.padEnd(15) +
|
|
470
|
-
data.count.toString().padStart(8) +
|
|
471
|
-
data.tokens.toLocaleString().padStart(12) +
|
|
472
|
-
(data.bytes / 1024).toFixed(1).padStart(12) +
|
|
473
|
-
data.lines.toLocaleString().padStart(10)
|
|
474
|
-
);
|
|
475
|
-
});
|
|
684
|
+
if (this.options.methodLevel) {
|
|
685
|
+
console.log(`š§ Total methods found: ${this.methodStats.totalMethods}`);
|
|
686
|
+
console.log(`ā
Methods included: ${this.methodStats.includedMethods}`);
|
|
476
687
|
}
|
|
477
688
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
689
|
+
console.log();
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
printReport() {
|
|
693
|
+
console.log('='.repeat(80));
|
|
694
|
+
console.log('šÆ PROJECT TOKEN ANALYSIS REPORT');
|
|
695
|
+
console.log('='.repeat(80));
|
|
696
|
+
console.log(`š Total files analyzed: ${this.stats.totalFiles.toLocaleString()}`);
|
|
697
|
+
console.log(`š¢ Total tokens: ${this.stats.totalTokens.toLocaleString()}`);
|
|
698
|
+
console.log(`š¾ Total size: ${(this.stats.totalBytes / 1024 / 1024).toFixed(2)} MB`);
|
|
699
|
+
console.log(`š Total lines: ${this.stats.totalLines.toLocaleString()}`);
|
|
700
|
+
|
|
701
|
+
const avgTokens =
|
|
702
|
+
this.stats.totalFiles > 0
|
|
703
|
+
? Math.round(this.stats.totalTokens / this.stats.totalFiles).toLocaleString()
|
|
704
|
+
: 'N/A';
|
|
705
|
+
console.log(`š Average tokens per file: ${avgTokens}`);
|
|
706
|
+
console.log(`š« Files ignored by .gitignore: ${this.stats.ignoredFiles.toLocaleString()}`);
|
|
707
|
+
if (this.stats.calculatorIgnoredFiles > 0) {
|
|
708
|
+
console.log(
|
|
709
|
+
`š Files ignored by context rules: ${this.stats.calculatorIgnoredFiles.toLocaleString()}`
|
|
710
|
+
);
|
|
487
711
|
}
|
|
712
|
+
console.log();
|
|
488
713
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
console.log('\nš TOP 5 LARGEST DIRECTORIES BY TOKEN COUNT:');
|
|
495
|
-
console.log('-'.repeat(80));
|
|
496
|
-
largestDirectories.slice(0, 5).forEach((dir, index) => {
|
|
497
|
-
const percentage = ((dir.tokens / this.stats.totalTokens) * 100).toFixed(1);
|
|
498
|
-
const avgTokensPerFile = Math.round(dir.tokens / dir.count);
|
|
499
|
-
console.log(`${(index + 1).toString().padStart(2)}. ${dir.tokens.toLocaleString().padStart(8)} tokens (${percentage}%) - ${dir.directory}/`);
|
|
500
|
-
console.log(` š ${dir.count.toString().padStart(3)} files, š ${avgTokensPerFile.toLocaleString().padStart(6)} avg tokens/file`);
|
|
501
|
-
});
|
|
714
|
+
this.printExtensionStats();
|
|
715
|
+
this.printTopFiles();
|
|
716
|
+
this.printTopDirectories();
|
|
717
|
+
if (this.options.verbose) {
|
|
718
|
+
this.printIncludedFiles();
|
|
502
719
|
}
|
|
503
720
|
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
721
|
+
console.log('\nš” Tip: Use --save-report flag to save detailed analysis to JSON file');
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
printExtensionStats() {
|
|
725
|
+
console.log('š BY FILE TYPE:');
|
|
726
|
+
console.log('-'.repeat(80));
|
|
727
|
+
console.log(
|
|
728
|
+
'Extension'.padEnd(15) +
|
|
729
|
+
'Files'.padStart(8) +
|
|
730
|
+
'Tokens'.padStart(12) +
|
|
731
|
+
'Size (KB)'.padStart(12) +
|
|
732
|
+
'Lines'.padStart(10)
|
|
733
|
+
);
|
|
734
|
+
console.log('-'.repeat(80));
|
|
735
|
+
|
|
736
|
+
Object.entries(this.stats.byExtension)
|
|
737
|
+
.sort((a, b) => b[1].tokens - a[1].tokens)
|
|
738
|
+
.forEach(([ext, data]) => {
|
|
739
|
+
console.log(
|
|
740
|
+
ext.padEnd(15) +
|
|
741
|
+
data.count.toString().padStart(8) +
|
|
742
|
+
data.tokens.toLocaleString().padStart(12) +
|
|
743
|
+
(data.bytes / 1024).toFixed(1).padStart(12) +
|
|
744
|
+
data.lines.toLocaleString().padStart(10)
|
|
745
|
+
);
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
printTopFiles() {
|
|
750
|
+
console.log('\nš TOP 5 LARGEST FILES BY TOKEN COUNT:');
|
|
751
|
+
console.log('-'.repeat(80));
|
|
752
|
+
this.stats.largestFiles.slice(0, 5).forEach((file, index) => {
|
|
753
|
+
const percentage = ((file.tokens / this.stats.totalTokens) * 100).toFixed(1);
|
|
754
|
+
const sizeKB = (file.sizeBytes / 1024).toFixed(1);
|
|
755
|
+
console.log(
|
|
756
|
+
`${(index + 1).toString().padStart(2)}. ${file.tokens.toLocaleString().padStart(8)} tokens (${percentage}%) - ${file.relativePath}`
|
|
757
|
+
);
|
|
758
|
+
console.log(
|
|
759
|
+
` š¾ ${sizeKB.padStart(8)} KB, š ${file.lines.toLocaleString().padStart(6)} lines`
|
|
760
|
+
);
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
printTopDirectories() {
|
|
765
|
+
const largestDirectories = Object.entries(this.stats.byDirectory)
|
|
766
|
+
.map(([dir, stats]) => ({ directory: dir, ...stats }))
|
|
767
|
+
.sort((a, b) => b.tokens - a.tokens);
|
|
768
|
+
|
|
769
|
+
console.log('\nš TOP 5 LARGEST DIRECTORIES BY TOKEN COUNT:');
|
|
770
|
+
console.log('-'.repeat(80));
|
|
771
|
+
largestDirectories.slice(0, 5).forEach((dir, index) => {
|
|
772
|
+
const percentage = ((dir.tokens / this.stats.totalTokens) * 100).toFixed(1);
|
|
773
|
+
const avgTokensPerFile = Math.round(dir.tokens / dir.count);
|
|
774
|
+
console.log(
|
|
775
|
+
`${(index + 1).toString().padStart(2)}. ${dir.tokens.toLocaleString().padStart(8)} tokens (${percentage}%) - ${dir.directory}/`
|
|
776
|
+
);
|
|
777
|
+
console.log(
|
|
778
|
+
` š ${dir.count.toString().padStart(3)} files, š ${avgTokensPerFile.toLocaleString().padStart(6)} avg tokens/file`
|
|
779
|
+
);
|
|
780
|
+
});
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
printIncludedFiles() {
|
|
784
|
+
console.log(`\nš INCLUDED FILES (${this.stats.largestFiles.length}):`);
|
|
785
|
+
console.log('-'.repeat(80));
|
|
786
|
+
[...this.stats.largestFiles]
|
|
787
|
+
.sort((a, b) => a.relativePath.localeCompare(b.relativePath))
|
|
788
|
+
.forEach((file) => {
|
|
789
|
+
console.log(`${file.tokens.toLocaleString().padStart(8)} tokens - ${file.relativePath}`);
|
|
790
|
+
});
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
handleExports(analysisResults) {
|
|
794
|
+
const { saveReport, contextExport, contextToClipboard } = this.options;
|
|
795
|
+
// -o gitingest and --chunk both mean "produce the GitIngest digest"
|
|
796
|
+
const gitingest =
|
|
797
|
+
this.options.gitingest ||
|
|
798
|
+
this.options.outputFormat === 'gitingest' ||
|
|
799
|
+
Boolean(this.options.chunking?.enabled);
|
|
800
|
+
|
|
801
|
+
if (contextExport || contextToClipboard || saveReport || gitingest) {
|
|
802
|
+
const requested = [
|
|
803
|
+
contextToClipboard ? '--context-clipboard' : contextExport && '--context-export',
|
|
804
|
+
saveReport && '--save-report',
|
|
805
|
+
gitingest && '--gitingest',
|
|
806
|
+
].filter(Boolean);
|
|
807
|
+
if (!this.checkExportPolicy(analysisResults, requested.join(' '))) return;
|
|
808
|
+
|
|
809
|
+
if (contextExport || contextToClipboard) {
|
|
810
|
+
console.log('\nš¤ Generating LLM Context...');
|
|
811
|
+
const context = this.generateLLMContext(analysisResults);
|
|
812
|
+
|
|
813
|
+
if (contextToClipboard) {
|
|
814
|
+
this.exportContextToClipboard(context);
|
|
528
815
|
} else {
|
|
529
|
-
|
|
816
|
+
this.saveContextToFile(context);
|
|
530
817
|
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
if (saveReport) {
|
|
821
|
+
this.saveDetailedReport(analysisResults);
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
if (gitingest) {
|
|
825
|
+
console.log('\nš Generating GitIngest digest...');
|
|
826
|
+
this.saveGitIngestDigest(analysisResults);
|
|
827
|
+
}
|
|
828
|
+
} else if (!this.options.simple) {
|
|
829
|
+
this.promptForExport(analysisResults);
|
|
531
830
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
saveDetailedReport(analysisResults) {
|
|
834
|
+
const files = analysisResults.sort((a, b) => b.tokens - a.tokens);
|
|
835
|
+
// Method-level reports carry method source; a policy that requires redaction masks it
|
|
836
|
+
const redactor = this.policy?.requireRedaction ? this.createMethodRedactor() : null;
|
|
837
|
+
const report = {
|
|
838
|
+
metadata: {
|
|
839
|
+
generatedAt: new Date().toISOString(),
|
|
840
|
+
projectRoot: this.projectRoot,
|
|
841
|
+
gitignoreRules: this.gitIgnore.patterns.map((p) => p.original),
|
|
842
|
+
calculatorRules: this.gitIgnore.contextPatterns.map((p) => p.original),
|
|
843
|
+
},
|
|
844
|
+
summary: redactor
|
|
845
|
+
? { ...this.stats, largestFiles: this.stats.largestFiles.map(redactor.file) }
|
|
846
|
+
: this.stats,
|
|
847
|
+
files: redactor ? files.map(redactor.file) : files,
|
|
848
|
+
};
|
|
849
|
+
|
|
850
|
+
const reportPath = path.join(this.projectRoot, 'token-analysis-report.json');
|
|
851
|
+
const text = JSON.stringify(report, null, 2);
|
|
852
|
+
fs.writeFileSync(reportPath, text);
|
|
853
|
+
console.log('š¾ Detailed analysis saved to: token-analysis-report.json');
|
|
854
|
+
this.recordAudit('cli --save-report', {
|
|
855
|
+
destination: reportPath,
|
|
856
|
+
format: 'json-report',
|
|
857
|
+
redactions: redactor?.count() ?? 0,
|
|
858
|
+
text,
|
|
859
|
+
});
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/**
|
|
863
|
+
* Masks secrets in the method source of analysis results. A result listed twice in the
|
|
864
|
+
* report (files and summary.largestFiles) is copied and counted once.
|
|
865
|
+
* @returns {{file: (entry: object) => object, count: () => number}}
|
|
866
|
+
*/
|
|
867
|
+
createMethodRedactor() {
|
|
868
|
+
const copies = new Map();
|
|
869
|
+
let count = 0;
|
|
870
|
+
const redactMethod = (method) => {
|
|
871
|
+
if (typeof method?.content !== 'string') return method;
|
|
872
|
+
const { text, findings } = redactSecrets(method.content);
|
|
873
|
+
count += findings.length;
|
|
874
|
+
return { ...method, content: text };
|
|
875
|
+
};
|
|
876
|
+
const file = (entry) => {
|
|
877
|
+
if (!Array.isArray(entry?.methods)) return entry;
|
|
878
|
+
if (!copies.has(entry)) {
|
|
879
|
+
copies.set(entry, { ...entry, methods: entry.methods.map(redactMethod) });
|
|
880
|
+
}
|
|
881
|
+
return copies.get(entry);
|
|
882
|
+
};
|
|
883
|
+
return { file, count: () => count };
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/**
|
|
887
|
+
* Check the requested exports against the export policy. A refused export is printed,
|
|
888
|
+
* logged with outcome "refused" and sets exit code 3.
|
|
889
|
+
* @param {Array<object>} analysisResults
|
|
890
|
+
* @param {string} [requested] - The export flags, for the audit log
|
|
891
|
+
* @returns {boolean} True when the exports may go ahead
|
|
892
|
+
*/
|
|
893
|
+
checkExportPolicy(analysisResults, requested = 'export') {
|
|
894
|
+
if (!this.policy) return true;
|
|
895
|
+
|
|
896
|
+
try {
|
|
897
|
+
enforceExport(this.policy, {
|
|
898
|
+
files: analysisResults,
|
|
899
|
+
tokens: this.stats.totalTokens,
|
|
900
|
+
redact: this.options.redactSecrets !== false,
|
|
901
|
+
targetModel: this.options.targetModel || null,
|
|
902
|
+
});
|
|
903
|
+
return true;
|
|
904
|
+
} catch (error) {
|
|
905
|
+
if (!(error instanceof PolicyViolationError)) throw error;
|
|
906
|
+
console.error(`\nā ${error.message}`);
|
|
907
|
+
recordExport(this.policy, {
|
|
908
|
+
...this.auditEvent(`cli ${requested}`),
|
|
909
|
+
outcome: 'refused',
|
|
910
|
+
violations: error.violations,
|
|
911
|
+
});
|
|
912
|
+
process.exitCode = error.exitCode;
|
|
913
|
+
return false;
|
|
548
914
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/**
|
|
918
|
+
* Audit event for an export of the current analysis
|
|
919
|
+
* @param {string} command
|
|
920
|
+
* @param {object} [extra] - destination, format, text, redactions, ...
|
|
921
|
+
* @returns {import('../policy/audit.js').AuditEvent}
|
|
922
|
+
*/
|
|
923
|
+
auditEvent(command, extra = {}) {
|
|
924
|
+
return {
|
|
925
|
+
command,
|
|
926
|
+
root: this.projectRoot,
|
|
927
|
+
targetModel: this.options.targetModel || null,
|
|
928
|
+
files: this.lastAnalysisResults ?? [],
|
|
929
|
+
tokens: this.stats.totalTokens,
|
|
930
|
+
removedByPolicy: this.stats.policyDeniedFiles ?? 0,
|
|
931
|
+
...extra,
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* Append an export to the policy's audit log (no-op without a policy that enables it)
|
|
937
|
+
* @param {string} command
|
|
938
|
+
* @param {object} extra
|
|
939
|
+
*/
|
|
940
|
+
recordAudit(command, extra) {
|
|
941
|
+
if (this.policy?.audit.enabled) recordExport(this.policy, this.auditEvent(command, extra));
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
/**
|
|
945
|
+
* Print Context Fit Analysis (v2.3.7)
|
|
946
|
+
* Shows if repository fits in target LLM's context window
|
|
947
|
+
*/
|
|
948
|
+
printContextFitAnalysis() {
|
|
949
|
+
try {
|
|
950
|
+
const profile = LLMDetector.getProfile(this.options.targetModel);
|
|
951
|
+
const repoStats = {
|
|
952
|
+
totalTokens: this.stats.totalTokens,
|
|
953
|
+
totalFiles: this.stats.totalFiles,
|
|
954
|
+
};
|
|
955
|
+
|
|
956
|
+
// Pass largest files for suggestions
|
|
957
|
+
const analysis = LLMDetector.analyzeContextFit(profile, repoStats, this.stats.largestFiles);
|
|
958
|
+
const formatted = LLMDetector.formatAnalysis(analysis);
|
|
959
|
+
|
|
960
|
+
console.log(formatted);
|
|
961
|
+
} catch (_error) {
|
|
962
|
+
// Silently fail if context fit analysis fails
|
|
963
|
+
// Don't break the main workflow
|
|
571
964
|
}
|
|
965
|
+
}
|
|
572
966
|
}
|
|
573
967
|
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
968
|
export default TokenCalculator;
|