ctxman 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ctxman/llm-profiles.json +1067 -39
- package/.ctxman/profiles/backend-team.json +13 -0
- package/.ctxman/profiles/default.json +12 -0
- package/.ctxman/profiles/devops-team.json +21 -0
- package/.ctxman/profiles/frontend-team.json +20 -0
- package/.ctxman/templates/bug-fix.json +17 -0
- package/.ctxman/templates/code-review.json +9 -0
- package/.ctxman/templates/documentation.json +9 -0
- package/.ctxman/templates/feature.json +9 -0
- package/.ctxman/templates/full-context.json +9 -0
- package/.ctxman/templates/refactoring.json +18 -0
- package/.ctxman/templates/testing.json +9 -0
- package/.ctxman/wizard-profiles/security-audit/profile.json +7 -1
- package/LICENSE +21 -0
- package/README.md +380 -136
- package/bin/cli.js +1880 -529
- package/bin/cm-ask.js +18 -13
- package/bin/cm-audit.js +124 -0
- package/bin/cm-count.js +151 -0
- package/bin/cm-gitingest.js +128 -115
- package/bin/cm-map.js +242 -0
- package/bin/cm-mcp.js +293 -0
- package/bin/cm-pack.js +405 -0
- package/bin/cm-policy.js +170 -0
- package/bin/cm-pr.js +248 -0
- package/bin/cm-search.js +178 -0
- package/bin/cm-update.js +128 -128
- package/bin/cm-why.js +197 -0
- package/bin/mcp-server.js +8 -24
- package/ctxman.js +237 -183
- package/examples/README.md +19 -4
- package/examples/github-actions/ctxman-check.yml +49 -0
- package/examples/wizard-profiles/security-audit/profile.json +7 -1
- package/index.d.ts +1422 -0
- package/index.js +55 -31
- package/lib/agents/pr-failure-agent.js +48 -45
- package/lib/analyzers/ai-suggester.js +451 -0
- package/lib/analyzers/dependency-scanner.js +505 -0
- package/lib/analyzers/duplicate-detector.js +264 -0
- package/lib/analyzers/go-method-analyzer.js +151 -110
- package/lib/analyzers/import-tracker.js +431 -0
- package/lib/analyzers/method-analyzer.js +1055 -434
- package/lib/analyzers/performance-dashboard.js +683 -0
- package/lib/analyzers/token-calculator.js +895 -504
- package/lib/analyzers/unused-exports-detector.js +336 -0
- package/lib/api/mcp/install.js +424 -0
- package/lib/api/mcp/prompts.js +281 -281
- package/lib/api/mcp/resources.js +300 -241
- package/lib/api/mcp/sandbox.js +89 -0
- package/lib/api/mcp/server.js +675 -312
- package/lib/api/mcp/start.js +125 -0
- package/lib/api/mcp/transports/HttpSseTransport.js +60 -52
- package/lib/api/rest/server.js +105 -28
- package/lib/cache/CacheManager.js +9 -17
- package/lib/core/Analyzer.js +16 -10
- package/lib/core/ContextBuilder.js +19 -28
- package/lib/core/Reporter.js +15 -14
- package/lib/core/Scanner.js +81 -12
- package/lib/count/index.js +100 -0
- package/lib/formatters/format-registry.js +394 -372
- package/lib/formatters/gitingest-formatter.js +656 -621
- package/lib/formatters/toon-formatter-v1.3.js +637 -632
- package/lib/formatters/toon-formatter.js +239 -240
- package/lib/integrations/git/BlameTracker.js +4 -4
- package/lib/integrations/git/DiffAnalyzer.js +38 -31
- package/lib/integrations/git/GitClient.js +71 -53
- package/lib/lsp/server.js +311 -0
- package/lib/map/graph.js +299 -0
- package/lib/map/imports.js +233 -0
- package/lib/map/index.js +33 -0
- package/lib/map/rank.js +253 -0
- package/lib/map/render.js +346 -0
- package/lib/map/resolver.js +574 -0
- package/lib/pack/budget.js +111 -0
- package/lib/pack/code-lexer.js +360 -0
- package/lib/pack/compress.js +414 -0
- package/lib/pack/index.js +354 -0
- package/lib/pack/renderers/common.js +134 -0
- package/lib/pack/renderers/index.js +24 -0
- package/lib/pack/renderers/json.js +47 -0
- package/lib/pack/renderers/markdown.js +54 -0
- package/lib/pack/renderers/plain.js +38 -0
- package/lib/pack/renderers/xml.js +85 -0
- package/lib/pack/select.js +177 -0
- package/lib/pack/split.js +182 -0
- package/lib/pack/tree.js +35 -0
- package/lib/parsers/gitignore-parser.js +113 -107
- package/lib/parsers/method-filter-parser.js +50 -33
- package/lib/plugins/ExporterPlugin.js +3 -3
- package/lib/plugins/LanguagePlugin.js +3 -3
- package/lib/plugins/PluginManager.js +7 -13
- package/lib/policy/audit.js +294 -0
- package/lib/policy/enforce.js +125 -0
- package/lib/policy/ignore-files.js +169 -0
- package/lib/policy/index.js +52 -0
- package/lib/policy/policy.js +489 -0
- package/lib/pr/diff-parser.js +267 -0
- package/lib/pr/method-mapper.js +148 -0
- package/lib/pr/pr-context.js +829 -0
- package/lib/pr/pr-git.js +149 -0
- package/lib/pr/pr-renderer.js +644 -0
- package/lib/pr/related-tests.js +296 -0
- package/lib/rag/DataSourcePlugin.js +5 -5
- package/lib/rag/EmbeddingProvider.js +19 -8
- package/lib/rag/Indexer.js +14 -5
- package/lib/rag/VectorStoreFactory.js +3 -3
- package/lib/rag/sources/FileDataSource.js +6 -6
- package/lib/rag/stores/BaseVectorStore.js +5 -5
- package/lib/rag/stores/ChromaDBStore.js +10 -10
- package/lib/rag/stores/LanceDBStore.js +29 -19
- package/lib/rag/stores/PineconeStore.js +11 -9
- package/lib/search/bm25.js +433 -0
- package/lib/search/code-units.js +201 -0
- package/lib/search/symbols.js +189 -0
- package/lib/ui/dashboard.js +120 -89
- package/lib/ui/index.js +1 -6
- package/lib/ui/init-wizard-ui.js +550 -0
- package/lib/ui/progress-bar.js +94 -88
- package/lib/ui/select-input.js +40 -32
- package/lib/ui/template-selector.js +99 -0
- package/lib/ui/wizard.js +374 -329
- package/lib/utils/claude-token-counter.js +151 -0
- package/lib/utils/cli-output.js +17 -0
- package/lib/utils/clipboard-utils.js +49 -50
- package/lib/utils/config-utils.js +81 -66
- package/lib/utils/context-versioning.js +532 -0
- package/lib/utils/error-handler.js +112 -112
- package/lib/utils/file-utils.js +153 -69
- package/lib/utils/format-converter.js +356 -283
- package/lib/utils/git-utils.js +366 -294
- package/lib/utils/llm-cost-estimator.js +281 -0
- package/lib/utils/llm-detector.js +242 -78
- package/lib/utils/logger.js +243 -237
- package/lib/utils/messages.js +355 -0
- package/lib/utils/multi-repo-manager.js +364 -0
- package/lib/utils/profile-manager.js +380 -0
- package/lib/utils/safe-read.js +61 -0
- package/lib/utils/secret-redactor.js +61 -0
- package/lib/utils/snapshot-manager.js +679 -0
- package/lib/utils/template-manager.js +378 -0
- package/lib/utils/token-utils.js +201 -170
- package/lib/utils/tokenizer-adapter.js +421 -377
- package/lib/utils/updater.js +539 -506
- package/lib/watch/FileWatcher.js +12 -8
- package/lib/watch/IncrementalAnalyzer.js +7 -7
- package/lib/why/coupling.js +81 -0
- package/lib/why/errors.js +23 -0
- package/lib/why/git.js +169 -0
- package/lib/why/history.js +108 -0
- package/lib/why/index.js +720 -0
- package/lib/why/ownership.js +79 -0
- package/lib/why/paths.js +207 -0
- package/lib/why/references.js +259 -0
- package/lib/why/render.js +722 -0
- package/lib/why/search.js +358 -0
- package/lib/why/target.js +389 -0
- package/lib/wizards/config-generator.js +255 -0
- package/lib/wizards/project-detector.js +571 -0
- package/package.json +44 -10
- package/.ctxman/logs/ctxman-2026-07-02.log +0 -156
- package/.ctxman/self-knowledge/documents.lance/_transactions/0-b3dd1558-f95b-44b9-a154-18e8c330ae7a.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/1-44bff6d4-e552-4010-a769-e09ad822698d.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/10-37260794-5947-4ecb-b4c8-d732f58f40f1.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/11-0d52a883-5518-463c-9b6d-986c3af89dfd.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/12-ae354646-52f2-49c9-90fb-2e6ed8932663.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/13-5d7d5e57-27da-498b-b734-091b9d38da13.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/14-558ce928-810b-4252-a364-fdd1ee6f8a25.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/15-27e3a58f-8b64-43b8-a9f4-e3369446968b.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/16-a110c390-e5c8-4464-b976-bc7f5195fa15.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/17-97513225-53de-4216-b61a-6337bd0cc4ad.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/18-5ba450f0-c527-459c-bf4f-6d79b5e4285f.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/19-600d3a63-5225-49e9-b8b7-30e037b564ab.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/2-48a61b4e-92cf-4342-8376-914c1b99a9f9.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/20-121dd912-5410-410e-84ff-82c7a23604a0.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/3-bc6844ab-7ea3-4bb3-9ee1-538ce5664a9c.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/4-d5e1754f-dbb5-454e-a9be-2a7da8d7bcf9.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/5-4fae93df-0568-4b6f-8acf-8bef4bba6047.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/6-4b1b8b87-068e-49a3-ba2b-d8dae628b252.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/7-63d040fb-d2fa-4c0f-bed0-53411918a7b0.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/8-c5628cf6-264c-420f-aab8-bc96a9083591.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_transactions/9-c94570aa-5e53-4fcb-971f-cecc1d1d5492.txn +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/1.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/10.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/11.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/12.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/13.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/14.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/15.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/16.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/17.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/18.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/19.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/2.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/20.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/21.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/3.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/4.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/5.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/6.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/7.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/8.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/_versions/9.manifest +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/000010010011011101000010de8621405d9d5cb06540dbf02c.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/0000101001011100011000003fb62a4aa8978974075c144f1c.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/000011010000100111001011c6cc824d628f3bb1f4e04cd5d0.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/001000001000011001011100d8f74349ada090b332cd1df669.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/01000001111010001000010045ab44467bb1e7e4d8ae06bd58.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/01000101101110111110011131ef904241a89e1ffb6ddaf8e3.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/010010110000010000011010fbfcc145c6a8a827b28057a07e.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1000011000101010001110003fb4f045c3835982efa1405324.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/100111011101110111101011f4396b410fa06af3b028d204e6.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/100111100111101000001100596b174e00995d6cc2969cc8dc.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1010001100011000110010102ea16b484d887c7c9725feffd6.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/101101100010101111110111c5230944d9b3af7301b6392dda.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1011011100100111111110105892bf41b5a530fedd7cf516f1.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/10111110000010001011110164e8774742bade87b483c10f36.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1100000100010110001110103ffaca40a2a42c85067ebaf9ae.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/110000010001110110101110664bde4c76a78dfc14b05e83b8.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/11001100111000010100110050645c4281b1f35b2022234f7f.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1101111011001100110111017214924a368f5719eab2246619.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1110000111001010001000002150c24dcab653d39d07032e63.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/1110100000000100000011014d20534967a18b5c87721659ff.lance +0 -0
- package/.ctxman/self-knowledge/documents.lance/data/11110101111010000101011135a0114ef2b35dc6f1297a3381.lance +0 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Find the test files related to a set of changed files: by file name (foo.js ↔
|
|
3
|
+
* foo.test.js, test_foo.py, foo_test.go, FooTest.java, __tests__/foo.js), by package (Go
|
|
4
|
+
* tests in the same directory), by imports of the changed modules (JavaScript/TypeScript
|
|
5
|
+
* through ImportTracker, Python `import`/`from`), and by references to changed class or
|
|
6
|
+
* method names
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import path from 'path';
|
|
10
|
+
import { ImportTracker } from '../analyzers/import-tracker.js';
|
|
11
|
+
import { supportsMethods } from './method-mapper.js';
|
|
12
|
+
|
|
13
|
+
const posix = path.posix;
|
|
14
|
+
|
|
15
|
+
const TEST_DIR = /(^|\/)(__tests__|__test__|tests?|specs?)\//i;
|
|
16
|
+
const NON_TEST_DIR = /(^|\/)(__fixtures__|fixtures?|__snapshots__|__mocks__|testdata)\//i;
|
|
17
|
+
const GENERIC_STEMS = new Set(['index', '__init__', 'main', 'mod', 'lib', 'init']);
|
|
18
|
+
const JS_EXTENSIONS = ['.js', '.ts', '.mjs', '.cjs', '.jsx', '.tsx', '.mts', '.cts'];
|
|
19
|
+
const JS_FILE = /\.(m|c)?(j|t)sx?$/i;
|
|
20
|
+
const CLASS_LANGUAGES = /\.(java|kt|kts|scala|cs|swift|php|groovy)$/i;
|
|
21
|
+
|
|
22
|
+
// Names too common to say anything when a test mentions them
|
|
23
|
+
const COMMON_NAMES = new Set(
|
|
24
|
+
(
|
|
25
|
+
'constructor init main run get set test setup setUp tearDown toString render handle ' +
|
|
26
|
+
'execute call apply process update create delete remove start stop close open load ' +
|
|
27
|
+
'save read write value index length size name then catch describe expect before ' +
|
|
28
|
+
'after beforeEach afterEach beforeAll afterAll __init__ __str__ __repr__ __eq__ equals ' +
|
|
29
|
+
'hashCode error parse format build reset clear next done send emit async await'
|
|
30
|
+
).split(' ')
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
const importTracker = new ImportTracker('.');
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Whether a path looks like a test source file
|
|
37
|
+
* @param {string} filePath - Repository-relative, forward slashes
|
|
38
|
+
* @returns {boolean}
|
|
39
|
+
*/
|
|
40
|
+
export function isTestFile(filePath) {
|
|
41
|
+
if (!supportsMethods(filePath) || NON_TEST_DIR.test(filePath)) return false;
|
|
42
|
+
const base = posix.basename(filePath);
|
|
43
|
+
return (
|
|
44
|
+
/[._-](test|spec)s?\.[^.]+$/i.test(base) ||
|
|
45
|
+
/^test_.+\.py$/i.test(base) ||
|
|
46
|
+
/[a-z0-9](Tests?|Spec|IT)\.[^.]+$/.test(base) ||
|
|
47
|
+
TEST_DIR.test(filePath)
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const normalize = (stem) => stem.toLowerCase().replace(/[-_.]/g, '');
|
|
52
|
+
|
|
53
|
+
/** Directory name to use for generic file names such as index.js or __init__.py */
|
|
54
|
+
function parentName(filePath) {
|
|
55
|
+
const parts = posix.dirname(filePath).split('/');
|
|
56
|
+
for (let i = parts.length - 1; i >= 0; i--) {
|
|
57
|
+
if (parts[i] && parts[i] !== '.' && !/^(__tests__|__test__|tests?|specs?)$/i.test(parts[i])) {
|
|
58
|
+
return parts[i];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return '';
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Name a source file is known by, for matching against test names
|
|
66
|
+
* @param {string} filePath
|
|
67
|
+
*/
|
|
68
|
+
export function sourceStem(filePath) {
|
|
69
|
+
const stem = posix.basename(filePath).replace(/\.[^.]+$/, '');
|
|
70
|
+
return normalize(GENERIC_STEMS.has(stem.toLowerCase()) ? parentName(filePath) : stem);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Name of the source file a test is named after (foo.test.js → foo, FooTest.java → foo)
|
|
75
|
+
* @param {string} filePath
|
|
76
|
+
*/
|
|
77
|
+
export function testStem(filePath) {
|
|
78
|
+
let stem = posix.basename(filePath).replace(/\.[^.]+$/, '');
|
|
79
|
+
stem = stem
|
|
80
|
+
.replace(/[._-](test|spec)s?$/i, '')
|
|
81
|
+
.replace(/^test_/i, '')
|
|
82
|
+
.replace(/([a-z0-9])(Tests?|Spec|IT)$/, '$1');
|
|
83
|
+
if (!stem || GENERIC_STEMS.has(stem.toLowerCase())) stem = parentName(filePath);
|
|
84
|
+
return normalize(stem);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Repository-relative module paths a JavaScript/TypeScript test imports */
|
|
88
|
+
function jsImports(testPath, content) {
|
|
89
|
+
const specifiers = [
|
|
90
|
+
...importTracker.parseES6Imports(content),
|
|
91
|
+
...importTracker.parseCommonJSImports(content),
|
|
92
|
+
].map((imp) => imp.package);
|
|
93
|
+
// Side-effect imports and module mocks, which ImportTracker does not report
|
|
94
|
+
const extra = /(?:\bimport\s+|\b(?:vi|jest)\.(?:mock|doMock)\s*\(\s*)['"]([^'"]+)['"]/g;
|
|
95
|
+
let match;
|
|
96
|
+
while ((match = extra.exec(content)) !== null) specifiers.push(match[1]);
|
|
97
|
+
|
|
98
|
+
const dir = posix.dirname(testPath);
|
|
99
|
+
const resolved = new Set();
|
|
100
|
+
for (const specifier of specifiers) {
|
|
101
|
+
if (!specifier.startsWith('.')) continue;
|
|
102
|
+
const base = posix.normalize(posix.join(dir, specifier));
|
|
103
|
+
const stripped = base.replace(JS_FILE, '');
|
|
104
|
+
resolved.add(base);
|
|
105
|
+
for (const ext of JS_EXTENSIONS) {
|
|
106
|
+
resolved.add(base + ext);
|
|
107
|
+
resolved.add(`${base}/index${ext}`);
|
|
108
|
+
resolved.add(stripped + ext);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return resolved;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Path suffixes (pkg/mod.py, pkg/mod/__init__.py) of the modules a Python test imports */
|
|
115
|
+
function pythonImports(testPath, content) {
|
|
116
|
+
const modules = [];
|
|
117
|
+
const pattern =
|
|
118
|
+
/^[ \t]*(?:from[ \t]+([.\w]+)[ \t]+import[ \t]+\(?([\w*, \t]+)|import[ \t]+([\w., \t]+))/gm;
|
|
119
|
+
let match;
|
|
120
|
+
while ((match = pattern.exec(content)) !== null) {
|
|
121
|
+
if (match[1]) {
|
|
122
|
+
modules.push(match[1]);
|
|
123
|
+
for (const name of match[2].split(',')) {
|
|
124
|
+
const clean = name.trim().split(/\s+/)[0];
|
|
125
|
+
if (clean && clean !== '*')
|
|
126
|
+
modules.push(`${match[1]}${match[1].endsWith('.') ? '' : '.'}${clean}`);
|
|
127
|
+
}
|
|
128
|
+
} else {
|
|
129
|
+
for (const name of match[3].split(',')) modules.push(name.trim().split(/\s+/)[0]);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const suffixes = [];
|
|
134
|
+
for (const moduleName of modules) {
|
|
135
|
+
if (!moduleName) continue;
|
|
136
|
+
const dots = /^\.*/.exec(moduleName)[0].length;
|
|
137
|
+
const rest = moduleName.slice(dots).split('.').filter(Boolean).join('/');
|
|
138
|
+
if (!rest) continue;
|
|
139
|
+
let base = rest;
|
|
140
|
+
if (dots > 0) {
|
|
141
|
+
let dir = posix.dirname(testPath);
|
|
142
|
+
for (let i = 1; i < dots; i++) dir = posix.dirname(dir);
|
|
143
|
+
base = posix.normalize(posix.join(dir, rest));
|
|
144
|
+
if (base.startsWith('..')) continue;
|
|
145
|
+
suffixes.push(
|
|
146
|
+
{ path: `${base}.py`, exact: true },
|
|
147
|
+
{ path: `${base}/__init__.py`, exact: true }
|
|
148
|
+
);
|
|
149
|
+
} else {
|
|
150
|
+
suffixes.push(
|
|
151
|
+
{ path: `${base}.py`, exact: false },
|
|
152
|
+
{ path: `${base}/__init__.py`, exact: false }
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return suffixes;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const escapeRegex = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Names that `content` mentions as whole identifiers (`call` set: only when called, as in
|
|
163
|
+
* `name(`, which keeps out variables and properties that share a method's name)
|
|
164
|
+
*/
|
|
165
|
+
function referencedNames(content, names, { call = false } = {}) {
|
|
166
|
+
if (names.length === 0) return [];
|
|
167
|
+
const after = call ? '\\s*\\(' : '(?![\\w$])';
|
|
168
|
+
const pattern = new RegExp(`(?<![\\w$])(${names.map(escapeRegex).join('|')})${after}`, 'g');
|
|
169
|
+
const found = new Set();
|
|
170
|
+
let match;
|
|
171
|
+
while ((match = pattern.exec(content)) !== null) found.add(match[1]);
|
|
172
|
+
return [...found];
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* @typedef {object} RelatedTest
|
|
177
|
+
* @property {string} path
|
|
178
|
+
* @property {string[]} reasons
|
|
179
|
+
* @property {boolean} changed - The test itself is part of the change
|
|
180
|
+
*/
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Test files related to a change
|
|
184
|
+
* @param {object} params
|
|
185
|
+
* @param {string[]} params.candidates - Every file of the head version (repository-relative)
|
|
186
|
+
* @param {Array<{path: string, oldPath: string|null, status: string}>} params.changedFiles
|
|
187
|
+
* @param {string[]} [params.methodNames] - Names of changed and deleted methods
|
|
188
|
+
* @param {(paths: string[]) => Map<string, string|null>} params.readContents - Head contents
|
|
189
|
+
* @param {number} [params.maxScan=3000] - Read at most this many test files for imports
|
|
190
|
+
* and references (beyond it only names are compared)
|
|
191
|
+
* @returns {RelatedTest[]} Strongest matches first
|
|
192
|
+
*/
|
|
193
|
+
export function findRelatedTests({
|
|
194
|
+
candidates,
|
|
195
|
+
changedFiles,
|
|
196
|
+
methodNames = [],
|
|
197
|
+
readContents,
|
|
198
|
+
maxScan = 3000,
|
|
199
|
+
}) {
|
|
200
|
+
const tests = candidates.filter(isTestFile);
|
|
201
|
+
const changedTests = new Set(changedFiles.filter((f) => f.status !== 'D').map((f) => f.path));
|
|
202
|
+
const sources = changedFiles.filter((f) => !isTestFile(f.path));
|
|
203
|
+
const sourcePaths = new Set();
|
|
204
|
+
for (const file of sources) {
|
|
205
|
+
sourcePaths.add(file.path);
|
|
206
|
+
if (file.oldPath) sourcePaths.add(file.oldPath);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const byStem = new Map();
|
|
210
|
+
for (const sourcePath of sourcePaths) {
|
|
211
|
+
const stem = sourceStem(sourcePath);
|
|
212
|
+
if (!stem) continue;
|
|
213
|
+
if (!byStem.has(stem)) byStem.set(stem, []);
|
|
214
|
+
byStem.get(stem).push(sourcePath);
|
|
215
|
+
}
|
|
216
|
+
const goDirs = new Map();
|
|
217
|
+
for (const sourcePath of sourcePaths) {
|
|
218
|
+
if (sourcePath.endsWith('.go')) goDirs.set(posix.dirname(sourcePath), sourcePath);
|
|
219
|
+
}
|
|
220
|
+
const classNames = [...sourcePaths]
|
|
221
|
+
.filter((p) => CLASS_LANGUAGES.test(p))
|
|
222
|
+
.map((p) => posix.basename(p).replace(/\.[^.]+$/, ''))
|
|
223
|
+
.filter((name) => /^[A-Z]\w{3,}$/.test(name));
|
|
224
|
+
const names = [
|
|
225
|
+
...new Set(
|
|
226
|
+
methodNames
|
|
227
|
+
.map((name) => name.split(/\s+/).pop())
|
|
228
|
+
.filter((name) => name && name.length >= 4 && !COMMON_NAMES.has(name))
|
|
229
|
+
),
|
|
230
|
+
];
|
|
231
|
+
|
|
232
|
+
const found = new Map();
|
|
233
|
+
const add = (testPath, reason, weight) => {
|
|
234
|
+
if (!found.has(testPath)) found.set(testPath, { path: testPath, reasons: [], score: 0 });
|
|
235
|
+
const entry = found.get(testPath);
|
|
236
|
+
if (!entry.reasons.includes(reason)) {
|
|
237
|
+
entry.reasons.push(reason);
|
|
238
|
+
entry.score += weight;
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
for (const testPath of tests) {
|
|
243
|
+
if (changedTests.has(testPath)) add(testPath, 'changed in this diff', 3);
|
|
244
|
+
for (const sourcePath of byStem.get(testStem(testPath)) || []) {
|
|
245
|
+
add(testPath, `named after ${sourcePath}`, 3);
|
|
246
|
+
}
|
|
247
|
+
if (testPath.endsWith('_test.go') && goDirs.has(posix.dirname(testPath))) {
|
|
248
|
+
add(testPath, `same Go package as ${goDirs.get(posix.dirname(testPath))}`, 2);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (sourcePaths.size > 0 && tests.length <= maxScan) {
|
|
253
|
+
const contents = readContents(tests);
|
|
254
|
+
const callers = new Map();
|
|
255
|
+
for (const testPath of tests) {
|
|
256
|
+
const content = contents.get(testPath);
|
|
257
|
+
if (!content) continue;
|
|
258
|
+
if (JS_FILE.test(testPath)) {
|
|
259
|
+
const imported = jsImports(testPath, content);
|
|
260
|
+
for (const sourcePath of sourcePaths) {
|
|
261
|
+
if (imported.has(sourcePath)) add(testPath, `imports ${sourcePath}`, 3);
|
|
262
|
+
}
|
|
263
|
+
} else if (testPath.endsWith('.py')) {
|
|
264
|
+
for (const suffix of pythonImports(testPath, content)) {
|
|
265
|
+
for (const sourcePath of sourcePaths) {
|
|
266
|
+
const matches = suffix.exact
|
|
267
|
+
? sourcePath === suffix.path
|
|
268
|
+
: sourcePath === suffix.path || sourcePath.endsWith(`/${suffix.path}`);
|
|
269
|
+
if (matches) add(testPath, `imports ${sourcePath}`, 3);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
for (const name of referencedNames(content, classNames)) {
|
|
274
|
+
add(testPath, `references ${name}`, 2);
|
|
275
|
+
}
|
|
276
|
+
for (const name of referencedNames(content, names, { call: true })) {
|
|
277
|
+
if (!callers.has(name)) callers.set(name, []);
|
|
278
|
+
callers.get(name).push(testPath);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
// A name that many tests call (a common helper name) says nothing about this change
|
|
282
|
+
const common = Math.max(10, tests.length * 0.1);
|
|
283
|
+
for (const [name, testPaths] of callers) {
|
|
284
|
+
if (testPaths.length > common) continue;
|
|
285
|
+
for (const testPath of testPaths) add(testPath, `calls ${name}()`, 1);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
return [...found.values()]
|
|
290
|
+
.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path))
|
|
291
|
+
.map(({ path: testPath, reasons }) => ({
|
|
292
|
+
path: testPath,
|
|
293
|
+
reasons,
|
|
294
|
+
changed: changedTests.has(testPath),
|
|
295
|
+
}));
|
|
296
|
+
}
|
|
@@ -11,21 +11,21 @@ export class DataSourcePlugin {
|
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Validates if the necessary configuration (tokens, urls) is present.
|
|
14
|
-
* @param {Object} config
|
|
14
|
+
* @param {Object} config
|
|
15
15
|
* @returns {Promise<boolean>}
|
|
16
16
|
*/
|
|
17
|
-
async validateConfig(
|
|
17
|
+
async validateConfig(_config) {
|
|
18
18
|
return true;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
22
|
* Collects data from the source.
|
|
23
23
|
* Generator function that yields documents to be indexed.
|
|
24
|
-
*
|
|
25
|
-
* @param {Object} options
|
|
24
|
+
*
|
|
25
|
+
* @param {Object} options
|
|
26
26
|
* @yields {import('./Indexer').IndexableDocument}
|
|
27
27
|
*/
|
|
28
|
-
async *collect(
|
|
28
|
+
async *collect(_options = {}) {
|
|
29
29
|
throw new Error('Not implemented');
|
|
30
30
|
}
|
|
31
31
|
}
|
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
import { pipeline } from '@xenova/transformers';
|
|
2
|
-
|
|
3
1
|
/**
|
|
4
2
|
* Abstract base class for embedding providers
|
|
5
3
|
*/
|
|
6
4
|
export class EmbeddingProvider {
|
|
7
5
|
/**
|
|
8
6
|
* Generates an embedding for the given text.
|
|
9
|
-
* @param {string} text
|
|
7
|
+
* @param {string} text
|
|
10
8
|
* @returns {Promise<number[]>}
|
|
11
9
|
*/
|
|
12
|
-
async embed(
|
|
10
|
+
async embed(_text) {
|
|
13
11
|
throw new Error('Not implemented');
|
|
14
12
|
}
|
|
15
13
|
}
|
|
@@ -27,8 +25,21 @@ export class TransformersEmbeddingProvider extends EmbeddingProvider {
|
|
|
27
25
|
|
|
28
26
|
async init() {
|
|
29
27
|
if (!this.pipe) {
|
|
28
|
+
// Optional dependency: only `ctxman ask` needs it, so load it on first use
|
|
29
|
+
let transformers;
|
|
30
|
+
try {
|
|
31
|
+
transformers = await import('@xenova/transformers');
|
|
32
|
+
} catch (error) {
|
|
33
|
+
if (error.code === 'ERR_MODULE_NOT_FOUND') {
|
|
34
|
+
throw new Error(
|
|
35
|
+
'Please install @xenova/transformers to use TransformersEmbeddingProvider (ctxman ask): npm install @xenova/transformers'
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
|
|
30
41
|
// Load the pipeline. This will download the model if not present.
|
|
31
|
-
this.pipe = await pipeline('feature-extraction', this.modelName);
|
|
42
|
+
this.pipe = await transformers.pipeline('feature-extraction', this.modelName);
|
|
32
43
|
}
|
|
33
44
|
}
|
|
34
45
|
|
|
@@ -57,10 +68,10 @@ export class MockEmbeddingProvider extends EmbeddingProvider {
|
|
|
57
68
|
// Let's make it pseudo-random based on text hash to be stable.
|
|
58
69
|
let hash = 0;
|
|
59
70
|
for (let i = 0; i < text.length; i++) {
|
|
60
|
-
hash = (
|
|
71
|
+
hash = (hash << 5) - hash + text.charCodeAt(i);
|
|
61
72
|
hash |= 0;
|
|
62
73
|
}
|
|
63
|
-
|
|
74
|
+
|
|
64
75
|
const vector = [];
|
|
65
76
|
for (let i = 0; i < this.dimension; i++) {
|
|
66
77
|
// Simple pseudo-random generator
|
|
@@ -70,6 +81,6 @@ export class MockEmbeddingProvider extends EmbeddingProvider {
|
|
|
70
81
|
|
|
71
82
|
// Normalize
|
|
72
83
|
const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
|
|
73
|
-
return vector.map(val => val / norm);
|
|
84
|
+
return vector.map((val) => val / norm);
|
|
74
85
|
}
|
|
75
86
|
}
|
package/lib/rag/Indexer.js
CHANGED
|
@@ -11,7 +11,7 @@ const logger = getLogger('Indexer');
|
|
|
11
11
|
|
|
12
12
|
export class Indexer {
|
|
13
13
|
/**
|
|
14
|
-
* @param {import('./stores/BaseVectorStore').BaseVectorStore} vectorStore
|
|
14
|
+
* @param {import('./stores/BaseVectorStore').BaseVectorStore} vectorStore
|
|
15
15
|
*/
|
|
16
16
|
constructor(vectorStore) {
|
|
17
17
|
this.vectorStore = vectorStore;
|
|
@@ -20,7 +20,7 @@ export class Indexer {
|
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
22
|
* Register a data source plugin
|
|
23
|
-
* @param {import('./DataSourcePlugin').DataSourcePlugin} sourcePlugin
|
|
23
|
+
* @param {import('./DataSourcePlugin').DataSourcePlugin} sourcePlugin
|
|
24
24
|
*/
|
|
25
25
|
registerSource(sourcePlugin) {
|
|
26
26
|
this.sources.set(sourcePlugin.name, sourcePlugin);
|
|
@@ -31,9 +31,10 @@ export class Indexer {
|
|
|
31
31
|
* Run indexing for a specific source or all sources
|
|
32
32
|
* @param {string} [sourceName] - Optional source name to run specific indexer
|
|
33
33
|
* @param {Object} options - Options passed to collect method
|
|
34
|
+
* @throws {Error} When a source fails, after the remaining sources have been indexed
|
|
34
35
|
*/
|
|
35
36
|
async index(sourceName, options = {}) {
|
|
36
|
-
const sourcesToRun = sourceName
|
|
37
|
+
const sourcesToRun = sourceName
|
|
37
38
|
? [this.sources.get(sourceName)].filter(Boolean)
|
|
38
39
|
: Array.from(this.sources.values());
|
|
39
40
|
|
|
@@ -42,6 +43,7 @@ export class Indexer {
|
|
|
42
43
|
return;
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
const failures = [];
|
|
45
47
|
for (const source of sourcesToRun) {
|
|
46
48
|
logger.info(`Starting indexing for source: ${source.name}`);
|
|
47
49
|
let count = 0;
|
|
@@ -56,8 +58,8 @@ export class Indexer {
|
|
|
56
58
|
...doc.metadata,
|
|
57
59
|
sourceId: doc.id,
|
|
58
60
|
source: source.name,
|
|
59
|
-
indexedAt: new Date().toISOString()
|
|
60
|
-
}
|
|
61
|
+
indexedAt: new Date().toISOString(),
|
|
62
|
+
},
|
|
61
63
|
});
|
|
62
64
|
|
|
63
65
|
if (batch.length >= BATCH_SIZE) {
|
|
@@ -77,7 +79,14 @@ export class Indexer {
|
|
|
77
79
|
logger.info(`Completed indexing ${source.name}. Total documents: ${count}`);
|
|
78
80
|
} catch (error) {
|
|
79
81
|
logger.error(`Error indexing ${source.name}: ${error.message}`);
|
|
82
|
+
failures.push(`${source.name}: ${error.message}`);
|
|
80
83
|
}
|
|
81
84
|
}
|
|
85
|
+
|
|
86
|
+
// Callers must not treat a failed source (e.g. the embedding model could not be
|
|
87
|
+
// loaded) as an empty but successful index
|
|
88
|
+
if (failures.length > 0) {
|
|
89
|
+
throw new Error(`Indexing failed for ${failures.join('; ')}`);
|
|
90
|
+
}
|
|
82
91
|
}
|
|
83
92
|
}
|
|
@@ -4,9 +4,9 @@ import { PineconeStore } from './stores/PineconeStore.js';
|
|
|
4
4
|
|
|
5
5
|
export class VectorStoreFactory {
|
|
6
6
|
/**
|
|
7
|
-
* @param {string} type
|
|
8
|
-
* @param {import('./EmbeddingProvider').EmbeddingProvider} embeddingProvider
|
|
9
|
-
* @param {Object} config
|
|
7
|
+
* @param {string} type
|
|
8
|
+
* @param {import('./EmbeddingProvider').EmbeddingProvider} embeddingProvider
|
|
9
|
+
* @param {Object} config
|
|
10
10
|
*/
|
|
11
11
|
static create(type, embeddingProvider, config = {}) {
|
|
12
12
|
switch (type.toLowerCase()) {
|
|
@@ -36,8 +36,8 @@ export class FileDataSource extends DataSourcePlugin {
|
|
|
36
36
|
|
|
37
37
|
for (const entry of entries) {
|
|
38
38
|
const fullPath = path.join(dir, entry.name);
|
|
39
|
-
|
|
40
|
-
if (exclude.some(ex => fullPath.includes(ex))) continue;
|
|
39
|
+
|
|
40
|
+
if (exclude.some((ex) => fullPath.includes(ex))) continue;
|
|
41
41
|
|
|
42
42
|
if (entry.isDirectory()) {
|
|
43
43
|
yield* this.walkDirectory(fullPath, depth - 1, include, exclude);
|
|
@@ -48,7 +48,7 @@ export class FileDataSource extends DataSourcePlugin {
|
|
|
48
48
|
try {
|
|
49
49
|
const content = await fs.readFile(fullPath, 'utf-8');
|
|
50
50
|
// Basic text check (skip binaries)
|
|
51
|
-
if (content.includes('\0')) continue;
|
|
51
|
+
if (content.includes('\0')) continue;
|
|
52
52
|
|
|
53
53
|
yield {
|
|
54
54
|
id: fullPath,
|
|
@@ -58,11 +58,11 @@ export class FileDataSource extends DataSourcePlugin {
|
|
|
58
58
|
path: fullPath,
|
|
59
59
|
extension: ext,
|
|
60
60
|
filename: entry.name,
|
|
61
|
-
size: content.length
|
|
62
|
-
}
|
|
61
|
+
size: content.length,
|
|
62
|
+
},
|
|
63
63
|
};
|
|
64
64
|
} catch (error) {
|
|
65
|
-
|
|
65
|
+
console.warn(`[FileDataSource] Failed to read ${fullPath}: ${error.message}`);
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
}
|
|
@@ -26,18 +26,18 @@ export class BaseVectorStore {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
|
-
* @param {Array<{text: string, metadata: Object}>} docs
|
|
29
|
+
* @param {Array<{text: string, metadata: Object}>} docs
|
|
30
30
|
*/
|
|
31
|
-
async addDocuments(
|
|
31
|
+
async addDocuments(_docs) {
|
|
32
32
|
throw new Error('Method not implemented');
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
|
-
* @param {string} queryText
|
|
37
|
-
* @param {number} limit
|
|
36
|
+
* @param {string} queryText
|
|
37
|
+
* @param {number} limit
|
|
38
38
|
* @returns {Promise<Array<SearchResult>>}
|
|
39
39
|
*/
|
|
40
|
-
async search(queryText,
|
|
40
|
+
async search(queryText, _limit = 5) {
|
|
41
41
|
throw new Error('Method not implemented');
|
|
42
42
|
}
|
|
43
43
|
}
|
|
@@ -21,7 +21,7 @@ export class ChromaDBStore extends BaseVectorStore {
|
|
|
21
21
|
try {
|
|
22
22
|
const { ChromaClient } = await import('chromadb');
|
|
23
23
|
this.client = new ChromaClient({ path: this.url });
|
|
24
|
-
|
|
24
|
+
|
|
25
25
|
this.collection = await this.client.getOrCreateCollection({
|
|
26
26
|
name: this.collectionName,
|
|
27
27
|
});
|
|
@@ -37,22 +37,22 @@ export class ChromaDBStore extends BaseVectorStore {
|
|
|
37
37
|
if (!this.collection) await this.init();
|
|
38
38
|
if (docs.length === 0) return;
|
|
39
39
|
|
|
40
|
-
const ids = docs.map(d => d.id || crypto.randomUUID());
|
|
41
|
-
const metadatas = docs.map(d => d.metadata || {});
|
|
42
|
-
const documents = docs.map(d => d.text);
|
|
43
|
-
|
|
40
|
+
const ids = docs.map((d) => d.id || crypto.randomUUID());
|
|
41
|
+
const metadatas = docs.map((d) => d.metadata || {});
|
|
42
|
+
const documents = docs.map((d) => d.text);
|
|
43
|
+
|
|
44
44
|
// Chroma can handle embeddings, but we have a provider.
|
|
45
45
|
// If we want to use our provider:
|
|
46
46
|
const embeddings = [];
|
|
47
47
|
for (const doc of docs) {
|
|
48
|
-
|
|
48
|
+
embeddings.push(await this.embeddingProvider.embed(doc.text));
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
await this.collection.add({
|
|
52
52
|
ids,
|
|
53
53
|
embeddings,
|
|
54
54
|
metadatas,
|
|
55
|
-
documents
|
|
55
|
+
documents,
|
|
56
56
|
});
|
|
57
57
|
}
|
|
58
58
|
|
|
@@ -63,7 +63,7 @@ export class ChromaDBStore extends BaseVectorStore {
|
|
|
63
63
|
|
|
64
64
|
const results = await this.collection.query({
|
|
65
65
|
queryEmbeddings: [queryEmbedding],
|
|
66
|
-
nResults: limit
|
|
66
|
+
nResults: limit,
|
|
67
67
|
});
|
|
68
68
|
|
|
69
69
|
// Chroma returns arrays of arrays
|
|
@@ -71,12 +71,12 @@ export class ChromaDBStore extends BaseVectorStore {
|
|
|
71
71
|
|
|
72
72
|
const hits = [];
|
|
73
73
|
const count = results.ids[0].length;
|
|
74
|
-
|
|
74
|
+
|
|
75
75
|
for (let i = 0; i < count; i++) {
|
|
76
76
|
hits.push({
|
|
77
77
|
text: results.documents[0][i],
|
|
78
78
|
score: results.distances ? results.distances[0][i] : 0,
|
|
79
|
-
metadata: results.metadatas[0][i]
|
|
79
|
+
metadata: results.metadatas[0][i],
|
|
80
80
|
});
|
|
81
81
|
}
|
|
82
82
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as lancedb from '@lancedb/lancedb';
|
|
2
1
|
import fs from 'fs/promises';
|
|
3
2
|
import { BaseVectorStore } from './BaseVectorStore.js';
|
|
4
3
|
|
|
@@ -19,11 +18,24 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
19
18
|
async init() {
|
|
20
19
|
if (this.initialized) return;
|
|
21
20
|
|
|
21
|
+
// Optional dependency: only `ctxman ask` needs it, so load it on first use
|
|
22
|
+
let lancedb;
|
|
23
|
+
try {
|
|
24
|
+
lancedb = await import('@lancedb/lancedb');
|
|
25
|
+
} catch (error) {
|
|
26
|
+
if (error.code === 'ERR_MODULE_NOT_FOUND') {
|
|
27
|
+
throw new Error(
|
|
28
|
+
'Please install @lancedb/lancedb to use LanceDBStore (ctxman ask): npm install @lancedb/lancedb'
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
|
|
22
34
|
// Ensure directory exists
|
|
23
35
|
await fs.mkdir(this.storagePath, { recursive: true });
|
|
24
36
|
|
|
25
37
|
this.db = await lancedb.connect(this.storagePath);
|
|
26
|
-
|
|
38
|
+
|
|
27
39
|
const tableNames = await this.db.tableNames();
|
|
28
40
|
if (tableNames.includes(this.tableName)) {
|
|
29
41
|
this.table = await this.db.openTable(this.tableName);
|
|
@@ -34,8 +46,8 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
34
46
|
|
|
35
47
|
/**
|
|
36
48
|
* Adds a single document to the store.
|
|
37
|
-
* @param {string} text
|
|
38
|
-
* @param {Object} metadata
|
|
49
|
+
* @param {string} text
|
|
50
|
+
* @param {Object} metadata
|
|
39
51
|
*/
|
|
40
52
|
async addDocument(text, metadata = {}) {
|
|
41
53
|
return this.addDocuments([{ text, metadata }]);
|
|
@@ -43,7 +55,7 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
43
55
|
|
|
44
56
|
/**
|
|
45
57
|
* Adds multiple documents to the store.
|
|
46
|
-
* @param {Array<{text: string, metadata: Object}>} docs
|
|
58
|
+
* @param {Array<{text: string, metadata: Object}>} docs
|
|
47
59
|
*/
|
|
48
60
|
async addDocuments(docs) {
|
|
49
61
|
if (docs.length === 0) return;
|
|
@@ -57,7 +69,7 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
57
69
|
text: doc.text,
|
|
58
70
|
id: crypto.randomUUID(),
|
|
59
71
|
timestamp: Date.now(),
|
|
60
|
-
...doc.metadata
|
|
72
|
+
...doc.metadata,
|
|
61
73
|
});
|
|
62
74
|
}
|
|
63
75
|
|
|
@@ -71,8 +83,8 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
71
83
|
|
|
72
84
|
/**
|
|
73
85
|
* Semantic search for documents.
|
|
74
|
-
* @param {string} queryText
|
|
75
|
-
* @param {number} limit
|
|
86
|
+
* @param {string} queryText
|
|
87
|
+
* @param {number} limit
|
|
76
88
|
* @returns {Promise<Array<{text: string, score: number, metadata: Object}>>}
|
|
77
89
|
*/
|
|
78
90
|
async search(queryText, limit = 5) {
|
|
@@ -80,19 +92,17 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
80
92
|
if (!this.table) return [];
|
|
81
93
|
|
|
82
94
|
const queryVector = await this.embeddingProvider.embed(queryText);
|
|
83
|
-
|
|
95
|
+
|
|
84
96
|
// LanceDB returns _distance by default (L2 distance usually).
|
|
85
97
|
// Lower is better.
|
|
86
|
-
const results = await this.table.search(queryVector)
|
|
87
|
-
.limit(limit)
|
|
88
|
-
.toArray();
|
|
98
|
+
const results = await this.table.search(queryVector).limit(limit).toArray();
|
|
89
99
|
|
|
90
|
-
return results.map(row => {
|
|
91
|
-
const { vector, text, _distance, ...metadata } = row;
|
|
100
|
+
return results.map((row) => {
|
|
101
|
+
const { vector: _vector, text, _distance, ...metadata } = row;
|
|
92
102
|
return {
|
|
93
103
|
text,
|
|
94
104
|
score: _distance,
|
|
95
|
-
metadata
|
|
105
|
+
metadata,
|
|
96
106
|
};
|
|
97
107
|
});
|
|
98
108
|
}
|
|
@@ -100,10 +110,10 @@ export class LanceDBStore extends BaseVectorStore {
|
|
|
100
110
|
async deleteTable() {
|
|
101
111
|
if (!this.initialized) await this.init();
|
|
102
112
|
try {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
} catch (
|
|
106
|
-
|
|
113
|
+
await this.db.dropTable(this.tableName);
|
|
114
|
+
this.table = null;
|
|
115
|
+
} catch (_e) {
|
|
116
|
+
// Ignore if not exists
|
|
107
117
|
}
|
|
108
118
|
}
|
|
109
119
|
}
|