brainbank 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +233 -126
- package/dist/{base-DZWtdgIf.d.ts → base-3SNc_CeY.d.ts} +24 -24
- package/dist/chunk-424UFCY7.js +78 -0
- package/dist/chunk-424UFCY7.js.map +1 -0
- package/dist/{chunk-HNPABX7L.js → chunk-7EZR47JV.js} +1 -1
- package/dist/{chunk-HNPABX7L.js.map → chunk-7EZR47JV.js.map} +1 -1
- package/dist/chunk-B77KABWH.js +41 -0
- package/dist/chunk-B77KABWH.js.map +1 -0
- package/dist/{chunk-MY36UPPQ.js → chunk-DI3H6JVZ.js} +357 -379
- package/dist/chunk-DI3H6JVZ.js.map +1 -0
- package/dist/{chunk-DDECTPRM.js → chunk-FGL32LUJ.js} +20 -14
- package/dist/chunk-FGL32LUJ.js.map +1 -0
- package/dist/{chunk-TTXVJFAE.js → chunk-JRSKWF6K.js} +4 -3
- package/dist/{chunk-TTXVJFAE.js.map → chunk-JRSKWF6K.js.map} +1 -1
- package/dist/{chunk-YRGUIRN5.js → chunk-VQ27YUHH.js} +18 -14
- package/dist/chunk-VQ27YUHH.js.map +1 -0
- package/dist/{chunk-BNV43SEF.js → chunk-VVXYZIIB.js} +5 -5
- package/dist/chunk-VVXYZIIB.js.map +1 -0
- package/dist/chunk-ZNLN2VWV.js +110 -0
- package/dist/chunk-ZNLN2VWV.js.map +1 -0
- package/dist/cli.js +102 -45
- package/dist/cli.js.map +1 -1
- package/dist/code.d.ts +4 -2
- package/dist/code.js +1 -1
- package/dist/docs.d.ts +7 -3
- package/dist/docs.js +1 -1
- package/dist/git.d.ts +4 -2
- package/dist/git.js +1 -1
- package/dist/index.d.ts +77 -17
- package/dist/index.js +21 -9
- package/dist/index.js.map +1 -1
- package/dist/local-embedding-ZIMTK6PU.js +8 -0
- package/dist/local-embedding-ZIMTK6PU.js.map +1 -0
- package/dist/memory.d.ts +2 -2
- package/dist/memory.js +1 -1
- package/dist/notes.d.ts +2 -2
- package/dist/notes.js +1 -1
- package/dist/qwen3-reranker-3MHEENT5.js +8 -0
- package/dist/qwen3-reranker-3MHEENT5.js.map +1 -0
- package/dist/resolve-CUJWY6HP.js +10 -0
- package/dist/resolve-CUJWY6HP.js.map +1 -0
- package/package.json +10 -9
- package/dist/chunk-BNV43SEF.js.map +0 -1
- package/dist/chunk-DDECTPRM.js.map +0 -1
- package/dist/chunk-MY36UPPQ.js.map +0 -1
- package/dist/chunk-YRGUIRN5.js.map +0 -1
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import {
|
|
2
|
+
__name
|
|
3
|
+
} from "./chunk-7QVYU63E.js";
|
|
4
|
+
|
|
5
|
+
// src/providers/rerankers/qwen3-reranker.ts
|
|
6
|
+
import { homedir } from "os";
|
|
7
|
+
import { join } from "path";
|
|
8
|
+
import { existsSync, mkdirSync } from "fs";
|
|
9
|
+
var DEFAULT_MODEL_URI = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf";
|
|
10
|
+
var CONTEXT_SIZE = 2048;
|
|
11
|
+
var MODEL_CACHE_DIR = join(homedir(), ".cache", "brainbank", "models");
|
|
12
|
+
var Qwen3Reranker = class {
|
|
13
|
+
static {
|
|
14
|
+
__name(this, "Qwen3Reranker");
|
|
15
|
+
}
|
|
16
|
+
_modelUri;
|
|
17
|
+
_cacheDir;
|
|
18
|
+
_contextSize;
|
|
19
|
+
_model = null;
|
|
20
|
+
_context = null;
|
|
21
|
+
_loadPromise = null;
|
|
22
|
+
constructor(options = {}) {
|
|
23
|
+
this._modelUri = options.modelUri ?? DEFAULT_MODEL_URI;
|
|
24
|
+
this._cacheDir = options.cacheDir ?? MODEL_CACHE_DIR;
|
|
25
|
+
this._contextSize = options.contextSize ?? CONTEXT_SIZE;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Lazy-load the reranker model and create a ranking context.
|
|
29
|
+
* Model is auto-downloaded from HuggingFace on first use.
|
|
30
|
+
*/
|
|
31
|
+
async _ensureLoaded() {
|
|
32
|
+
if (this._context) return;
|
|
33
|
+
if (this._loadPromise) {
|
|
34
|
+
await this._loadPromise;
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
this._loadPromise = (async () => {
|
|
38
|
+
try {
|
|
39
|
+
const llamaModule = "node-llama-cpp";
|
|
40
|
+
const { getLlama, resolveModelFile } = await import(
|
|
41
|
+
/* webpackIgnore: true */
|
|
42
|
+
llamaModule
|
|
43
|
+
);
|
|
44
|
+
if (!existsSync(this._cacheDir)) {
|
|
45
|
+
mkdirSync(this._cacheDir, { recursive: true });
|
|
46
|
+
}
|
|
47
|
+
const modelPath = await resolveModelFile(this._modelUri, this._cacheDir);
|
|
48
|
+
const llama = await getLlama();
|
|
49
|
+
this._model = await llama.loadModel({ modelPath });
|
|
50
|
+
try {
|
|
51
|
+
this._context = await this._model.createRankingContext({
|
|
52
|
+
contextSize: this._contextSize,
|
|
53
|
+
flashAttention: true
|
|
54
|
+
});
|
|
55
|
+
} catch {
|
|
56
|
+
this._context = await this._model.createRankingContext({
|
|
57
|
+
contextSize: this._contextSize
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
} finally {
|
|
61
|
+
this._loadPromise = null;
|
|
62
|
+
}
|
|
63
|
+
})();
|
|
64
|
+
await this._loadPromise;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Score each document's relevance to the query.
|
|
68
|
+
* Returns scores (0.0 - 1.0) in same order as input documents.
|
|
69
|
+
*
|
|
70
|
+
* Deduplicates identical documents to avoid redundant computation.
|
|
71
|
+
*/
|
|
72
|
+
async rank(query, documents) {
|
|
73
|
+
if (documents.length === 0) return [];
|
|
74
|
+
await this._ensureLoaded();
|
|
75
|
+
const uniqueTexts = [...new Set(documents)];
|
|
76
|
+
const textToScore = /* @__PURE__ */ new Map();
|
|
77
|
+
const truncated = uniqueTexts.map((text) => {
|
|
78
|
+
if (this._model) {
|
|
79
|
+
const tokens = this._model.tokenize(text);
|
|
80
|
+
const queryTokens = this._model.tokenize(query).length;
|
|
81
|
+
const maxDocTokens = this._contextSize - 200 - queryTokens;
|
|
82
|
+
if (tokens.length > maxDocTokens && maxDocTokens > 0) {
|
|
83
|
+
return this._model.detokenize(tokens.slice(0, maxDocTokens));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return text;
|
|
87
|
+
});
|
|
88
|
+
const scores = await this._context.rankAll(query, truncated);
|
|
89
|
+
for (let i = 0; i < uniqueTexts.length; i++) {
|
|
90
|
+
textToScore.set(uniqueTexts[i], scores[i] ?? 0);
|
|
91
|
+
}
|
|
92
|
+
return documents.map((doc) => textToScore.get(doc) ?? 0);
|
|
93
|
+
}
|
|
94
|
+
/** Release model resources. */
|
|
95
|
+
async close() {
|
|
96
|
+
if (this._context) {
|
|
97
|
+
await this._context.dispose();
|
|
98
|
+
this._context = null;
|
|
99
|
+
}
|
|
100
|
+
if (this._model) {
|
|
101
|
+
await this._model.dispose?.();
|
|
102
|
+
this._model = null;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
export {
|
|
108
|
+
Qwen3Reranker
|
|
109
|
+
};
|
|
110
|
+
//# sourceMappingURL=chunk-ZNLN2VWV.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/providers/rerankers/qwen3-reranker.ts"],"sourcesContent":["/**\n * BrainBank — Qwen3 Local Reranker\n * \n * Cross-encoder reranker using Qwen3-Reranker-0.6B via node-llama-cpp.\n * Auto-downloads the GGUF model from HuggingFace (~640MB, cached).\n * \n * Based on QMD's reranker architecture:\n * - Lazy model loading (loads on first rank() call)\n * - Flash attention for 20× less VRAM\n * - Document deduplication (identical texts scored once)\n * - Tokenizer-based truncation for oversized documents\n */\n\nimport type { Reranker } from '@/types.ts';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport { existsSync, mkdirSync } from 'node:fs';\n\n// Default model — Qwen3-Reranker-0.6B quantized to Q8_0 (~640MB)\nconst DEFAULT_MODEL_URI = 'hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf';\n\n// Context size for reranking (Qwen3 template ~200 tokens overhead + query + doc)\nconst CONTEXT_SIZE = 2048;\n\n// Cache directory for downloaded models\nconst MODEL_CACHE_DIR = join(homedir(), '.cache', 'brainbank', 'models');\n\nexport interface Qwen3RerankerOptions {\n /** HuggingFace model URI. Default: Qwen3-Reranker-0.6B-Q8_0 */\n modelUri?: string;\n /** Model cache directory. Default: ~/.cache/brainbank/models/ */\n cacheDir?: string;\n /** Context size for ranking. Default: 2048 */\n contextSize?: number;\n}\n\nexport class Qwen3Reranker implements Reranker {\n private readonly _modelUri: string;\n private readonly _cacheDir: string;\n private readonly _contextSize: number;\n\n private _model: any = null;\n private _context: any = null;\n private _loadPromise: Promise<void> | null = null;\n\n constructor(options: Qwen3RerankerOptions = {}) {\n this._modelUri = options.modelUri ?? DEFAULT_MODEL_URI;\n this._cacheDir = options.cacheDir ?? MODEL_CACHE_DIR;\n this._contextSize = options.contextSize ?? CONTEXT_SIZE;\n }\n\n /**\n * Lazy-load the reranker model and create a ranking context.\n * Model is auto-downloaded from HuggingFace on first use.\n */\n private async _ensureLoaded(): Promise<void> {\n if (this._context) return;\n if (this._loadPromise) {\n await this._loadPromise;\n return;\n }\n\n this._loadPromise = (async () => {\n try {\n // Dynamic import — node-llama-cpp is an optional peer dependency.\n // String indirection prevents DTS from resolving the specifier at build time.\n const llamaModule = 'node-llama-cpp';\n const { getLlama, resolveModelFile } = await import(/* webpackIgnore: true */ llamaModule);\n\n // Ensure cache directory exists\n if (!existsSync(this._cacheDir)) {\n mkdirSync(this._cacheDir, { recursive: true });\n }\n\n // Download model if needed (resolveModelFile handles caching)\n const modelPath = await resolveModelFile(this._modelUri, this._cacheDir);\n\n // Initialize llama engine\n const llama = await getLlama();\n\n // Load model\n this._model = await llama.loadModel({ modelPath });\n\n // Create ranking context with flash attention for lower VRAM\n try {\n this._context = await this._model.createRankingContext({\n contextSize: this._contextSize,\n flashAttention: true,\n });\n } catch {\n // Flash attention might not be supported — retry without it\n this._context = await this._model.createRankingContext({\n contextSize: this._contextSize,\n });\n }\n } finally {\n this._loadPromise = null;\n }\n })();\n\n await this._loadPromise;\n }\n\n /**\n * Score each document's relevance to the query.\n * Returns scores (0.0 - 1.0) in same order as input documents.\n * \n * Deduplicates identical documents to avoid redundant computation.\n */\n async rank(query: string, documents: string[]): Promise<number[]> {\n if (documents.length === 0) return [];\n\n await this._ensureLoaded();\n\n // Deduplicate — identical texts get scored once\n const uniqueTexts = [...new Set(documents)];\n const textToScore = new Map<string, number>();\n\n // Truncate documents that exceed context size\n const truncated = uniqueTexts.map(text => {\n if (this._model) {\n const tokens = this._model.tokenize(text);\n // Budget: contextSize - ~200 overhead - query tokens\n const queryTokens = this._model.tokenize(query).length;\n const maxDocTokens = this._contextSize - 200 - queryTokens;\n if (tokens.length > maxDocTokens && maxDocTokens > 0) {\n return this._model.detokenize(tokens.slice(0, maxDocTokens));\n }\n }\n return text;\n });\n\n // Rank all unique documents at once\n const scores: number[] = await this._context.rankAll(query, truncated);\n\n // Map scores back\n for (let i = 0; i < uniqueTexts.length; i++) {\n textToScore.set(uniqueTexts[i], scores[i] ?? 0);\n }\n\n // Return scores in original document order\n return documents.map(doc => textToScore.get(doc) ?? 0);\n }\n\n /** Release model resources. */\n async close(): Promise<void> {\n if (this._context) {\n await this._context.dispose();\n this._context = null;\n }\n if (this._model) {\n await this._model.dispose?.();\n this._model = null;\n }\n }\n}\n"],"mappings":";;;;;AAcA,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,YAAY,iBAAiB;AAGtC,IAAM,oBAAoB;AAG1B,IAAM,eAAe;AAGrB,IAAM,kBAAkB,KAAK,QAAQ,GAAG,UAAU,aAAa,QAAQ;AAWhE,IAAM,gBAAN,MAAwC;AAAA,EApC/C,OAoC+C;AAAA;AAAA;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EAET,SAAc;AAAA,EACd,WAAgB;AAAA,EAChB,eAAqC;AAAA,EAE7C,YAAY,UAAgC,CAAC,GAAG;AAC5C,SAAK,YAAY,QAAQ,YAAY;AACrC,SAAK,YAAY,QAAQ,YAAY;AACrC,SAAK,eAAe,QAAQ,eAAe;AAAA,EAC/C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,gBAA+B;AACzC,QAAI,KAAK,SAAU;AACnB,QAAI,KAAK,cAAc;AACnB,YAAM,KAAK;AACX;AAAA,IACJ;AAEA,SAAK,gBAAgB,YAAY;AAC7B,UAAI;AAGA,cAAM,cAAc;AACpB,cAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;AAAA;AAAA,UAAiC;AAAA;AAG9E,YAAI,CAAC,WAAW,KAAK,SAAS,GAAG;AAC7B,oBAAU,KAAK,WAAW,EAAE,WAAW,KAAK,CAAC;AAAA,QACjD;AAGA,cAAM,YAAY,MAAM,iBAAiB,KAAK,WAAW,KAAK,SAAS;AAGvE,cAAM,QAAQ,MAAM,SAAS;AAG7B,aAAK,SAAS,MAAM,MAAM,UAAU,EAAE,UAAU,CAAC;AAGjD,YAAI;AACA,eAAK,WAAW,MAAM,KAAK,OAAO,qBAAqB;AAAA,YACnD,aAAa,KAAK;AAAA,YAClB,gBAAgB;AAAA,UACpB,CAAC;AAAA,QACL,QAAQ;AAEJ,eAAK,WAAW,MAAM,KAAK,OAAO,qBAAqB;AAAA,YACnD,aAAa,KAAK;AAAA,UACtB,CAAC;AAAA,QACL;AAAA,MACJ,UAAE;AACE,aAAK,eAAe;AAAA,MACxB;AAAA,IACJ,GAAG;AAEH,UAAM,KAAK;AAAA,EACf;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,KAAK,OAAe,WAAwC;AAC9D,QAAI,UAAU,WAAW,EAAG,QAAO,CAAC;AAEpC,UAAM,KAAK,cAAc;AAGzB,UAAM,cAAc,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC;AAC1C,UAAM,cAAc,oBAAI,IAAoB;AAG5C,UAAM,YAAY,YAAY,IAAI,UAAQ;AACtC,UAAI,KAAK,QAAQ;AACb,cAAM,SAAS,KAAK,OAAO,SAAS,IAAI;AAExC,cAAM,cAAc,KAAK,OAAO,SAAS,KAAK,EAAE;AAChD,cAAM,eAAe,KAAK,eAAe,MAAM;AAC/C,YAAI,OAAO,SAAS,gBAAgB,eAAe,GAAG;AAClD,iBAAO,KAAK,OAAO,WAAW,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,QAC/D;AAAA,MACJ;AACA,aAAO;AAAA,IACX,CAAC;AAGD,UAAM,SAAmB,MAAM,KAAK,SAAS,QAAQ,OAAO,SAAS;AAGrE,aAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AACzC,kBAAY,IAAI,YAAY,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC;AAAA,IAClD;AAGA,WAAO,UAAU,IAAI,SAAO,YAAY,IAAI,GAAG,KAAK,CAAC;AAAA,EACzD;AAAA;AAAA,EAGA,MAAM,QAAuB;AACzB,QAAI,KAAK,UAAU;AACf,YAAM,KAAK,SAAS,QAAQ;AAC5B,WAAK,WAAW;AAAA,IACpB;AACA,QAAI,KAAK,QAAQ;AACb,YAAM,KAAK,OAAO,UAAU;AAC5B,WAAK,SAAS;AAAA,IAClB;AAAA,EACJ;AACJ;","names":[]}
|
package/dist/cli.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
BrainBank
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-DI3H6JVZ.js";
|
|
5
|
+
import "./chunk-B77KABWH.js";
|
|
5
6
|
import {
|
|
6
7
|
code
|
|
7
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-FGL32LUJ.js";
|
|
8
9
|
import {
|
|
9
10
|
git
|
|
10
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-JRSKWF6K.js";
|
|
11
12
|
import {
|
|
12
13
|
docs
|
|
13
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-VQ27YUHH.js";
|
|
14
15
|
import "./chunk-YOLKSYWK.js";
|
|
15
16
|
import "./chunk-U2Q2XGPZ.js";
|
|
16
17
|
import {
|
|
@@ -106,38 +107,51 @@ import * as path2 from "path";
|
|
|
106
107
|
// src/cli/factory.ts
|
|
107
108
|
import * as path from "path";
|
|
108
109
|
import * as fs from "fs";
|
|
109
|
-
var CONFIG_NAMES = ["config.ts", "config.js", "config.mjs"];
|
|
110
|
+
var CONFIG_NAMES = ["config.json", "config.ts", "config.js", "config.mjs"];
|
|
110
111
|
var INDEXER_EXTENSIONS = [".ts", ".js", ".mjs"];
|
|
111
112
|
var NOT_LOADED = /* @__PURE__ */ Symbol("not-loaded");
|
|
112
113
|
var _configCache = NOT_LOADED;
|
|
113
|
-
var
|
|
114
|
+
var _folderPluginsCache = NOT_LOADED;
|
|
114
115
|
async function loadConfig() {
|
|
115
116
|
if (_configCache !== NOT_LOADED) return _configCache;
|
|
116
117
|
const repoPath = getFlag("repo") ?? ".";
|
|
117
118
|
const brainbankDir = path.resolve(repoPath, ".brainbank");
|
|
118
119
|
for (const name of CONFIG_NAMES) {
|
|
119
120
|
const configPath = path.join(brainbankDir, name);
|
|
120
|
-
if (fs.existsSync(configPath))
|
|
121
|
-
|
|
121
|
+
if (!fs.existsSync(configPath)) continue;
|
|
122
|
+
try {
|
|
123
|
+
if (name === "config.json") {
|
|
124
|
+
const raw = fs.readFileSync(configPath, "utf-8");
|
|
125
|
+
_configCache = JSON.parse(raw);
|
|
126
|
+
} else {
|
|
122
127
|
const mod = await import(configPath);
|
|
123
128
|
_configCache = mod.default ?? mod;
|
|
124
|
-
return _configCache;
|
|
125
|
-
} catch (err) {
|
|
126
|
-
console.error(c.red(`Error loading .brainbank/${name}: ${err.message}`));
|
|
127
|
-
process.exit(1);
|
|
128
129
|
}
|
|
130
|
+
return _configCache;
|
|
131
|
+
} catch (err) {
|
|
132
|
+
console.error(c.red(`Error loading .brainbank/${name}: ${err.message}`));
|
|
133
|
+
process.exit(1);
|
|
129
134
|
}
|
|
130
135
|
}
|
|
131
136
|
_configCache = null;
|
|
132
137
|
return null;
|
|
133
138
|
}
|
|
134
139
|
__name(loadConfig, "loadConfig");
|
|
135
|
-
async function
|
|
136
|
-
|
|
140
|
+
async function getConfig() {
|
|
141
|
+
return loadConfig();
|
|
142
|
+
}
|
|
143
|
+
__name(getConfig, "getConfig");
|
|
144
|
+
async function resolveEmbeddingKey(key) {
|
|
145
|
+
const { resolveEmbedding } = await import("./resolve-CUJWY6HP.js");
|
|
146
|
+
return resolveEmbedding(key);
|
|
147
|
+
}
|
|
148
|
+
__name(resolveEmbeddingKey, "resolveEmbeddingKey");
|
|
149
|
+
async function discoverFolderPlugins() {
|
|
150
|
+
if (_folderPluginsCache !== NOT_LOADED) return _folderPluginsCache;
|
|
137
151
|
const repoPath = getFlag("repo") ?? ".";
|
|
138
152
|
const indexersDir = path.resolve(repoPath, ".brainbank", "indexers");
|
|
139
153
|
if (!fs.existsSync(indexersDir)) {
|
|
140
|
-
|
|
154
|
+
_folderPluginsCache = [];
|
|
141
155
|
return [];
|
|
142
156
|
}
|
|
143
157
|
const files = fs.readdirSync(indexersDir).filter((f) => INDEXER_EXTENSIONS.some((ext) => f.endsWith(ext))).sort();
|
|
@@ -150,16 +164,16 @@ async function discoverFolderIndexers() {
|
|
|
150
164
|
if (indexer && typeof indexer === "object" && indexer.name) {
|
|
151
165
|
indexers.push(indexer);
|
|
152
166
|
} else {
|
|
153
|
-
console.error(c.yellow(`\u26A0 ${file}: must export a default
|
|
167
|
+
console.error(c.yellow(`\u26A0 ${file}: must export a default Plugin with a 'name' property, skipping`));
|
|
154
168
|
}
|
|
155
169
|
} catch (err) {
|
|
156
170
|
console.error(c.red(`Error loading indexer ${file}: ${err.message}`));
|
|
157
171
|
}
|
|
158
172
|
}
|
|
159
|
-
|
|
173
|
+
_folderPluginsCache = indexers;
|
|
160
174
|
return indexers;
|
|
161
175
|
}
|
|
162
|
-
__name(
|
|
176
|
+
__name(discoverFolderPlugins, "discoverFolderPlugins");
|
|
163
177
|
function detectGitSubdirs(parentPath) {
|
|
164
178
|
try {
|
|
165
179
|
const entries = fs.readdirSync(parentPath, { withFileTypes: true });
|
|
@@ -174,12 +188,13 @@ __name(detectGitSubdirs, "detectGitSubdirs");
|
|
|
174
188
|
async function createBrain(repoPath) {
|
|
175
189
|
const rp = repoPath ?? getFlag("repo") ?? ".";
|
|
176
190
|
const config = await loadConfig();
|
|
177
|
-
const folderIndexers = await
|
|
191
|
+
const folderIndexers = await discoverFolderPlugins();
|
|
178
192
|
const brainOpts = { repoPath: rp, ...config?.brainbank ?? {} };
|
|
179
|
-
|
|
193
|
+
if (config?.maxFileSize) brainOpts.maxFileSize = config.maxFileSize;
|
|
194
|
+
await setupProviders(brainOpts, config);
|
|
180
195
|
const brain = new BrainBank(brainOpts);
|
|
181
|
-
const builtins = config?.builtins ?? ["code", "git", "docs"];
|
|
182
|
-
registerBuiltins(brain, rp, builtins);
|
|
196
|
+
const builtins = config?.plugins ?? config?.builtins ?? ["code", "git", "docs"];
|
|
197
|
+
await registerBuiltins(brain, rp, builtins, config);
|
|
183
198
|
for (const indexer of folderIndexers) brain.use(indexer);
|
|
184
199
|
if (config?.indexers) {
|
|
185
200
|
for (const indexer of config.indexers) brain.use(indexer);
|
|
@@ -187,47 +202,87 @@ async function createBrain(repoPath) {
|
|
|
187
202
|
return brain;
|
|
188
203
|
}
|
|
189
204
|
__name(createBrain, "createBrain");
|
|
190
|
-
async function setupProviders(brainOpts) {
|
|
191
|
-
const rerankerFlag = getFlag("reranker");
|
|
205
|
+
async function setupProviders(brainOpts, config) {
|
|
206
|
+
const rerankerFlag = getFlag("reranker") ?? config?.reranker;
|
|
192
207
|
if (rerankerFlag === "qwen3") {
|
|
193
|
-
const { Qwen3Reranker } = await import("
|
|
208
|
+
const { Qwen3Reranker } = await import("./qwen3-reranker-3MHEENT5.js");
|
|
194
209
|
brainOpts.reranker = new Qwen3Reranker();
|
|
195
210
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
const provider =
|
|
199
|
-
brainOpts.embeddingProvider = provider;
|
|
200
|
-
brainOpts.embeddingDims = provider.dims;
|
|
201
|
-
} else if (process.env.BRAINBANK_EMBEDDING === "perplexity") {
|
|
202
|
-
const { PerplexityEmbedding } = await import("./perplexity-embedding-227WQY4R.js");
|
|
203
|
-
const provider = new PerplexityEmbedding();
|
|
204
|
-
brainOpts.embeddingProvider = provider;
|
|
205
|
-
brainOpts.embeddingDims = provider.dims;
|
|
206
|
-
} else if (process.env.BRAINBANK_EMBEDDING === "perplexity-context") {
|
|
207
|
-
const { PerplexityContextEmbedding } = await import("./perplexity-context-embedding-KSVSZXMD.js");
|
|
208
|
-
const provider = new PerplexityContextEmbedding();
|
|
211
|
+
const embFlag = getFlag("embedding") ?? config?.embedding;
|
|
212
|
+
if (embFlag) {
|
|
213
|
+
const provider = await resolveEmbeddingKey(embFlag);
|
|
209
214
|
brainOpts.embeddingProvider = provider;
|
|
210
215
|
brainOpts.embeddingDims = provider.dims;
|
|
211
216
|
}
|
|
212
217
|
}
|
|
213
218
|
__name(setupProviders, "setupProviders");
|
|
214
|
-
function registerBuiltins(brain, rp, builtins) {
|
|
219
|
+
async function registerBuiltins(brain, rp, builtins, config) {
|
|
215
220
|
const resolvedRp = path.resolve(rp);
|
|
216
221
|
const hasRootGit = fs.existsSync(path.join(resolvedRp, ".git"));
|
|
217
222
|
const gitSubdirs = !hasRootGit ? detectGitSubdirs(resolvedRp) : [];
|
|
223
|
+
const codeEmb = config?.code?.embedding ? await resolveEmbeddingKey(config.code.embedding) : void 0;
|
|
224
|
+
const gitEmb = config?.git?.embedding ? await resolveEmbeddingKey(config.git.embedding) : void 0;
|
|
225
|
+
const docsEmb = config?.docs?.embedding ? await resolveEmbeddingKey(config.docs.embedding) : void 0;
|
|
218
226
|
if (gitSubdirs.length > 0 && (builtins.includes("code") || builtins.includes("git"))) {
|
|
219
227
|
console.log(c.cyan(` Multi-repo: found ${gitSubdirs.length} git repos: ${gitSubdirs.map((d) => d.name).join(", ")}`));
|
|
220
228
|
for (const sub of gitSubdirs) {
|
|
221
|
-
if (builtins.includes("code"))
|
|
222
|
-
|
|
229
|
+
if (builtins.includes("code")) {
|
|
230
|
+
brain.use(code({
|
|
231
|
+
repoPath: sub.path,
|
|
232
|
+
name: `code:${sub.name}`,
|
|
233
|
+
embeddingProvider: codeEmb,
|
|
234
|
+
maxFileSize: config?.code?.maxFileSize
|
|
235
|
+
}));
|
|
236
|
+
}
|
|
237
|
+
if (builtins.includes("git")) {
|
|
238
|
+
brain.use(git({
|
|
239
|
+
repoPath: sub.path,
|
|
240
|
+
name: `git:${sub.name}`,
|
|
241
|
+
embeddingProvider: gitEmb,
|
|
242
|
+
depth: config?.git?.depth,
|
|
243
|
+
maxDiffBytes: config?.git?.maxDiffBytes
|
|
244
|
+
}));
|
|
245
|
+
}
|
|
223
246
|
}
|
|
224
247
|
} else {
|
|
225
|
-
if (builtins.includes("code"))
|
|
226
|
-
|
|
248
|
+
if (builtins.includes("code")) {
|
|
249
|
+
brain.use(code({
|
|
250
|
+
repoPath: rp,
|
|
251
|
+
embeddingProvider: codeEmb,
|
|
252
|
+
maxFileSize: config?.code?.maxFileSize
|
|
253
|
+
}));
|
|
254
|
+
}
|
|
255
|
+
if (builtins.includes("git")) {
|
|
256
|
+
brain.use(git({
|
|
257
|
+
embeddingProvider: gitEmb,
|
|
258
|
+
depth: config?.git?.depth,
|
|
259
|
+
maxDiffBytes: config?.git?.maxDiffBytes
|
|
260
|
+
}));
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
if (builtins.includes("docs")) {
|
|
264
|
+
brain.use(docs({ embeddingProvider: docsEmb }));
|
|
227
265
|
}
|
|
228
|
-
if (builtins.includes("docs")) brain.use(docs());
|
|
229
266
|
}
|
|
230
267
|
__name(registerBuiltins, "registerBuiltins");
|
|
268
|
+
async function registerConfigCollections(brain, config) {
|
|
269
|
+
const collections = config?.docs?.collections;
|
|
270
|
+
if (!collections?.length) return;
|
|
271
|
+
for (const coll of collections) {
|
|
272
|
+
const absPath = path.resolve(coll.path);
|
|
273
|
+
try {
|
|
274
|
+
await brain.addCollection({
|
|
275
|
+
name: coll.name,
|
|
276
|
+
path: absPath,
|
|
277
|
+
pattern: coll.pattern ?? "**/*.md",
|
|
278
|
+
ignore: coll.ignore,
|
|
279
|
+
context: coll.context
|
|
280
|
+
});
|
|
281
|
+
} catch {
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
__name(registerConfigCollections, "registerConfigCollections");
|
|
231
286
|
|
|
232
287
|
// src/cli/commands/index-cmd.ts
|
|
233
288
|
async function cmdIndex() {
|
|
@@ -247,6 +302,8 @@ async function cmdIndex() {
|
|
|
247
302
|
if (modules) console.log(c.dim(` Modules: ${modules.join(", ")}`));
|
|
248
303
|
if (docsPath) console.log(c.dim(` Docs path: ${docsPath}`));
|
|
249
304
|
const brain = await createBrain(repoPath);
|
|
305
|
+
const config = await getConfig();
|
|
306
|
+
await registerConfigCollections(brain, config);
|
|
250
307
|
if (docsPath) {
|
|
251
308
|
const absDocsPath = path2.resolve(docsPath);
|
|
252
309
|
const collName = path2.basename(absDocsPath);
|
|
@@ -629,7 +686,7 @@ async function cmdStats() {
|
|
|
629
686
|
await brain.initialize();
|
|
630
687
|
const s = brain.stats();
|
|
631
688
|
console.log(c.bold("\n\u2501\u2501\u2501 BrainBank Stats \u2501\u2501\u2501\n"));
|
|
632
|
-
console.log(` ${c.cyan("
|
|
689
|
+
console.log(` ${c.cyan("Plugins")}: ${brain.plugins.join(", ")}
|
|
633
690
|
`);
|
|
634
691
|
if (s.code) {
|
|
635
692
|
console.log(` ${c.cyan("Code")}`);
|