crbro-memory 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -8
- package/bin/crbro.mjs +109 -0
- package/dist/engine/brain.d.ts +3 -0
- package/dist/engine/brain.d.ts.map +1 -1
- package/dist/engine/brain.js +15 -1
- package/dist/engine/brain.js.map +1 -1
- package/dist/engine/cortex.d.ts +61 -7
- package/dist/engine/cortex.d.ts.map +1 -1
- package/dist/engine/cortex.js +168 -23
- package/dist/engine/cortex.js.map +1 -1
- package/dist/engine/maintenance.d.ts +6 -1
- package/dist/engine/maintenance.d.ts.map +1 -1
- package/dist/engine/maintenance.js +19 -5
- package/dist/engine/maintenance.js.map +1 -1
- package/dist/engine/prefrontal.d.ts +13 -2
- package/dist/engine/prefrontal.d.ts.map +1 -1
- package/dist/engine/prefrontal.js +46 -10
- package/dist/engine/prefrontal.js.map +1 -1
- package/dist/miner/index.d.ts.map +1 -1
- package/dist/miner/index.js +35 -30
- package/dist/miner/index.js.map +1 -1
- package/dist/search/index.d.ts +46 -6
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +335 -74
- package/dist/search/index.js.map +1 -1
- package/dist/search/tokenize.d.ts +23 -0
- package/dist/search/tokenize.d.ts.map +1 -0
- package/dist/search/tokenize.js +83 -0
- package/dist/search/tokenize.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +125 -14
- package/dist/server.js.map +1 -1
- package/dist/types/index.d.ts +39 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/utils/fs.d.ts +3 -1
- package/dist/utils/fs.d.ts.map +1 -1
- package/dist/utils/fs.js +6 -2
- package/dist/utils/fs.js.map +1 -1
- package/dist/utils/hash.d.ts +20 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +45 -0
- package/dist/utils/hash.js.map +1 -0
- package/package.json +53 -53
package/dist/miner/index.js
CHANGED
|
@@ -177,66 +177,71 @@ class Miner {
|
|
|
177
177
|
* Feed extracted content into CRBRO's cortex.
|
|
178
178
|
*/
|
|
179
179
|
async feedCortex(extracted, sourcePath) {
|
|
180
|
-
|
|
180
|
+
// The miner no longer creates neurons, so this stays at zero.
|
|
181
|
+
const created = 0;
|
|
181
182
|
let updated = 0;
|
|
182
183
|
let factsAdded = 0;
|
|
183
184
|
let decisionsAdded = 0;
|
|
184
|
-
//
|
|
185
|
+
// The miner ENRICHES neurons; it does not invent them.
|
|
186
|
+
//
|
|
187
|
+
// It used to create one per detected topic, which on a real brain produced
|
|
188
|
+
// roughly a thousand junk neurons (`lang_juego_billar_8_ball`,
|
|
189
|
+
// `lang_current_goal`, a markdown heading turned into `project_findings`
|
|
190
|
+
// that then acted as a magnet for unrelated writes). They were all tiny, so
|
|
191
|
+
// they outranked the real knowledge, and they poisoned name resolution.
|
|
192
|
+
// A miss is now simply skipped: neurons are created by a human calling
|
|
193
|
+
// crbro_learn, never by a background pass guessing at topic names.
|
|
194
|
+
const MINER = { source: 'miner', createIfMissing: false };
|
|
195
|
+
// Technologies: recorded only against a neuron that already exists.
|
|
196
|
+
// The old "Referenced in: <file>" note is gone entirely -- it carried no
|
|
197
|
+
// knowledge and accounted for 708 of 4,273 facts on the reference brain.
|
|
185
198
|
for (const tech of extracted.technologies) {
|
|
186
|
-
const neuronId = `tech_${tech.replace(/[^a-z0-9]/gi, '_').toLowerCase()}`;
|
|
187
199
|
try {
|
|
188
|
-
const
|
|
189
|
-
if (
|
|
190
|
-
|
|
191
|
-
domain: 'conocimiento',
|
|
192
|
-
});
|
|
193
|
-
created++;
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
catch {
|
|
197
|
-
// Neuron might not exist yet
|
|
198
|
-
try {
|
|
199
|
-
await this.cortex.learn(tech, 'fact', `Referenced in: ${(0, path_1.basename)(sourcePath)}`, {
|
|
200
|
-
domain: 'conocimiento',
|
|
201
|
-
});
|
|
202
|
-
created++;
|
|
203
|
-
}
|
|
204
|
-
catch { /* skip */ }
|
|
200
|
+
const result = await this.cortex.learn(tech, 'fact', `Used in ${(0, path_1.basename)(sourcePath)}.`, { ...MINER, domain: 'conocimiento', confidence: 0.6 });
|
|
201
|
+
if (result.action === 'updated')
|
|
202
|
+
updated++;
|
|
205
203
|
}
|
|
204
|
+
catch { /* skip */ }
|
|
206
205
|
}
|
|
207
|
-
//
|
|
206
|
+
// Topic summaries, into existing neurons only.
|
|
208
207
|
for (const topic of extracted.topics.slice(0, 3)) {
|
|
209
208
|
if (topic.length < 5)
|
|
210
209
|
continue;
|
|
210
|
+
if (!extracted.summary)
|
|
211
|
+
continue;
|
|
211
212
|
try {
|
|
212
|
-
const result = await this.cortex.learn(topic, 'fact', extracted.summary
|
|
213
|
+
const result = await this.cortex.learn(topic, 'fact', extracted.summary, {
|
|
214
|
+
...MINER,
|
|
213
215
|
domain: 'general',
|
|
216
|
+
confidence: 0.7,
|
|
214
217
|
});
|
|
215
|
-
if (result.action === '
|
|
216
|
-
created++;
|
|
217
|
-
else
|
|
218
|
+
if (result.action === 'updated')
|
|
218
219
|
updated++;
|
|
219
220
|
}
|
|
220
221
|
catch { /* skip duplicates */ }
|
|
221
222
|
}
|
|
222
|
-
//
|
|
223
|
+
// Facts and decisions attach to the primary topic, again only if it exists.
|
|
223
224
|
if (extracted.topics.length > 0) {
|
|
224
225
|
const primaryTopic = extracted.topics[0];
|
|
225
226
|
for (const fact of extracted.facts.slice(0, 5)) {
|
|
226
227
|
try {
|
|
227
|
-
await this.cortex.learn(primaryTopic, 'fact', fact, {
|
|
228
|
+
const result = await this.cortex.learn(primaryTopic, 'fact', fact, {
|
|
229
|
+
...MINER,
|
|
228
230
|
confidence: 0.7, // Lower confidence for auto-mined data
|
|
229
231
|
});
|
|
230
|
-
|
|
232
|
+
if (result.action !== 'skipped')
|
|
233
|
+
factsAdded++;
|
|
231
234
|
}
|
|
232
235
|
catch { /* skip */ }
|
|
233
236
|
}
|
|
234
237
|
for (const decision of extracted.decisions.slice(0, 3)) {
|
|
235
238
|
try {
|
|
236
|
-
await this.cortex.learn(primaryTopic, 'decision', decision, {
|
|
239
|
+
const result = await this.cortex.learn(primaryTopic, 'decision', decision, {
|
|
240
|
+
...MINER,
|
|
237
241
|
rationale: `Auto-mined from ${(0, path_1.basename)(sourcePath)}`,
|
|
238
242
|
});
|
|
239
|
-
|
|
243
|
+
if (result.action !== 'skipped')
|
|
244
|
+
decisionsAdded++;
|
|
240
245
|
}
|
|
241
246
|
catch { /* skip */ }
|
|
242
247
|
}
|
package/dist/miner/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/miner/index.ts"],"names":[],"mappings":";AAAA,iEAAiE;AACjE,yDAAyD;AACzD,2CAA2C;;;AA+C3C,wCA0BC;AAvED,0CAAwE;AACxE,+BAA+C;AAC/C,2BAA6B;AAC7B,2BAAgC;AAChC,iDAAuE;AACvE,iDAA2C;AAC3C,mDAA6C;AAsB7C,MAAM,cAAc,GAAgB;IAClC,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE;QACf,YAAY;QACZ,WAAW;QACX,MAAM;QACN,cAAc;QACd,MAAM;QACN,kBAAkB;KACnB;CACF,CAAC;AAEF;;GAEG;AACH,SAAgB,cAAc;IAC5B,MAAM,IAAI,GAAG,IAAA,YAAO,GAAE,CAAC;IACvB,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,MAAM,UAAU,GAAG;QACjB,8BAA8B;QAC9B,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC;QAC7C,SAAS;QACT,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,EAAE,eAAe,CAAC;QACtC,WAAW;QACX,IAAA,WAAI,EAAC,IAAI,EAAE,WAAW,EAAE,eAAe,CAAC;QACxC,2BAA2B;QAC3B,IAAA,WAAI,EAAC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC;QAC1D,yBAAyB;QACzB,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,EAAE,qBAAqB,EAAE,QAAQ,EAAE,eAAe,CAAC;QACvE,qBAAqB;QACrB,IAAA,WAAI,EAAC,IAAI,EAAE,WAAW,EAAE,UAAU,CAAC;KACpC,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,GAAG,IAAI,IAAA,eAAU,EAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAa,KAAK;IACR,KAAK,CAAQ;IACb,MAAM,CAAS;IACf,MAAM,CAAc;IACpB,SAAS,CAAS;IAE1B,YAAY,MAA6B;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,gBAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,kBAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;QAEjE,yCAAyC;QACzC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,cAAc,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,SAAkB;QAC3B,MAAM,MAAM,GAAe;YACzB,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,WAAW,EAAE,CAAC;YACd,eAAe,EAAE,CAAC;YAClB,kBAAkB,EAAE,EAAE;YACtB,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,aAAa;QACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAErC,sCAAsC;QACtC,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAE5D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;YAClG,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,IAAA,eAAU,EAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAC;gBAClD,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;gBAE3C,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;oBAC7B,MAAM,CAAC,OAAO,EAAE,CAAC;oBAEjB,qCAAqC;oBACrC,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAI,EAAC,QAAQ,CAAC,CAAC;oBACtC,MAAM,IAAI,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAEhE,IAAI,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;wBACzC,SAAS,CAAC,sBAAsB;oBAClC,CAAC;oBAED,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAClD,MAAM,SAAS,GAAG,IAAA,6BAAc,EAAC,OAAO,EAAE,IAAA,eAAQ,EAAC,QAAQ,CAAC,CAAC,CAAC;wBAE9D,kBAAkB;wBAClB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;wBAE9D,MAAM,CAAC,SAAS,EAAE,CAAC;wBACnB,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC,OAAO,CAAC;wBAC7C,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC,OAAO,CAAC;wBAC7C,MAAM,CAAC,WAAW,IAAI,UAAU,CAAC,KAAK,CAAC;wBACvC,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC,SAAS,CAAC;wBAE/C,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;4BAC1C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;wBACrB,CAAC;wBAED,gBAAgB;wBAChB,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;oBACrC,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC1G,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;QACH,CAAC;QAED,MAAM,CAAC,kBAAkB,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;QAE1C,aAAa;QACb,KAAK,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS,CAAC;QACtC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAE5B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAO,EAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YAE5D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAEvC,2BAA2B;gBAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClE,SAAS;gBACX,CAAC;gBAED,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;oBACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;oBACnD,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC5B,CAAC;qBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;oBAC1B,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAEpD,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAI,EAAC,QAAQ,CAAC,CAAC;oBACtC,IAAI,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW;wBAAE,SAAS;oBAEtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CACtB,SAA2B,EAC3B,UAAkB;QAElB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/miner/index.ts"],"names":[],"mappings":";AAAA,iEAAiE;AACjE,yDAAyD;AACzD,2CAA2C;;;AA+C3C,wCA0BC;AAvED,0CAAwE;AACxE,+BAA+C;AAC/C,2BAA6B;AAC7B,2BAAgC;AAChC,iDAAuE;AACvE,iDAA2C;AAC3C,mDAA6C;AAsB7C,MAAM,cAAc,GAAgB;IAClC,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE;QACf,YAAY;QACZ,WAAW;QACX,MAAM;QACN,cAAc;QACd,MAAM;QACN,kBAAkB;KACnB;CACF,CAAC;AAEF;;GAEG;AACH,SAAgB,cAAc;IAC5B,MAAM,IAAI,GAAG,IAAA,YAAO,GAAE,CAAC;IACvB,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,MAAM,UAAU,GAAG;QACjB,8BAA8B;QAC9B,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC;QAC7C,SAAS;QACT,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,EAAE,eAAe,CAAC;QACtC,WAAW;QACX,IAAA,WAAI,EAAC,IAAI,EAAE,WAAW,EAAE,eAAe,CAAC;QACxC,2BAA2B;QAC3B,IAAA,WAAI,EAAC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC;QAC1D,yBAAyB;QACzB,IAAA,WAAI,EAAC,IAAI,EAAE,SAAS,EAAE,qBAAqB,EAAE,QAAQ,EAAE,eAAe,CAAC;QACvE,qBAAqB;QACrB,IAAA,WAAI,EAAC,IAAI,EAAE,WAAW,EAAE,UAAU,CAAC;KACpC,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,GAAG,IAAI,IAAA,eAAU,EAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAa,KAAK;IACR,KAAK,CAAQ;IACb,MAAM,CAAS;IACf,MAAM,CAAc;IACpB,SAAS,CAAS;IAE1B,YAAY,MAA6B;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,gBAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,kBAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;QAEjE,yCAAyC;QACzC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,cAAc,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,SAAkB;QAC3B,MAAM,MAAM,GAAe;YACzB,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,WAAW,EAAE,CAAC;YACd,eAAe,EAAE,CAAC;YAClB,kBAAkB,EAAE,EAAE;YACtB,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,aAAa;QACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAErC,sCAAsC;QACtC,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAE5D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;YAClG,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,IAAA,eAAU,EAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAC;gBAClD,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;gBAE3C,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;oBAC7B,MAAM,CAAC,OAAO,EAAE,CAAC;oBAEjB,qCAAqC;oBACrC,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAI,EAAC,QAAQ,CAAC,CAAC;oBACtC,MAAM,IAAI,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAEhE,IAAI,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;wBACzC,SAAS,CAAC,sBAAsB;oBAClC,CAAC;oBAED,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAClD,MAAM,SAAS,GAAG,IAAA,6BAAc,EAAC,OAAO,EAAE,IAAA,eAAQ,EAAC,QAAQ,CAAC,CAAC,CAAC;wBAE9D,kBAAkB;wBAClB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;wBAE9D,MAAM,CAAC,SAAS,EAAE,CAAC;wBACnB,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC,OAAO,CAAC;wBAC7C,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC,OAAO,CAAC;wBAC7C,MAAM,CAAC,WAAW,IAAI,UAAU,CAAC,KAAK,CAAC;wBACvC,MAAM,CAAC,eAAe,IAAI,UAAU,CAAC,SAAS,CAAC;wBAE/C,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;4BAC1C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;wBACrB,CAAC;wBAED,gBAAgB;wBAChB,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;oBACrC,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC1G,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;QACH,CAAC;QAED,MAAM,CAAC,kBAAkB,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;QAE1C,aAAa;QACb,KAAK,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS,CAAC;QACtC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAE5B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAO,EAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YAE5D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAEvC,2BAA2B;gBAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClE,SAAS;gBACX,CAAC;gBAED,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;oBACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;oBACnD,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC5B,CAAC;qBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;oBAC1B,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAEpD,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAI,EAAC,QAAQ,CAAC,CAAC;oBACtC,IAAI,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW;wBAAE,SAAS;oBAEtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CACtB,SAA2B,EAC3B,UAAkB;QAElB,8DAA8D;QAC9D,MAAM,OAAO,GAAG,CAAC,CAAC;QAClB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,uDAAuD;QACvD,EAAE;QACF,2EAA2E;QAC3E,+DAA+D;QAC/D,yEAAyE;QACzE,4EAA4E;QAC5E,wEAAwE;QACxE,uEAAuE;QACvE,mEAAmE;QACnE,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,KAAc,EAAE,CAAC;QAEnE,oEAAoE;QACpE,yEAAyE;QACzE,yEAAyE;QACzE,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CACpC,IAAI,EACJ,MAAM,EACN,WAAW,IAAA,eAAQ,EAAC,UAAU,CAAC,GAAG,EAClC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,EAAE,CACtD,CAAC;gBACF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;oBAAE,OAAO,EAAE,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;QACxB,CAAC;QAED,+CAA+C;QAC/C,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAC/B,IAAI,CAAC,SAAS,CAAC,OAAO;gBAAE,SAAS;YACjC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,OAAO,EAAE;oBACvE,GAAG,KAAK;oBACR,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;oBAAE,OAAO,EAAE,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;QACnC,CAAC;QAED,4EAA4E;QAC5E,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAEzC,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE;wBACjE,GAAG,KAAK;wBACR,UAAU,EAAE,GAAG,EAAE,uCAAuC;qBACzD,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;wBAAE,UAAU,EAAE,CAAC;gBAChD,CAAC;gBAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;YACxB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE;wBACzE,GAAG,KAAK;wBACR,SAAS,EAAE,mBAAmB,IAAA,eAAQ,EAAC,UAAU,CAAC,EAAE;qBACrD,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;wBAAE,cAAc,EAAE,CAAC;gBACpD,CAAC;gBAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC;YACH,IAAI,IAAA,eAAU,EAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/B,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACpD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,iCAAiC,CAAC,CAAC;QAE7C,OAAO;YACL,WAAW,EAAE,EAAE;YACf,QAAQ,EAAE,IAAI;YACd,WAAW,EAAE,CAAC;SACf,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,KAAiB;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,IAAA,eAAU,EAAC,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAA,gBAAK,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,MAAM,IAAA,oBAAS,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QAKb,OAAO;YACL,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YACrC,aAAa,EAAE,cAAc,EAAE;YAC/B,KAAK,EAAE,MAAM,IAAI,CAAC,SAAS,EAAE;SAC9B,CAAC;IACJ,CAAC;CACF;AAzQD,sBAyQC"}
|
package/dist/search/index.d.ts
CHANGED
|
@@ -1,31 +1,71 @@
|
|
|
1
1
|
import type { Brain } from '../engine/brain.js';
|
|
2
2
|
import type { Neuron, SearchResult } from '../types/index.js';
|
|
3
|
+
/** Bump when the schema changes so old on-disk indexes are rebuilt, not loaded. */
|
|
4
|
+
export declare const INDEX_VERSION = 2;
|
|
3
5
|
export declare class SearchEngine {
|
|
4
6
|
private brain;
|
|
5
7
|
private db;
|
|
8
|
+
private docCount;
|
|
9
|
+
private dirty;
|
|
10
|
+
private persistTimer;
|
|
6
11
|
constructor(brain: Brain);
|
|
7
12
|
/**
|
|
8
|
-
*
|
|
13
|
+
* Load the index from disk, or rebuild it if missing / stale / corrupt.
|
|
14
|
+
*
|
|
15
|
+
* Version check matters: Orama's `load()` does NOT throw when the stored
|
|
16
|
+
* schema differs — it silently overwrites the live schema with the old one,
|
|
17
|
+
* so the try/catch never fires and every later search fails with
|
|
18
|
+
* `Invalid property name`, permanently, on every boot. Reading the version
|
|
19
|
+
* ourselves is the only reliable guard.
|
|
9
20
|
*/
|
|
10
21
|
init(): Promise<void>;
|
|
11
22
|
/**
|
|
12
|
-
* Rebuild the
|
|
23
|
+
* Rebuild the whole index from the cortex.
|
|
24
|
+
* One bad neuron must not take the server down with it, so every neuron
|
|
25
|
+
* is isolated: CRBRO is published and other people's brains are not ours
|
|
26
|
+
* to assume well-formed.
|
|
13
27
|
*/
|
|
14
28
|
rebuild(): Promise<number>;
|
|
15
29
|
/**
|
|
16
|
-
*
|
|
30
|
+
* Re-index a single neuron: drop its old chunks, insert the current ones.
|
|
17
31
|
*/
|
|
18
32
|
indexNeuron(neuron: Neuron): Promise<void>;
|
|
33
|
+
/** Write the index to disk now. */
|
|
34
|
+
persist(): Promise<void>;
|
|
35
|
+
/** Flush if there are unsaved changes. Called on consolidate/shutdown. */
|
|
36
|
+
flush(): Promise<void>;
|
|
19
37
|
/**
|
|
20
|
-
*
|
|
38
|
+
* Find neurons matching a query.
|
|
39
|
+
*
|
|
40
|
+
* Late interaction: each query term is searched separately and its scores
|
|
41
|
+
* normalised against the best score for that term, then a chunk's score is
|
|
42
|
+
* the sum of its normalised hits weighted by how much of the query it
|
|
43
|
+
* covers. This is what stops one very common term from deciding the ranking
|
|
44
|
+
* and what makes a chunk matching five of six terms beat a scrap matching one.
|
|
21
45
|
*/
|
|
22
46
|
search(query: string, options?: {
|
|
23
47
|
domain?: string;
|
|
24
48
|
limit?: number;
|
|
25
49
|
}): Promise<SearchResult[]>;
|
|
26
50
|
/**
|
|
27
|
-
*
|
|
51
|
+
* Search one term. Exact first; only if a reasonably long term finds
|
|
52
|
+
* nothing do we allow a single edit of slack, which covers typos without
|
|
53
|
+
* the indiscriminate noise of the old `tolerance: 2` (measured: it matched
|
|
54
|
+
* 1,166 of 1,183 documents, i.e. the ranking was noise).
|
|
28
55
|
*/
|
|
29
|
-
|
|
56
|
+
private searchTerm;
|
|
57
|
+
/** Build and insert every chunk of a neuron. */
|
|
58
|
+
private insertNeuronChunks;
|
|
59
|
+
private put;
|
|
60
|
+
/** Drop every chunk belonging to a neuron. */
|
|
61
|
+
private removeNeuronChunks;
|
|
62
|
+
private markDirty;
|
|
63
|
+
private countDocs;
|
|
64
|
+
/**
|
|
65
|
+
* Remove the pre-v2 index once a rebuild has succeeded. It is dead weight
|
|
66
|
+
* (25 MB on the reference brain) and nothing reads it any more. Downgrading
|
|
67
|
+
* to an older CRBRO simply finds no index and rebuilds its own.
|
|
68
|
+
*/
|
|
69
|
+
private dropLegacyIndex;
|
|
30
70
|
}
|
|
31
71
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAQ,MAAM,mBAAmB,CAAC;AAEpE,mFAAmF;AACnF,eAAO,MAAM,aAAa,IAAI,CAAC;AAwC/B,qBAAa,YAAY;IAMX,OAAO,CAAC,KAAK;IALzB,OAAO,CAAC,EAAE,CAAyB;IACnC,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,YAAY,CAA+B;gBAE/B,KAAK,EAAE,KAAK;IAIhC;;;;;;;;OAQG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB3B;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAqBhC;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAShD,mCAAmC;IAC7B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAa9B,0EAA0E;IACpE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B;;;;;;;;OAQG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC5C,OAAO,CAAC,YAAY,EAAE,CAAC;IAiF1B;;;;;OAKG;YACW,UAAU;IAqBxB,gDAAgD;YAClC,kBAAkB;YA0ElB,GAAG;IASjB,8CAA8C;YAChC,kBAAkB;IAqBhC,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,SAAS;IAYjB;;;;OAIG;YACW,eAAe;CAO9B"}
|