crbro-memory 2.4.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -3
- package/bin/crbro.mjs +234 -0
- package/dist/daemon/daemon.d.ts +25 -0
- package/dist/daemon/daemon.d.ts.map +1 -0
- package/dist/daemon/daemon.js +387 -0
- package/dist/daemon/daemon.js.map +1 -0
- package/dist/daemon/endpoint.d.ts +65 -0
- package/dist/daemon/endpoint.d.ts.map +1 -0
- package/dist/daemon/endpoint.js +308 -0
- package/dist/daemon/endpoint.js.map +1 -0
- package/dist/daemon/lines.d.ts +55 -0
- package/dist/daemon/lines.d.ts.map +1 -0
- package/dist/daemon/lines.js +142 -0
- package/dist/daemon/lines.js.map +1 -0
- package/dist/daemon/proxy.d.ts +55 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +412 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/engine/backup.d.ts +68 -0
- package/dist/engine/backup.d.ts.map +1 -0
- package/dist/engine/backup.js +224 -0
- package/dist/engine/backup.js.map +1 -0
- package/dist/engine/brain.d.ts.map +1 -1
- package/dist/engine/brain.js +6 -2
- package/dist/engine/brain.js.map +1 -1
- package/dist/engine/cortex.d.ts +76 -2
- package/dist/engine/cortex.d.ts.map +1 -1
- package/dist/engine/cortex.js +169 -9
- package/dist/engine/cortex.js.map +1 -1
- package/dist/engine/dates.d.ts +28 -0
- package/dist/engine/dates.d.ts.map +1 -0
- package/dist/engine/dates.js +121 -0
- package/dist/engine/dates.js.map +1 -0
- package/dist/engine/maintenance.d.ts +85 -0
- package/dist/engine/maintenance.d.ts.map +1 -1
- package/dist/engine/maintenance.js +340 -0
- package/dist/engine/maintenance.js.map +1 -1
- package/dist/engine/triggers.d.ts +46 -0
- package/dist/engine/triggers.d.ts.map +1 -0
- package/dist/engine/triggers.js +201 -0
- package/dist/engine/triggers.js.map +1 -0
- package/dist/index.js +33 -5
- package/dist/index.js.map +1 -1
- package/dist/search/index.d.ts +56 -3
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +249 -12
- package/dist/search/index.js.map +1 -1
- package/dist/server.d.ts +32 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +151 -19
- package/dist/server.js.map +1 -1
- package/hooks/crbro-guard.mjs +157 -0
- package/package.json +1 -1
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ─── Dates an entry states about itself (2.5) ────────────────────
|
|
3
|
+
//
|
|
4
|
+
// Patterns, preferences, errors and debts only carry a date since 1.13. On the
|
|
5
|
+
// reference brain that left 249 of 457 without one, and recall answered
|
|
6
|
+
// `matched_added: ""` for them, so "prefer the more recent" had nothing to
|
|
7
|
+
// stand on in exactly the ledgers where it matters.
|
|
8
|
+
//
|
|
9
|
+
// Most of those entries say when they happened — "Hallazgo (2026-08-30): …",
|
|
10
|
+
// "el 14-sep-2026 …". This module reads that and nothing else. The tempting
|
|
11
|
+
// fallback, the neuron's `created`, was measured and thrown away: for the
|
|
12
|
+
// entries with no date in their text the window between `created` and the day
|
|
13
|
+
// dating began had a median of 117 days. A date that loose is not a date, it
|
|
14
|
+
// is a guess with a timestamp's authority. What cannot be dated from its own
|
|
15
|
+
// words stays undated and is counted as such.
|
|
16
|
+
//
|
|
17
|
+
// A recovered date is written as a DAY ("2026-08-30"), never an instant.
|
|
18
|
+
// `cortex.learn` stamps full ISO instants, so the value itself says how it
|
|
19
|
+
// was obtained, the sidecar needs no second field, and every reader — string
|
|
20
|
+
// comparison, `new Date()`, the sync `at` — takes it as it is.
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.datesInText = datesInText;
|
|
23
|
+
exports.inferEntryDay = inferEntryDay;
|
|
24
|
+
exports.isDayPrecision = isDayPrecision;
|
|
25
|
+
const MONTHS = {};
|
|
26
|
+
const add = (n, ...names) => { for (const s of names)
|
|
27
|
+
MONTHS[s] = n; };
|
|
28
|
+
// Full names and the abbreviations people actually write, es/en/fr/de/it/pt.
|
|
29
|
+
// "out" (pt) and "mar"-like collisions are avoided by listing forms, not prefixes.
|
|
30
|
+
add(1, 'enero', 'ene', 'january', 'jan', 'janvier', 'janv', 'januar', 'gennaio', 'gen', 'janeiro');
|
|
31
|
+
add(2, 'febrero', 'feb', 'february', 'fevrier', 'fev', 'februar', 'febbraio', 'fevereiro');
|
|
32
|
+
add(3, 'marzo', 'mar', 'march', 'mars', 'marz', 'maerz', 'marco');
|
|
33
|
+
add(4, 'abril', 'abr', 'april', 'apr', 'avril', 'avr', 'aprile');
|
|
34
|
+
add(5, 'mayo', 'may', 'mai', 'maggio', 'mag', 'maio');
|
|
35
|
+
add(6, 'junio', 'jun', 'june', 'juin', 'juni', 'giugno', 'giu', 'junho');
|
|
36
|
+
add(7, 'julio', 'jul', 'july', 'juillet', 'juil', 'juli', 'luglio', 'lug', 'julho');
|
|
37
|
+
add(8, 'agosto', 'ago', 'august', 'aug', 'aout', 'aou');
|
|
38
|
+
add(9, 'septiembre', 'setiembre', 'sep', 'sept', 'september', 'septembre', 'settembre', 'set', 'setembro');
|
|
39
|
+
add(10, 'octubre', 'oct', 'october', 'octobre', 'oktober', 'okt', 'ottobre', 'ott', 'outubro');
|
|
40
|
+
add(11, 'noviembre', 'nov', 'november', 'novembre', 'novembro');
|
|
41
|
+
add(12, 'diciembre', 'dic', 'december', 'dec', 'decembre', 'dezember', 'dez', 'dicembre', 'dezembro');
|
|
42
|
+
const plain = (s) => s.normalize('NFD').replace(/\p{M}/gu, '').toLowerCase().replace(/\.$/, '');
|
|
43
|
+
/** "2026-08-30" if y-m-d is a real calendar day, else ''. */
|
|
44
|
+
function day(y, m, d) {
|
|
45
|
+
if (!(y >= 2000 && y <= 2099) || !(m >= 1 && m <= 12) || !(d >= 1 && d <= 31))
|
|
46
|
+
return '';
|
|
47
|
+
const t = new Date(Date.UTC(y, m - 1, d));
|
|
48
|
+
if (t.getUTCFullYear() !== y || t.getUTCMonth() !== m - 1 || t.getUTCDate() !== d)
|
|
49
|
+
return '';
|
|
50
|
+
return `${y}-${String(m).padStart(2, '0')}-${String(d).padStart(2, '0')}`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Every calendar date a text states, in order of appearance. Four shapes:
|
|
54
|
+
* 2026-08-30 · 30/08/2026 (or - and .) · 30 de agosto de 2026 / 14-sep-2026 ·
|
|
55
|
+
* August 30, 2026. Two-digit years and bare "en agosto" are not dates.
|
|
56
|
+
*/
|
|
57
|
+
function datesInText(text) {
|
|
58
|
+
const found = [];
|
|
59
|
+
let m;
|
|
60
|
+
const iso = /(?<![\d\-])(20\d{2})[-/](\d{1,2})[-/](\d{1,2})(?![\d])/g;
|
|
61
|
+
while ((m = iso.exec(text))) {
|
|
62
|
+
const d = day(+m[1], +m[2], +m[3]);
|
|
63
|
+
if (d)
|
|
64
|
+
found.push({ at: m.index, day: d });
|
|
65
|
+
}
|
|
66
|
+
const numeric = /(?<![\d\-/.])(\d{1,2})([-/.])(\d{1,2})\2(20\d{2})(?![\d])/g;
|
|
67
|
+
while ((m = numeric.exec(text))) {
|
|
68
|
+
const dayFirst = day(+m[4], +m[3], +m[1]);
|
|
69
|
+
const monthFirst = day(+m[4], +m[1], +m[3]);
|
|
70
|
+
if (dayFirst && monthFirst && dayFirst !== monthFirst)
|
|
71
|
+
found.push({ at: m.index, day: dayFirst, alt: monthFirst });
|
|
72
|
+
else if (dayFirst || monthFirst)
|
|
73
|
+
found.push({ at: m.index, day: dayFirst || monthFirst });
|
|
74
|
+
}
|
|
75
|
+
const named = /(?<![\d])(\d{1,2})(?:º|\.)?[-\s]+(?:de\s+|of\s+)?([A-Za-zÀ-ÿ]{3,10})\.?,?[-\s]+(?:de\s+|del\s+)?(20\d{2})(?![\d])/g;
|
|
76
|
+
while ((m = named.exec(text))) {
|
|
77
|
+
const mes = MONTHS[plain(m[2])];
|
|
78
|
+
const d = mes ? day(+m[3], mes, +m[1]) : '';
|
|
79
|
+
if (d)
|
|
80
|
+
found.push({ at: m.index, day: d });
|
|
81
|
+
}
|
|
82
|
+
const english = /\b([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(20\d{2})(?![\d])/g;
|
|
83
|
+
while ((m = english.exec(text))) {
|
|
84
|
+
const mes = MONTHS[plain(m[1])];
|
|
85
|
+
const d = mes ? day(+m[3], mes, +m[2]) : '';
|
|
86
|
+
if (d)
|
|
87
|
+
found.push({ at: m.index, day: d });
|
|
88
|
+
}
|
|
89
|
+
return found.sort((a, b) => a.at - b.at).map(({ day: d, alt }) => (alt ? { day: d, alt } : { day: d }));
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The day an undated entry was written, as far as its own text proves it.
|
|
93
|
+
*
|
|
94
|
+
* An entry cannot predate the latest past day it mentions, and entries are
|
|
95
|
+
* overwhelmingly written the day the thing happened — so the answer is the
|
|
96
|
+
* LATEST stated date inside [floor, ceiling]. `floor` is the neuron's
|
|
97
|
+
* `created`; `ceiling` is the day dating began on this brain (an entry
|
|
98
|
+
* written after that would carry a stamp), or today. Dates outside the window
|
|
99
|
+
* are deadlines, history or typos, not the day of writing. A numeric date
|
|
100
|
+
* that reads both ways counts only if exactly one reading fits the window.
|
|
101
|
+
* '' when the text proves nothing.
|
|
102
|
+
*/
|
|
103
|
+
function inferEntryDay(text, floor, ceiling) {
|
|
104
|
+
const lo = (floor || '').slice(0, 10);
|
|
105
|
+
const hi = (ceiling || '').slice(0, 10);
|
|
106
|
+
const fits = (d) => (!lo || d >= lo) && (!hi || d <= hi);
|
|
107
|
+
let best = '';
|
|
108
|
+
for (const f of datesInText(text)) {
|
|
109
|
+
const readings = [f.day, ...(f.alt ? [f.alt] : [])].filter(fits);
|
|
110
|
+
if (readings.length !== 1)
|
|
111
|
+
continue;
|
|
112
|
+
if (readings[0] > best)
|
|
113
|
+
best = readings[0];
|
|
114
|
+
}
|
|
115
|
+
return best;
|
|
116
|
+
}
|
|
117
|
+
/** True for a value written by the backfill (a day), false for a learn() stamp (an instant). */
|
|
118
|
+
function isDayPrecision(value) {
|
|
119
|
+
return !!value && /^\d{4}-\d{2}-\d{2}$/.test(value);
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=dates.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dates.js","sourceRoot":"","sources":["../../src/engine/dates.ts"],"names":[],"mappings":";AAAA,oEAAoE;AACpE,EAAE;AACF,+EAA+E;AAC/E,wEAAwE;AACxE,2EAA2E;AAC3E,oDAAoD;AACpD,EAAE;AACF,6EAA6E;AAC7E,4EAA4E;AAC5E,0EAA0E;AAC1E,8EAA8E;AAC9E,6EAA6E;AAC7E,6EAA6E;AAC7E,8CAA8C;AAC9C,EAAE;AACF,yEAAyE;AACzE,2EAA2E;AAC3E,6EAA6E;AAC7E,+DAA+D;;AAyC/D,kCAiCC;AAcD,sCAWC;AAGD,wCAEC;AAtGD,MAAM,MAAM,GAA2B,EAAE,CAAC;AAC1C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,GAAG,KAAe,EAAE,EAAE,GAAG,KAAK,MAAM,CAAC,IAAI,KAAK;IAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACzF,6EAA6E;AAC7E,mFAAmF;AACnF,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;AACnG,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAC3F,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAClE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;AACjE,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AACtD,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AACzE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AACpF,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;AACxD,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;AAC3G,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;AAC/F,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAChE,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAEtG,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAExG,6DAA6D;AAC7D,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC1C,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAAE,OAAO,EAAE,CAAC;IACzF,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7F,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AAC5E,CAAC;AASD;;;;GAIG;AACH,SAAgB,WAAW,CAAC,IAAY;IACtC,MAAM,KAAK,GAAqC,EAAE,CAAC;IACnD,IAAI,CAAyB,CAAC;IAE9B,MAAM,GAAG,GAAG,yDAAyD,CAAC;IACtE,OAAO,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,4DAA4D,CAAC;IAC7E,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,IAAI,QAAQ,IAAI,UAAU,IAAI,QAAQ,KAAK,UAAU;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;aAC9G,IAAI,QAAQ,IAAI,UAAU;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,IAAI,UAAU,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,KAAK,GAAG,oHAAoH,CAAC;IACnI,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,yEAAyE,CAAC;IAC1F,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,aAAa,CAAC,IAAY,EAAE,KAAa,EAAE,OAAe;IACxE,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpC,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;YAAE,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,gGAAgG;AAChG,SAAgB,cAAc,CAAC,KAAyB;IACtD,OAAO,CAAC,CAAC,KAAK,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC"}
|
|
@@ -32,8 +32,56 @@ export interface MaintenanceReport {
|
|
|
32
32
|
repaired: number;
|
|
33
33
|
/** One line per fix applied. */
|
|
34
34
|
repairs: string[];
|
|
35
|
+
/** Patterns, preferences, errors and debts with no date: written before 1.13 stamped them. */
|
|
36
|
+
undated_entries: number;
|
|
37
|
+
/** How many of those state a date in their own text — what backfill_dates recovers. */
|
|
38
|
+
datable_entries: number;
|
|
39
|
+
/** Dates written by this run (0 unless backfill_dates:true and not a dry run). */
|
|
40
|
+
dates_backfilled: number;
|
|
41
|
+
/** Live entries that named a day still ahead of them when written, and that day has passed. */
|
|
42
|
+
expired_entries: number;
|
|
43
|
+
/** The longest-expired of them, to review: keep, crbro_revise, or crbro_learn supersedes. Never touched by maintenance. */
|
|
44
|
+
expired_sample: ExpiredEntry[];
|
|
45
|
+
/** Neurons past SPLIT_MIN_ENTRIES live entries, each with the groups its own words suggest. */
|
|
46
|
+
split_candidates: SplitCandidate[];
|
|
47
|
+
/** One-line neurons left by a bulk import, by the day they were created. compact:true folds each group into one digest neuron. */
|
|
48
|
+
compact_groups: CompactGroup[];
|
|
49
|
+
/** How many neurons those groups add up to. */
|
|
50
|
+
compactable_neurons: number;
|
|
51
|
+
/** Neurons folded by this run (0 unless compact:true and not a dry run). */
|
|
52
|
+
compacted_neurons: number;
|
|
35
53
|
notes: string[];
|
|
36
54
|
}
|
|
55
|
+
export interface ExpiredEntry {
|
|
56
|
+
neuron_id: string;
|
|
57
|
+
entry_id: string;
|
|
58
|
+
kind: string;
|
|
59
|
+
/** When the entry was written. */
|
|
60
|
+
added: string;
|
|
61
|
+
/** The day it looked forward to, now behind us. */
|
|
62
|
+
due: string;
|
|
63
|
+
preview: string;
|
|
64
|
+
}
|
|
65
|
+
export interface SplitCandidate {
|
|
66
|
+
neuron_id: string;
|
|
67
|
+
name: string;
|
|
68
|
+
entries: number;
|
|
69
|
+
/** Words that gather a sizeable share of the entries without gathering all of them. */
|
|
70
|
+
groups: Array<{
|
|
71
|
+
term: string;
|
|
72
|
+
entries: number;
|
|
73
|
+
}>;
|
|
74
|
+
}
|
|
75
|
+
export interface CompactGroup {
|
|
76
|
+
day: string;
|
|
77
|
+
domain: string;
|
|
78
|
+
neurons: number;
|
|
79
|
+
/** Distinct texts among them: the rest are the same line under another name. */
|
|
80
|
+
unique_entries: number;
|
|
81
|
+
/** The digest neuron they fold into. */
|
|
82
|
+
digest: string;
|
|
83
|
+
sample: string[];
|
|
84
|
+
}
|
|
37
85
|
export declare class Maintenance {
|
|
38
86
|
private brain;
|
|
39
87
|
private cortex;
|
|
@@ -51,6 +99,9 @@ export declare class Maintenance {
|
|
|
51
99
|
purgeBoilerplate?: boolean;
|
|
52
100
|
repair?: boolean;
|
|
53
101
|
unarchive?: string[] | 'all';
|
|
102
|
+
backfillDates?: boolean;
|
|
103
|
+
compact?: boolean;
|
|
104
|
+
sharedIds?: ReadonlySet<string>;
|
|
54
105
|
}): Promise<MaintenanceReport>;
|
|
55
106
|
/**
|
|
56
107
|
* Consolidate session — called at end of session.
|
|
@@ -101,6 +152,40 @@ export declare class Maintenance {
|
|
|
101
152
|
* fix blindly.
|
|
102
153
|
*/
|
|
103
154
|
private checkIntegrity;
|
|
155
|
+
/**
|
|
156
|
+
* The signature of a bulk import, all of it required: exactly one live
|
|
157
|
+
* entry and nothing else (no map, no summary, no tags, no connections),
|
|
158
|
+
* not a protocol, not shared, older than COMPACT_MIN_AGE_DAYS, and born
|
|
159
|
+
* the same day in the same domain as at least COMPACT_MIN_GROUP others
|
|
160
|
+
* like it. A lone one-fact neuron is somebody's note and is never touched.
|
|
161
|
+
*/
|
|
162
|
+
private findCompactGroups;
|
|
163
|
+
/**
|
|
164
|
+
* Two read-only reviews in one pass over the cortex.
|
|
165
|
+
*
|
|
166
|
+
* Expired: a live, dated entry whose text states a day LATER than the day
|
|
167
|
+
* it was written — so it was looking forward — and that day is now behind
|
|
168
|
+
* us. Language-independent on purpose: no list of "hasta / until / vence"
|
|
169
|
+
* to maintain, just two dates and today. An entry that only mentions past
|
|
170
|
+
* days is history, not a promise, and is never flagged. Undated entries
|
|
171
|
+
* cannot be judged and are skipped (backfill_dates first).
|
|
172
|
+
*
|
|
173
|
+
* Split: a neuron with SPLIT_MIN_ENTRIES+ live entries, with the words
|
|
174
|
+
* that gather between 8% and 45% of them — frequent enough to be a
|
|
175
|
+
* subtopic, not so frequent that they are just the neuron's own name.
|
|
176
|
+
*/
|
|
177
|
+
private reviewEntries;
|
|
178
|
+
/**
|
|
179
|
+
* Count the undated patterns, preferences, errors and debts, and — when
|
|
180
|
+
* `write` — date the ones whose own text says when they happened.
|
|
181
|
+
*
|
|
182
|
+
* The ceiling is the first instant learn() ever stamped on this brain: an
|
|
183
|
+
* entry written after that would carry a stamp, so a later date in an
|
|
184
|
+
* undated entry is a deadline, not the day it was written. Day-precision
|
|
185
|
+
* values are skipped when looking for it — they are earlier backfills, and
|
|
186
|
+
* counting them would drag the ceiling back on every run.
|
|
187
|
+
*/
|
|
188
|
+
private backfillDates;
|
|
104
189
|
/**
|
|
105
190
|
* Fix what checkIntegrity() reports: dangling connection ids, synapse files
|
|
106
191
|
* with a missing end, stale entry_dates / entry_status keys, manifest
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"maintenance.d.ts","sourceRoot":"","sources":["../../src/engine/maintenance.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"maintenance.d.ts","sourceRoot":"","sources":["../../src/engine/maintenance.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,MAAM,WAAW,iBAAiB;IAChC,gBAAgB,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wDAAwD;IACxD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kFAAkF;IAClF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,mIAAmI;IACnI,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iDAAiD;IACjD,cAAc,EAAE,MAAM,CAAC;IACvB,kEAAkE;IAClE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sFAAsF;IACtF,UAAU,EAAE,MAAM,CAAC;IACnB,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAC;IACjB,gCAAgC;IAChC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8FAA8F;IAC9F,eAAe,EAAE,MAAM,CAAC;IACxB,uFAAuF;IACvF,eAAe,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,gBAAgB,EAAE,MAAM,CAAC;IACzB,+FAA+F;IAC/F,eAAe,EAAE,MAAM,CAAC;IACxB,2HAA2H;IAC3H,cAAc,EAAE,YAAY,EAAE,CAAC;IAC/B,+FAA+F;IAC/F,gBAAgB,EAAE,cAAc,EAAE,CAAC;IACnC,kIAAkI;IAClI,cAAc,EAAE,YAAY,EAAE,CAAC;IAC/B,+CAA+C;IAC/C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4EAA4E;IAC5E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAiCD,qBAAa,WAAW;IAEpB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,YAAY;gBANZ,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,UAAU,EACtB,YAAY,EAAE,YAAY;IAGpC;;OAEG;IACG,GAAG,CACP,MAAM,GAAE,OAAe,EACvB,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;KAAE,GACvL,OAAO,CAAC,iBAAiB,CAAC;IAqP7B;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAC3C;;;;;;WAMG;QACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;KAC1B,GAAG,OAAO,CAAC;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,+EAA+E;QAC/E,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,yFAAyF;QACzF,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,kHAAkH;QAClH,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,8EAA8E;QAC9E,cAAc,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;IAgGF,wEAAwE;IACxE,OAAO,CAAC,aAAa;YAIP,gBAAgB;YAUhB,gBAAgB;IAiB9B,iEAAiE;IACjE,OAAO,CAAC,MAAM;YAMA,kBAAkB;YAqBlB,gBAAgB;IAc9B;;;;OAIG;YACW,SAAS;IA8BvB;;;;OAIG;YACW,cAAc;IAoE5B;;;;;;OAMG;YACW,iBAAiB;IAsC/B;;;;;;;;;;;;;OAaG;YACW,aAAa;IA8G3B;;;;;;;;;OASG;YACW,aAAa;IAkD3B;;;;OAIG;YACW,MAAM;CAkFrB"}
|
|
@@ -6,6 +6,29 @@ exports.Maintenance = void 0;
|
|
|
6
6
|
const fs_js_1 = require("../utils/fs.js");
|
|
7
7
|
const lock_js_1 = require("../utils/lock.js");
|
|
8
8
|
const ops_js_1 = require("../sync/ops.js");
|
|
9
|
+
const dates_js_1 = require("./dates.js");
|
|
10
|
+
const tokenize_js_1 = require("../search/tokenize.js");
|
|
11
|
+
const hash_js_1 = require("../utils/hash.js");
|
|
12
|
+
/**
|
|
13
|
+
* A bulk import, not a person: this many one-line neurons born the same day
|
|
14
|
+
* in the same domain. On the reference brain one day holds 766 of them — a
|
|
15
|
+
* transcript miner that made a neuron out of every checklist line — and 482
|
|
16
|
+
* repeat a text another one already has. Three of those with identical text
|
|
17
|
+
* took the top three places of a recall.
|
|
18
|
+
*/
|
|
19
|
+
const COMPACT_MIN_GROUP = 25;
|
|
20
|
+
/** Young neurons are left alone: one entry today may be the start of a topic. */
|
|
21
|
+
const COMPACT_MIN_AGE_DAYS = 30;
|
|
22
|
+
/** A neuron this big no longer fits one read: crbro_inspect pages it, and every recall hit from it competes with 80 siblings. */
|
|
23
|
+
const SPLIT_MIN_ENTRIES = 80;
|
|
24
|
+
const SPLIT_MAX_CANDIDATES = 8;
|
|
25
|
+
const SPLIT_GROUPS = 5;
|
|
26
|
+
const EXPIRED_SAMPLE = 10;
|
|
27
|
+
/** Too common to name a subtopic. Short on purpose: the share bounds below do most of the work. */
|
|
28
|
+
const SPLIT_STOP = new Set(('para como pero este esta estos estas desde hasta entre sobre cuando donde porque tambien todos todas cada '
|
|
29
|
+
+ 'tiene tienen hace hacer puede pueden debe deben usar solo antes despues ahora siempre nunca mismo misma nueva nuevo '
|
|
30
|
+
+ 'with from that this those these when where which while into over under after before there their have does must should '
|
|
31
|
+
+ 'only always never using used more than then also each both were been being').split(' '));
|
|
9
32
|
/** Every text an entry_dates / entry_status key may legitimately point at. */
|
|
10
33
|
function liveEntryKeys(n) {
|
|
11
34
|
return new Set([
|
|
@@ -53,6 +76,15 @@ class Maintenance {
|
|
|
53
76
|
repairable: 0,
|
|
54
77
|
repaired: 0,
|
|
55
78
|
repairs: [],
|
|
79
|
+
undated_entries: 0,
|
|
80
|
+
datable_entries: 0,
|
|
81
|
+
dates_backfilled: 0,
|
|
82
|
+
expired_entries: 0,
|
|
83
|
+
expired_sample: [],
|
|
84
|
+
split_candidates: [],
|
|
85
|
+
compact_groups: [],
|
|
86
|
+
compactable_neurons: 0,
|
|
87
|
+
compacted_neurons: 0,
|
|
56
88
|
notes: [],
|
|
57
89
|
};
|
|
58
90
|
// 0. Bring neurons back from the archive, before anything else could
|
|
@@ -147,6 +179,58 @@ class Maintenance {
|
|
|
147
179
|
report.notes.push(`${report.repairable} integrity issue(s) found. Pass repair:true to fix dangling connections, ` +
|
|
148
180
|
'orphan synapses, stale sidecar keys and manifest counters.');
|
|
149
181
|
}
|
|
182
|
+
// 4c. Entries from before 1.13 have no date, so recall cannot say which of
|
|
183
|
+
// two tellings is the recent one. Counting is free; writing is opt-in, and
|
|
184
|
+
// only what the entry's own text proves is ever written (see dates.ts).
|
|
185
|
+
const fechas = await this.backfillDates(!!options?.backfillDates && !dryRun);
|
|
186
|
+
report.undated_entries = fechas.undated;
|
|
187
|
+
report.datable_entries = fechas.datable;
|
|
188
|
+
report.dates_backfilled = fechas.written;
|
|
189
|
+
if (fechas.written > 0) {
|
|
190
|
+
report.notes.push(`Dated ${fechas.written} entr${fechas.written === 1 ? 'y' : 'ies'} from the date stated in their own text, ` +
|
|
191
|
+
`to the day. ${fechas.undated - fechas.written} state none and stay undated: nothing was guessed.`);
|
|
192
|
+
}
|
|
193
|
+
else if (fechas.datable > 0) {
|
|
194
|
+
report.notes.push(`${fechas.undated} pattern/preference/error/debt entr${fechas.undated === 1 ? 'y has' : 'ies have'} no date ` +
|
|
195
|
+
`(written before 1.13); ${fechas.datable} state one in their own text. Pass backfill_dates:true to record it.`);
|
|
196
|
+
}
|
|
197
|
+
// 4e. What a bulk import left behind: hundreds of one-line neurons.
|
|
198
|
+
// Reported always; folded only when asked, and never in a dry run.
|
|
199
|
+
const grupos = await this.findCompactGroups(options?.sharedIds);
|
|
200
|
+
report.compact_groups = grupos.map(({ ids: _ids, type: _type, ...g }) => g);
|
|
201
|
+
report.compactable_neurons = grupos.reduce((a, g) => a + g.neurons, 0);
|
|
202
|
+
if (options?.compact && !dryRun) {
|
|
203
|
+
for (const g of grupos) {
|
|
204
|
+
const r = await this.cortex.compact(g.ids, g.digest, { type: g.type, domain: g.domain });
|
|
205
|
+
report.compacted_neurons += r.folded.length;
|
|
206
|
+
}
|
|
207
|
+
if (report.compacted_neurons > 0) {
|
|
208
|
+
report.notes.push(`Folded ${report.compacted_neurons} one-line neuron(s) into ${grupos.length} digest neuron(s) (tag "digest"); each source's name ` +
|
|
209
|
+
'travels as the keys of its line, identical lines are kept once, and a copy of every source is in .quarantine ' +
|
|
210
|
+
'(crbro_forget restore brings one back).');
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
else if (report.compactable_neurons > 0) {
|
|
214
|
+
report.notes.push(`${report.compactable_neurons} one-line neuron(s) were created in bulk (compact_groups: same day, same domain, no tags, ` +
|
|
215
|
+
'no links, never written again). They crowd recall — one result per neuron — with near-identical lines. ' +
|
|
216
|
+
'Pass compact:true to fold each group into one digest neuron; run dry_run first and read the samples.');
|
|
217
|
+
}
|
|
218
|
+
// 4d. What the brain still says as current although its own words put a
|
|
219
|
+
// date on it, and what has outgrown a single read. Read-only, both: what
|
|
220
|
+
// to do with a line is a judgement, and maintenance does not make it.
|
|
221
|
+
const revision = await this.reviewEntries();
|
|
222
|
+
report.expired_entries = revision.expired;
|
|
223
|
+
report.expired_sample = revision.sample;
|
|
224
|
+
report.split_candidates = revision.splits;
|
|
225
|
+
if (revision.expired > 0) {
|
|
226
|
+
report.notes.push(`${revision.expired} live entr${revision.expired === 1 ? 'y names' : 'ies name'} a day that was still ahead when written and has ` +
|
|
227
|
+
'since passed — a deadline, a "until", a planned step. expired_sample lists the oldest: confirm each is still true, ' +
|
|
228
|
+
'or retire it with crbro_revise / replace it with crbro_learn supersedes.');
|
|
229
|
+
}
|
|
230
|
+
if (revision.splits.length > 0) {
|
|
231
|
+
report.notes.push(`${revision.splits.length} neuron(s) hold ${SPLIT_MIN_ENTRIES}+ live entries (split_candidates, with the groups their own words suggest). ` +
|
|
232
|
+
'To split one: crbro_inspect view=neuron for the entry ids, then crbro_revise move_to — the entries keep their dates.');
|
|
233
|
+
}
|
|
150
234
|
// The debt ledger: a deferral that never named its revisit condition is
|
|
151
235
|
// on its way to becoming permanent by accident. Count them and say so —
|
|
152
236
|
// "later" without a trigger means "never". Retired debts (entry_status)
|
|
@@ -457,6 +541,262 @@ class Maintenance {
|
|
|
457
541
|
}
|
|
458
542
|
return issues;
|
|
459
543
|
}
|
|
544
|
+
/**
|
|
545
|
+
* The signature of a bulk import, all of it required: exactly one live
|
|
546
|
+
* entry and nothing else (no map, no summary, no tags, no connections),
|
|
547
|
+
* not a protocol, not shared, older than COMPACT_MIN_AGE_DAYS, and born
|
|
548
|
+
* the same day in the same domain as at least COMPACT_MIN_GROUP others
|
|
549
|
+
* like it. A lone one-fact neuron is somebody's note and is never touched.
|
|
550
|
+
*/
|
|
551
|
+
async findCompactGroups(sharedIds) {
|
|
552
|
+
const limite = new Date(Date.now() - COMPACT_MIN_AGE_DAYS * 86_400_000).toISOString();
|
|
553
|
+
const porGrupo = new Map();
|
|
554
|
+
for (const id of await this.cortex.allIds()) {
|
|
555
|
+
let n;
|
|
556
|
+
try {
|
|
557
|
+
n = await (0, fs_js_1.readJSON)(this.brain.paths.neuron(id));
|
|
558
|
+
}
|
|
559
|
+
catch {
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
if (!n || n.type === 'protocol' || sharedIds?.has(n.id))
|
|
563
|
+
continue;
|
|
564
|
+
if (n.map?.text || n.summary || (n.tags || []).length || (n.connections || []).length)
|
|
565
|
+
continue;
|
|
566
|
+
if (!n.created || n.created > limite)
|
|
567
|
+
continue;
|
|
568
|
+
const entradas = (n.facts || []).length + (n.decisions || []).length + (n.patterns || []).length
|
|
569
|
+
+ (n.preferences || []).length + (n.errors || []).length + (n.debts || []).length;
|
|
570
|
+
if (entradas !== 1 || (n.facts || []).length !== 1)
|
|
571
|
+
continue;
|
|
572
|
+
if (n.facts[0].status && n.facts[0].status !== 'active')
|
|
573
|
+
continue;
|
|
574
|
+
const k = `${n.created.slice(0, 10)}|${n.domain || 'general'}`;
|
|
575
|
+
const lista = porGrupo.get(k);
|
|
576
|
+
if (lista)
|
|
577
|
+
lista.push(n);
|
|
578
|
+
else
|
|
579
|
+
porGrupo.set(k, [n]);
|
|
580
|
+
}
|
|
581
|
+
const out = [];
|
|
582
|
+
for (const [k, lista] of porGrupo) {
|
|
583
|
+
if (lista.length < COMPACT_MIN_GROUP)
|
|
584
|
+
continue;
|
|
585
|
+
const [day, domain] = k.split('|');
|
|
586
|
+
lista.sort((a, b) => (a.id < b.id ? -1 : 1));
|
|
587
|
+
out.push({
|
|
588
|
+
day, domain, neurons: lista.length,
|
|
589
|
+
unique_entries: new Set(lista.map(n => n.facts[0].text.trim().toLowerCase())).size,
|
|
590
|
+
digest: `Imported notes ${day} (${domain})`,
|
|
591
|
+
sample: [0, Math.floor(lista.length / 2), lista.length - 1].map(i => `${lista[i].name}: ${lista[i].facts[0].text.slice(0, 80)}`),
|
|
592
|
+
ids: lista.map(n => n.id),
|
|
593
|
+
type: 'project',
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
return out.sort((a, b) => b.neurons - a.neurons);
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Two read-only reviews in one pass over the cortex.
|
|
600
|
+
*
|
|
601
|
+
* Expired: a live, dated entry whose text states a day LATER than the day
|
|
602
|
+
* it was written — so it was looking forward — and that day is now behind
|
|
603
|
+
* us. Language-independent on purpose: no list of "hasta / until / vence"
|
|
604
|
+
* to maintain, just two dates and today. An entry that only mentions past
|
|
605
|
+
* days is history, not a promise, and is never flagged. Undated entries
|
|
606
|
+
* cannot be judged and are skipped (backfill_dates first).
|
|
607
|
+
*
|
|
608
|
+
* Split: a neuron with SPLIT_MIN_ENTRIES+ live entries, with the words
|
|
609
|
+
* that gather between 8% and 45% of them — frequent enough to be a
|
|
610
|
+
* subtopic, not so frequent that they are just the neuron's own name.
|
|
611
|
+
*/
|
|
612
|
+
async reviewEntries() {
|
|
613
|
+
const hoy = (0, fs_js_1.today)();
|
|
614
|
+
const vencidas = [];
|
|
615
|
+
const gordas = [];
|
|
616
|
+
const enNeuronas = new Map();
|
|
617
|
+
const grandes = [];
|
|
618
|
+
let total = 0;
|
|
619
|
+
for (const id of await this.cortex.allIds()) {
|
|
620
|
+
let n;
|
|
621
|
+
try {
|
|
622
|
+
n = await (0, fs_js_1.readJSON)(this.brain.paths.neuron(id));
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
if (!n || n.type === 'protocol')
|
|
628
|
+
continue;
|
|
629
|
+
const retirada = (t) => !!n.entry_status?.[(0, ops_js_1.entryId)(t)];
|
|
630
|
+
const vivas = [];
|
|
631
|
+
for (const f of n.facts || []) {
|
|
632
|
+
if (!f?.text || (f.status && f.status !== 'active'))
|
|
633
|
+
continue;
|
|
634
|
+
vivas.push({ eid: f.id || (0, hash_js_1.factId)(f.text), kind: 'fact', text: f.text, added: f.added || '' });
|
|
635
|
+
}
|
|
636
|
+
for (const d of n.decisions || []) {
|
|
637
|
+
if (!d?.text || retirada(d.text))
|
|
638
|
+
continue;
|
|
639
|
+
vivas.push({ eid: d.id || (0, ops_js_1.entryId)(d.text), kind: 'decision', text: d.text, added: d.date || '' });
|
|
640
|
+
}
|
|
641
|
+
const sueltas = [
|
|
642
|
+
['pattern', n.patterns], ['preference', n.preferences], ['error', n.errors], ['debt', n.debts],
|
|
643
|
+
];
|
|
644
|
+
for (const [kind, lista] of sueltas) {
|
|
645
|
+
for (const t of lista || []) {
|
|
646
|
+
if (!t || retirada(t))
|
|
647
|
+
continue;
|
|
648
|
+
vivas.push({ eid: (0, ops_js_1.entryId)(t), kind, text: t, added: n.entry_dates?.[(0, ops_js_1.entryId)(t)] || '' });
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
for (const e of vivas) {
|
|
652
|
+
const escrita = e.added.slice(0, 10);
|
|
653
|
+
if (!escrita)
|
|
654
|
+
continue;
|
|
655
|
+
// Two days of margin, not one: stamps are UTC and people write local
|
|
656
|
+
// dates, so an entry saved at 00:30 on the 13th in Madrid is stamped
|
|
657
|
+
// the 12th and says "13/06". On the reference brain that alone was
|
|
658
|
+
// 250 of 277 flags — every one of them a false alarm.
|
|
659
|
+
const desde = new Date(Date.parse(escrita) + 2 * 86_400_000).toISOString().slice(0, 10);
|
|
660
|
+
let due = '';
|
|
661
|
+
for (const f of (0, dates_js_1.datesInText)(e.text)) {
|
|
662
|
+
if (f.alt)
|
|
663
|
+
continue; // reads two ways: not evidence of anything
|
|
664
|
+
if (f.day >= desde && f.day < hoy && f.day > due)
|
|
665
|
+
due = f.day;
|
|
666
|
+
}
|
|
667
|
+
if (!due)
|
|
668
|
+
continue;
|
|
669
|
+
vencidas.push({
|
|
670
|
+
neuron_id: n.id, entry_id: e.eid, kind: e.kind, added: escrita, due,
|
|
671
|
+
preview: e.text.length > 160 ? `${e.text.slice(0, 160).trimEnd()}…` : e.text,
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
// Words per entry, once: they feed this neuron's groups and the
|
|
675
|
+
// brain-wide count that tells a subtopic from a habit of speech.
|
|
676
|
+
const propias = new Set((0, tokenize_js_1.fold)(n.name).split(/[^a-z0-9]+/).filter(Boolean));
|
|
677
|
+
const palabras = vivas.map(e => new Set((0, tokenize_js_1.fold)(e.text).split(/[^a-z0-9]+/).filter(w => w.length >= 5 && !SPLIT_STOP.has(w) && !/^\d+$/.test(w))));
|
|
678
|
+
total++;
|
|
679
|
+
for (const w of new Set(palabras.flatMap(set => [...set])))
|
|
680
|
+
enNeuronas.set(w, (enNeuronas.get(w) || 0) + 1);
|
|
681
|
+
if (vivas.length >= SPLIT_MIN_ENTRIES && !(n.tags || []).includes('digest')) {
|
|
682
|
+
grandes.push({ n, vivas: vivas.length, palabras, propias });
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
// A word found in many neurons is how this person writes ("verificado",
|
|
686
|
+
// "usuario"), not what this neuron is about. Measured on the reference
|
|
687
|
+
// brain, the top group of its largest neuron was "verificado (180)".
|
|
688
|
+
const comun = Math.max(10, Math.ceil(total * 0.04));
|
|
689
|
+
for (const { n, vivas, palabras, propias } of grandes) {
|
|
690
|
+
{
|
|
691
|
+
const donde = new Map();
|
|
692
|
+
palabras.forEach((set, i) => {
|
|
693
|
+
for (const w of set) {
|
|
694
|
+
if (propias.has(w) || (enNeuronas.get(w) || 0) > comun)
|
|
695
|
+
continue;
|
|
696
|
+
const d = donde.get(w);
|
|
697
|
+
if (d)
|
|
698
|
+
d.add(i);
|
|
699
|
+
else
|
|
700
|
+
donde.set(w, new Set([i]));
|
|
701
|
+
}
|
|
702
|
+
});
|
|
703
|
+
const min = Math.ceil(vivas * 0.08), max = Math.floor(vivas * 0.45);
|
|
704
|
+
// Greedy, by size: a word joins only if most of its entries are not
|
|
705
|
+
// already gathered by a word chosen before it — otherwise the five
|
|
706
|
+
// groups are five names for the same one ("newsletter", "envio", ...).
|
|
707
|
+
const cubiertas = new Set();
|
|
708
|
+
const groups = [];
|
|
709
|
+
const orden = [...donde.entries()]
|
|
710
|
+
.filter(([, set]) => set.size >= min && set.size <= max)
|
|
711
|
+
.sort((a, b) => b[1].size - a[1].size || (a[0] < b[0] ? -1 : 1));
|
|
712
|
+
for (const [term, set] of orden) {
|
|
713
|
+
if (groups.length >= SPLIT_GROUPS)
|
|
714
|
+
break;
|
|
715
|
+
let repetidas = 0;
|
|
716
|
+
for (const i of set)
|
|
717
|
+
if (cubiertas.has(i))
|
|
718
|
+
repetidas++;
|
|
719
|
+
if (repetidas > set.size / 2)
|
|
720
|
+
continue;
|
|
721
|
+
groups.push({ term, entries: set.size });
|
|
722
|
+
for (const i of set)
|
|
723
|
+
cubiertas.add(i);
|
|
724
|
+
}
|
|
725
|
+
gordas.push({ neuron_id: n.id, name: n.name, entries: vivas, groups });
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
vencidas.sort((a, b) => (a.due < b.due ? -1 : a.due > b.due ? 1 : a.entry_id < b.entry_id ? -1 : 1));
|
|
729
|
+
gordas.sort((a, b) => b.entries - a.entries);
|
|
730
|
+
return { expired: vencidas.length, sample: vencidas.slice(0, EXPIRED_SAMPLE), splits: gordas.slice(0, SPLIT_MAX_CANDIDATES) };
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* Count the undated patterns, preferences, errors and debts, and — when
|
|
734
|
+
* `write` — date the ones whose own text says when they happened.
|
|
735
|
+
*
|
|
736
|
+
* The ceiling is the first instant learn() ever stamped on this brain: an
|
|
737
|
+
* entry written after that would carry a stamp, so a later date in an
|
|
738
|
+
* undated entry is a deadline, not the day it was written. Day-precision
|
|
739
|
+
* values are skipped when looking for it — they are earlier backfills, and
|
|
740
|
+
* counting them would drag the ceiling back on every run.
|
|
741
|
+
*/
|
|
742
|
+
async backfillDates(write) {
|
|
743
|
+
const KINDS = ['patterns', 'preferences', 'errors', 'debts'];
|
|
744
|
+
const ids = await this.cortex.allIds();
|
|
745
|
+
const neuronas = [];
|
|
746
|
+
let techo = '';
|
|
747
|
+
for (const id of ids) {
|
|
748
|
+
let n;
|
|
749
|
+
try {
|
|
750
|
+
n = await (0, fs_js_1.readJSON)(this.brain.paths.neuron(id));
|
|
751
|
+
}
|
|
752
|
+
catch {
|
|
753
|
+
continue; // corrupted: the integrity check reports it
|
|
754
|
+
}
|
|
755
|
+
if (!n)
|
|
756
|
+
continue;
|
|
757
|
+
neuronas.push(n);
|
|
758
|
+
for (const v of Object.values(n.entry_dates || {})) {
|
|
759
|
+
if (v && !(0, dates_js_1.isDayPrecision)(v) && (!techo || v < techo))
|
|
760
|
+
techo = v;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
if (!techo)
|
|
764
|
+
techo = (0, fs_js_1.today)();
|
|
765
|
+
let undated = 0, datable = 0, written = 0;
|
|
766
|
+
for (const n of neuronas) {
|
|
767
|
+
const nuevas = {};
|
|
768
|
+
for (const k of KINDS) {
|
|
769
|
+
for (const t of n[k] || []) {
|
|
770
|
+
if (!t || n.entry_dates?.[(0, ops_js_1.entryId)(t)])
|
|
771
|
+
continue;
|
|
772
|
+
undated++;
|
|
773
|
+
const dia = (0, dates_js_1.inferEntryDay)(t, n.created, techo);
|
|
774
|
+
if (!dia)
|
|
775
|
+
continue;
|
|
776
|
+
datable++;
|
|
777
|
+
nuevas[(0, ops_js_1.entryId)(t)] = dia;
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
if (!write || Object.keys(nuevas).length === 0)
|
|
781
|
+
continue;
|
|
782
|
+
await (0, fs_js_1.updateJSON)(this.brain.paths.neuron(n.id), current => {
|
|
783
|
+
if (!current)
|
|
784
|
+
return null;
|
|
785
|
+
if (!current.entry_dates)
|
|
786
|
+
current.entry_dates = {};
|
|
787
|
+
let tocado = false;
|
|
788
|
+
for (const [k, dia] of Object.entries(nuevas)) {
|
|
789
|
+
if (current.entry_dates[k])
|
|
790
|
+
continue; // stamped by someone else since the scan
|
|
791
|
+
current.entry_dates[k] = dia;
|
|
792
|
+
tocado = true;
|
|
793
|
+
written++;
|
|
794
|
+
}
|
|
795
|
+
return tocado ? current : null;
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
return { undated, datable, written };
|
|
799
|
+
}
|
|
460
800
|
/**
|
|
461
801
|
* Fix what checkIntegrity() reports: dangling connection ids, synapse files
|
|
462
802
|
* with a missing end, stale entry_dates / entry_status keys, manifest
|