@hilbras/remembra 3.6.0 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/README.md +9 -1
- package/dist/backend.d.ts +17 -0
- package/dist/crypto.d.ts +10 -0
- package/dist/crypto.js +77 -0
- package/dist/crypto.js.map +1 -0
- package/dist/diff.d.ts +15 -0
- package/dist/diff.js +135 -0
- package/dist/diff.js.map +1 -0
- package/dist/errors.d.ts +3 -8
- package/dist/errors.js +11 -0
- package/dist/errors.js.map +1 -1
- package/dist/http.d.ts +5 -1
- package/dist/http.js +84 -4
- package/dist/http.js.map +1 -1
- package/dist/index.js +64 -4
- package/dist/index.js.map +1 -1
- package/dist/llm.d.ts +2 -0
- package/dist/llm.js +3 -0
- package/dist/llm.js.map +1 -1
- package/dist/log.d.ts +19 -0
- package/dist/log.js +28 -0
- package/dist/log.js.map +1 -0
- package/dist/metrics.d.ts +22 -0
- package/dist/metrics.js +121 -0
- package/dist/metrics.js.map +1 -0
- package/dist/redact.d.ts +33 -0
- package/dist/redact.js +104 -0
- package/dist/redact.js.map +1 -0
- package/dist/service.d.ts +75 -0
- package/dist/service.js +230 -10
- package/dist/service.js.map +1 -1
- package/dist/store.d.ts +17 -1
- package/dist/store.js +134 -6
- package/dist/store.js.map +1 -1
- package/dist/types.d.ts +74 -0
- package/dist/types.js +30 -0
- package/dist/types.js.map +1 -1
- package/dist/version.d.ts +6 -0
- package/dist/version.js +7 -0
- package/dist/version.js.map +1 -0
- package/docs/architecture.md +52 -3
- package/docs/chatgpt.md +25 -0
- package/docs/clients.md +4 -0
- package/docs/lifecycle.md +10 -3
- package/docs/memory-model.md +42 -3
- package/docs/observability.md +125 -0
- package/docs/security.md +63 -3
- package/docs/tools.md +48 -1
- package/package.json +1 -1
package/dist/redact.d.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content redaction filter for PII (audit Phase 8).
|
|
3
|
+
*
|
|
4
|
+
* Opt-in: REMEMBRA_REDACT=1|true|yes. Applied at the service ingest layer
|
|
5
|
+
* (memory_store + every digest item/merge) so raw PII never reaches disk,
|
|
6
|
+
* embeddings, or export snapshots.
|
|
7
|
+
*
|
|
8
|
+
* Design notes:
|
|
9
|
+
* - Patterns run most-specific first: cards before phones, prefixes before
|
|
10
|
+
* generic long tokens — each match is replaced by a typed placeholder so
|
|
11
|
+
* downstream text stays readable ("<EMAIL> confirmed the deploy").
|
|
12
|
+
* - Cards must pass the Luhn check — digit soup that is not a valid card
|
|
13
|
+
* number stays untouched (false positives here would be worse than misses).
|
|
14
|
+
* - Phone matching requires separators and 10–15 digits, so dates
|
|
15
|
+
* (2026-09-23 → 8 digits) and versions (3.6.0) never match.
|
|
16
|
+
* - Redaction is IRREVERSIBLE by design: the original bytes are not kept
|
|
17
|
+
* anywhere (including in the LLM-visible transcript only at extraction —
|
|
18
|
+
* see docs/security.md).
|
|
19
|
+
*/
|
|
20
|
+
export type RedactionKind = "email" | "ssn" | "card" | "phone" | "secret";
|
|
21
|
+
export interface RedactionResult {
|
|
22
|
+
text: string;
|
|
23
|
+
counts: Partial<Record<RedactionKind, number>>;
|
|
24
|
+
changed: boolean;
|
|
25
|
+
}
|
|
26
|
+
export declare function redactionEnabled(): boolean;
|
|
27
|
+
/** Redact PII patterns out of text, returning what was found. */
|
|
28
|
+
export declare function redact(text: string): RedactionResult;
|
|
29
|
+
/** Redact a tag list (tags are short, but emails/secrets do sneak in). */
|
|
30
|
+
export declare function redactTags(tags: string[]): {
|
|
31
|
+
tags: string[];
|
|
32
|
+
counts: Partial<Record<RedactionKind, number>>;
|
|
33
|
+
};
|
package/dist/redact.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content redaction filter for PII (audit Phase 8).
|
|
3
|
+
*
|
|
4
|
+
* Opt-in: REMEMBRA_REDACT=1|true|yes. Applied at the service ingest layer
|
|
5
|
+
* (memory_store + every digest item/merge) so raw PII never reaches disk,
|
|
6
|
+
* embeddings, or export snapshots.
|
|
7
|
+
*
|
|
8
|
+
* Design notes:
|
|
9
|
+
* - Patterns run most-specific first: cards before phones, prefixes before
|
|
10
|
+
* generic long tokens — each match is replaced by a typed placeholder so
|
|
11
|
+
* downstream text stays readable ("<EMAIL> confirmed the deploy").
|
|
12
|
+
* - Cards must pass the Luhn check — digit soup that is not a valid card
|
|
13
|
+
* number stays untouched (false positives here would be worse than misses).
|
|
14
|
+
* - Phone matching requires separators and 10–15 digits, so dates
|
|
15
|
+
* (2026-09-23 → 8 digits) and versions (3.6.0) never match.
|
|
16
|
+
* - Redaction is IRREVERSIBLE by design: the original bytes are not kept
|
|
17
|
+
* anywhere (including in the LLM-visible transcript only at extraction —
|
|
18
|
+
* see docs/security.md).
|
|
19
|
+
*/
|
|
20
|
+
export function redactionEnabled() {
|
|
21
|
+
const v = (process.env.REMEMBRA_REDACT ?? "").toLowerCase();
|
|
22
|
+
return v === "1" || v === "true" || v === "yes";
|
|
23
|
+
}
|
|
24
|
+
/** Luhn mod-10 — rejects digit groups that merely look card-shaped. */
|
|
25
|
+
function luhnOk(digits) {
|
|
26
|
+
let sum = 0;
|
|
27
|
+
let alt = false;
|
|
28
|
+
for (let i = digits.length - 1; i >= 0; i--) {
|
|
29
|
+
let d = digits.charCodeAt(i) - 48;
|
|
30
|
+
if (alt) {
|
|
31
|
+
d *= 2;
|
|
32
|
+
if (d > 9)
|
|
33
|
+
d -= 9;
|
|
34
|
+
}
|
|
35
|
+
sum += d;
|
|
36
|
+
alt = !alt;
|
|
37
|
+
}
|
|
38
|
+
return sum % 10 === 0;
|
|
39
|
+
}
|
|
40
|
+
// Order matters: specific formats first.
|
|
41
|
+
const RULES = [
|
|
42
|
+
{ kind: "email", pattern: /\b[\w.+-]+@[\w-]+(?:\.[\w-]+)+\b/g },
|
|
43
|
+
{ kind: "ssn", pattern: /\b\d{3}-\d{2}-\d{4}\b/g },
|
|
44
|
+
{
|
|
45
|
+
// 13–19 digits, optionally grouped by spaces/dashes: `4111 1111 1111 1111`
|
|
46
|
+
kind: "card",
|
|
47
|
+
pattern: /\b(?:\d[ -]?){12,18}\d\b/g,
|
|
48
|
+
accept: (m) => luhnOk(m.replace(/[ -]/g, "")),
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
// Requires separators AND 10–15 total digits (dates/versions excluded).
|
|
52
|
+
// Lookbehind keeps the match at the token start (a leading `+` is not a
|
|
53
|
+
// word boundary); the paren form consumes its own `)`.
|
|
54
|
+
kind: "phone",
|
|
55
|
+
pattern: /(?<=^|\s)(?:\+\d{1,3}[ .-]?)?(?:\(\d{2,4}\)[ .-]?|\d{2,4}[ .-])\d{3,4}[ .-]\d{3,4}\b/g,
|
|
56
|
+
accept: (m) => {
|
|
57
|
+
const digits = m.replace(/\D/g, "").length;
|
|
58
|
+
return digits >= 10 && digits <= 15;
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
// Provider token shapes: OpenAI sk-..., GitHub ghp_/gho_/github_pat-, AWS AKIA…,
|
|
62
|
+
// Slack xoxb-, GitLab glpat-, generic <prefix><32+ chars>.
|
|
63
|
+
{
|
|
64
|
+
kind: "secret",
|
|
65
|
+
pattern: /\b(?:sk|pk|rk|ghp|gho|ghs|xoxb|xoxp|glpat)[-_][A-Za-z0-9_-]{16,}\b|\bAKIA[0-9A-Z]{16}\b|\bghu_[A-Za-z0-9]{20,}\b|github_pat_[A-Za-z0-9_]{20,}\b/g,
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
// High-entropy blob: 40+ unbroken base64/hex-ish chars (keys, hashes).
|
|
69
|
+
// UUIDs are 36 chars with dashes; English words never reach 40.
|
|
70
|
+
kind: "secret",
|
|
71
|
+
pattern: /\b[A-Za-z0-9+/_-]{40,}\b/g,
|
|
72
|
+
accept: (m) => !/^[\w-]+$/.test(m) || /[0-9]/.test(m), // require a digit: skips megawords
|
|
73
|
+
},
|
|
74
|
+
];
|
|
75
|
+
/** Redact PII patterns out of text, returning what was found. */
|
|
76
|
+
export function redact(text) {
|
|
77
|
+
const counts = {};
|
|
78
|
+
let out = text;
|
|
79
|
+
for (const rule of RULES) {
|
|
80
|
+
out = out.replace(rule.pattern, (match) => {
|
|
81
|
+
if (rule.accept && !rule.accept(match))
|
|
82
|
+
return match;
|
|
83
|
+
counts[rule.kind] = (counts[rule.kind] ?? 0) + 1;
|
|
84
|
+
return `<${rule.kind.toUpperCase()}>`;
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
return { text: out, counts, changed: Object.keys(counts).length > 0 };
|
|
88
|
+
}
|
|
89
|
+
/** Redact a tag list (tags are short, but emails/secrets do sneak in). */
|
|
90
|
+
export function redactTags(tags) {
|
|
91
|
+
const merged = {};
|
|
92
|
+
let changed = false;
|
|
93
|
+
const out = tags.map((t) => {
|
|
94
|
+
const r = redact(t);
|
|
95
|
+
for (const [k, v] of Object.entries(r.counts)) {
|
|
96
|
+
merged[k] = (merged[k] ?? 0) + v;
|
|
97
|
+
}
|
|
98
|
+
if (r.changed)
|
|
99
|
+
changed = true;
|
|
100
|
+
return r.text;
|
|
101
|
+
});
|
|
102
|
+
return { tags: out, counts: changed ? merged : {} };
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=redact.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.js","sourceRoot":"","sources":["../src/redact.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAUH,MAAM,UAAU,gBAAgB;IAC9B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5D,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC;AAED,uEAAuE;AACvE,SAAS,MAAM,CAAC,MAAc;IAC5B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,GAAG,EAAE,CAAC;YACR,CAAC,IAAI,CAAC,CAAC;YACP,IAAI,CAAC,GAAG,CAAC;gBAAE,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,GAAG,IAAI,CAAC,CAAC;QACT,GAAG,GAAG,CAAC,GAAG,CAAC;IACb,CAAC;IACD,OAAO,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;AACxB,CAAC;AASD,yCAAyC;AACzC,MAAM,KAAK,GAAW;IACpB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,mCAAmC,EAAE;IAC/D,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,wBAAwB,EAAE;IAClD;QACE,2EAA2E;QAC3E,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,2BAA2B;QACpC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;KAC9C;IACD;QACE,wEAAwE;QACxE,wEAAwE;QACxE,uDAAuD;QACvD,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,uFAAuF;QAChG,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;YACZ,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC;YAC3C,OAAO,MAAM,IAAI,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC;QACtC,CAAC;KACF;IACD,iFAAiF;IACjF,2DAA2D;IAC3D;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,kJAAkJ;KAC5J;IACD;QACE,uEAAuE;QACvE,gEAAgE;QAChE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,2BAA2B;QACpC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,mCAAmC;KAC3F;CACF,CAAC;AAEF,iEAAiE;AACjE,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,MAAM,MAAM,GAA2C,EAAE,CAAC;IAC1D,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACjD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,UAAU,CAAC,IAAc;IACvC,MAAM,MAAM,GAA2C,EAAE,CAAC;IAC1D,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,MAAM,CAAC,CAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAkB,CAAC,IAAI,CAAC,CAAC,GAAI,CAAY,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,CAAC,OAAO;YAAE,OAAO,GAAG,IAAI,CAAC;QAC9B,OAAO,CAAC,CAAC,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACtD,CAAC"}
|
package/dist/service.d.ts
CHANGED
|
@@ -38,6 +38,8 @@ interface ServiceDeps {
|
|
|
38
38
|
archiveAfterDays?: number;
|
|
39
39
|
/** Archived memory older than this gets auto-deleted. Default 365. */
|
|
40
40
|
archiveTtlDays?: number;
|
|
41
|
+
/** Force the PII redaction filter on/off (default: REMEMBRA_REDACT env). */
|
|
42
|
+
redact?: boolean;
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
43
45
|
* Transport-agnostic handlers. Both the MCP tools and the HTTP API
|
|
@@ -51,6 +53,7 @@ export declare class MemoryService {
|
|
|
51
53
|
private readonly decayIntervalMs;
|
|
52
54
|
private readonly archiveAfterDays;
|
|
53
55
|
private readonly archiveTtlDays;
|
|
56
|
+
private readonly redactOn;
|
|
54
57
|
private lastDecayRun;
|
|
55
58
|
private decayRunning;
|
|
56
59
|
constructor(db: MemoryBackend, deps?: ServiceDeps);
|
|
@@ -89,6 +92,78 @@ export declare class MemoryService {
|
|
|
89
92
|
ok: boolean;
|
|
90
93
|
text: string;
|
|
91
94
|
}>;
|
|
95
|
+
/** Fetch one memory with its links resolved (audit Phase 8: graph view). */
|
|
96
|
+
get(id: string): Promise<{
|
|
97
|
+
memory: Memory;
|
|
98
|
+
related: ({
|
|
99
|
+
id: string;
|
|
100
|
+
type: "fact" | "decision" | "role" | "history";
|
|
101
|
+
scope: string;
|
|
102
|
+
content: string;
|
|
103
|
+
} | {
|
|
104
|
+
id: string;
|
|
105
|
+
missing: true;
|
|
106
|
+
})[];
|
|
107
|
+
backlinks: {
|
|
108
|
+
id: string;
|
|
109
|
+
type: "fact" | "decision" | "role" | "history";
|
|
110
|
+
scope: string;
|
|
111
|
+
content: string;
|
|
112
|
+
}[];
|
|
113
|
+
text: string;
|
|
114
|
+
}>;
|
|
115
|
+
/**
|
|
116
|
+
* Manage directed links between memories (audit Phase 8: relationship
|
|
117
|
+
* graph). Targets are validated on add; backlinks are derived at read
|
|
118
|
+
* time, so one write keeps the edge consistent.
|
|
119
|
+
*/
|
|
120
|
+
relate(input: unknown): Promise<{
|
|
121
|
+
id: string;
|
|
122
|
+
related: string[];
|
|
123
|
+
added: string[];
|
|
124
|
+
removed: string[];
|
|
125
|
+
text: string;
|
|
126
|
+
}>;
|
|
127
|
+
/**
|
|
128
|
+
* Version history with line diffs (audit Phase 8: diff/history view).
|
|
129
|
+
* Newest first; each past version carries a unified diff against its
|
|
130
|
+
* predecessor (the current version diffs against the newest snapshot).
|
|
131
|
+
*/
|
|
132
|
+
history(input: unknown): Promise<{
|
|
133
|
+
id: string;
|
|
134
|
+
versions: {
|
|
135
|
+
current?: true;
|
|
136
|
+
file?: string;
|
|
137
|
+
at?: string;
|
|
138
|
+
snapshotAt?: string;
|
|
139
|
+
content: string;
|
|
140
|
+
diff: string;
|
|
141
|
+
}[];
|
|
142
|
+
text: string;
|
|
143
|
+
}>;
|
|
144
|
+
/**
|
|
145
|
+
* Redact a content+tags pair, emitting metrics + a log event when anything
|
|
146
|
+
* was found. Returns null when disabled or clean (callers keep originals).
|
|
147
|
+
*/
|
|
148
|
+
private redactPair;
|
|
149
|
+
/** PII redaction of a store input (audit Phase 8). */
|
|
150
|
+
private applyRedaction;
|
|
151
|
+
/** Readiness probe (audit Phase 7): can the backend actually be read? */
|
|
152
|
+
health(): Promise<{
|
|
153
|
+
status: "ok" | "unready";
|
|
154
|
+
version: string;
|
|
155
|
+
uptime_s: number;
|
|
156
|
+
storage: string;
|
|
157
|
+
cache?: {
|
|
158
|
+
size: number;
|
|
159
|
+
capacity: number;
|
|
160
|
+
};
|
|
161
|
+
}>;
|
|
162
|
+
/** Parse-cache stats when the backend exposes them (file backend does). */
|
|
163
|
+
storageStats(): {
|
|
164
|
+
size: number;
|
|
165
|
+
capacity: number;
|
|
166
|
+
} | null;
|
|
92
167
|
/**
|
|
93
168
|
* Session digest (v2) with contradiction-merge (v3):
|
|
94
169
|
* extract worth-keeping memories, skip exact duplicates, and let the LLM
|
package/dist/service.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import { search } from "./retrieval.js";
|
|
2
2
|
import { StoreInput, SnapshotInput, SNAPSHOT_FORMAT, SCHEMA_VERSION } from "./types.js";
|
|
3
|
-
import { RemembraError, inputError } from "./errors.js";
|
|
3
|
+
import { RemembraError, inputError, errorLabel } from "./errors.js";
|
|
4
4
|
import { resolveEmbeddingProvider, embedText, cosine } from "./embeddings.js";
|
|
5
|
+
import { logEvent } from "./log.js";
|
|
6
|
+
import { metrics } from "./metrics.js";
|
|
7
|
+
import { VERSION } from "./version.js";
|
|
8
|
+
import { performance } from "node:perf_hooks";
|
|
9
|
+
import { redact, redactTags, redactionEnabled } from "./redact.js";
|
|
10
|
+
import { unifiedDiff } from "./diff.js";
|
|
11
|
+
import { RelateInput, HistoryInput } from "./types.js";
|
|
5
12
|
import { resolveLlmProvider, extractMemories, resolveMerge, } from "./llm.js";
|
|
6
13
|
/**
|
|
7
14
|
* Transport-agnostic handlers. Both the MCP tools and the HTTP API
|
|
@@ -15,6 +22,7 @@ export class MemoryService {
|
|
|
15
22
|
decayIntervalMs;
|
|
16
23
|
archiveAfterDays;
|
|
17
24
|
archiveTtlDays;
|
|
25
|
+
redactOn;
|
|
18
26
|
lastDecayRun = 0;
|
|
19
27
|
decayRunning = false;
|
|
20
28
|
constructor(db, deps = {}) {
|
|
@@ -29,6 +37,7 @@ export class MemoryService {
|
|
|
29
37
|
this.decayIntervalMs = deps.decayIntervalMs ?? 3_600_000; // 1h
|
|
30
38
|
this.archiveAfterDays = deps.archiveAfterDays ?? Number(process.env.REMEMBRA_ARCHIVE_AFTER_DAYS ?? 90);
|
|
31
39
|
this.archiveTtlDays = deps.archiveTtlDays ?? Number(process.env.REMEMBRA_ARCHIVE_TTL_DAYS ?? 365);
|
|
40
|
+
this.redactOn = deps.redact ?? redactionEnabled();
|
|
32
41
|
}
|
|
33
42
|
get embeddingsEnabled() {
|
|
34
43
|
return this.embedFn !== undefined;
|
|
@@ -41,7 +50,7 @@ export class MemoryService {
|
|
|
41
50
|
}
|
|
42
51
|
catch (err) {
|
|
43
52
|
const msg = err instanceof Error ? err.message : String(err);
|
|
44
|
-
|
|
53
|
+
logEvent("warn", "embedding_failed", { error: msg.slice(0, 200) }, `Remembra: embedding failed (${msg.slice(0, 200)}); continuing without`);
|
|
45
54
|
return undefined;
|
|
46
55
|
}
|
|
47
56
|
}
|
|
@@ -53,10 +62,15 @@ export class MemoryService {
|
|
|
53
62
|
catch (err) {
|
|
54
63
|
throw inputError(err, "INVALID_INPUT");
|
|
55
64
|
}
|
|
65
|
+
// PII redaction (audit Phase 8, opt-in REMEMBRA_REDACT): before embed,
|
|
66
|
+
// before disk, before export — raw patterns never leave this process.
|
|
67
|
+
if (this.redactOn)
|
|
68
|
+
parsed = this.applyRedaction(parsed);
|
|
56
69
|
const embedding = await this.maybeEmbed(parsed.content);
|
|
57
70
|
const memory = await this.db.store(parsed, embedding, {
|
|
58
71
|
provenance: opts?.provenance ?? "explicit",
|
|
59
72
|
});
|
|
73
|
+
metrics.inc("remembra_stores_total");
|
|
60
74
|
return {
|
|
61
75
|
id: memory.id,
|
|
62
76
|
message: `Stored ${memory.type} memory ${memory.id} (scope: ${memory.scope})`,
|
|
@@ -64,14 +78,28 @@ export class MemoryService {
|
|
|
64
78
|
};
|
|
65
79
|
}
|
|
66
80
|
async search(q) {
|
|
81
|
+
const t0 = performance.now();
|
|
67
82
|
let queryVec = null;
|
|
68
83
|
if (q.query && this.embedFn)
|
|
69
84
|
queryVec = (await this.maybeEmbed(q.query)) ?? null;
|
|
70
85
|
const results = search(await this.db.all(), q, queryVec);
|
|
86
|
+
const durationMs = performance.now() - t0;
|
|
87
|
+
// Observability (audit Phase 7): counters + hygiene-first query logging —
|
|
88
|
+
// raw query text only under REMEMBRA_DEBUG (same rule as the root path).
|
|
89
|
+
metrics.inc("remembra_searches_total");
|
|
90
|
+
metrics.observe("remembra_search_duration_seconds", durationMs / 1000);
|
|
91
|
+
logEvent("info", "search", {
|
|
92
|
+
scope: q.scope,
|
|
93
|
+
terms: (q.query ?? "").split(/\s+/).filter((t) => t.length > 1).length,
|
|
94
|
+
results: results.length,
|
|
95
|
+
limit: q.limit,
|
|
96
|
+
duration_ms: Math.round(durationMs * 10) / 10,
|
|
97
|
+
...(process.env.REMEMBRA_DEBUG && q.query ? { query: q.query } : {}),
|
|
98
|
+
});
|
|
71
99
|
// Refresh decay clocks for memories that surfaced (fire-and-forget).
|
|
72
100
|
for (const m of results)
|
|
73
101
|
this.db.touch(m.id).catch((err) => {
|
|
74
|
-
|
|
102
|
+
logEvent("warn", "touch_failed", { id: m.id, error: String(err).slice(0, 150) }, `Remembra: touch failed (${m.id}): ${String(err).slice(0, 150)}`);
|
|
75
103
|
});
|
|
76
104
|
// Opportunistic decay pass, debounced (decision v3-Q1: piggyback on search).
|
|
77
105
|
this.maybeRunDecay();
|
|
@@ -117,6 +145,173 @@ export class MemoryService {
|
|
|
117
145
|
const ok = await this.db.forget(id);
|
|
118
146
|
return { ok, text: ok ? `Deleted memory ${id}.` : `No memory with id ${id}.` };
|
|
119
147
|
}
|
|
148
|
+
/** Fetch one memory with its links resolved (audit Phase 8: graph view). */
|
|
149
|
+
async get(id) {
|
|
150
|
+
const memory = await this.db.get(id);
|
|
151
|
+
if (!memory)
|
|
152
|
+
throw new RemembraError("NOT_FOUND", `No memory with id ${id}`);
|
|
153
|
+
const all = await this.db.all(true);
|
|
154
|
+
const brief = (m) => ({
|
|
155
|
+
id: m.id,
|
|
156
|
+
type: m.type,
|
|
157
|
+
scope: m.scope,
|
|
158
|
+
content: m.content.split("\n")[0],
|
|
159
|
+
});
|
|
160
|
+
const related = (memory.related ?? []).map((rid) => {
|
|
161
|
+
const target = all.find((m) => m.id === rid);
|
|
162
|
+
return target ? brief(target) : { id: rid, missing: true };
|
|
163
|
+
});
|
|
164
|
+
const backlinks = all.filter((m) => m.id !== id && (m.related ?? []).includes(id)).map(brief);
|
|
165
|
+
const meta = `${memory.type} (scope: ${memory.scope}, importance: ${memory.importance}` +
|
|
166
|
+
`${memory.confidence !== undefined ? `, confidence: ${memory.confidence}` : ""})`;
|
|
167
|
+
const text = `[${memory.id}] ${meta}\n${memory.content}` +
|
|
168
|
+
(related.length ? `\n\nRelated: ${related.map((r) => r.id).join(", ")}` : "") +
|
|
169
|
+
(backlinks.length ? `\nReferenced by: ${backlinks.map((b) => b.id).join(", ")}` : "");
|
|
170
|
+
return { memory, related, backlinks, text };
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Manage directed links between memories (audit Phase 8: relationship
|
|
174
|
+
* graph). Targets are validated on add; backlinks are derived at read
|
|
175
|
+
* time, so one write keeps the edge consistent.
|
|
176
|
+
*/
|
|
177
|
+
async relate(input) {
|
|
178
|
+
let parsed;
|
|
179
|
+
try {
|
|
180
|
+
parsed = RelateInput.parse(input);
|
|
181
|
+
}
|
|
182
|
+
catch (err) {
|
|
183
|
+
throw inputError(err, "INVALID_INPUT");
|
|
184
|
+
}
|
|
185
|
+
const memory = await this.db.get(parsed.id);
|
|
186
|
+
if (!memory)
|
|
187
|
+
throw new RemembraError("NOT_FOUND", `No memory with id ${parsed.id}`);
|
|
188
|
+
if (parsed.related.includes(parsed.id)) {
|
|
189
|
+
throw new RemembraError("INVALID_INPUT", "a memory cannot be related to itself");
|
|
190
|
+
}
|
|
191
|
+
if (parsed.action === "add") {
|
|
192
|
+
const missing = [];
|
|
193
|
+
for (const rid of parsed.related) {
|
|
194
|
+
if (!(await this.db.get(rid)))
|
|
195
|
+
missing.push(rid);
|
|
196
|
+
}
|
|
197
|
+
if (missing.length > 0) {
|
|
198
|
+
throw new RemembraError("NOT_FOUND", `related target(s) not found: ${missing.join(", ")}`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const current = memory.related ?? [];
|
|
202
|
+
const added = parsed.action === "add" ? parsed.related.filter((r) => !current.includes(r)) : [];
|
|
203
|
+
const removed = parsed.action === "remove" ? current.filter((r) => parsed.related.includes(r)) : [];
|
|
204
|
+
let next = parsed.action === "add" ? [...current, ...added] : current.filter((r) => !parsed.related.includes(r));
|
|
205
|
+
if (added.length === 0 && removed.length === 0) {
|
|
206
|
+
next = current; // no-op: don't churn updatedAt for an idempotent call
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
await this.db.update({ ...memory, related: next });
|
|
210
|
+
metrics.inc("remembra_relate_total", { action: parsed.action });
|
|
211
|
+
}
|
|
212
|
+
const verb = parsed.action === "add" ? "Linked" : "Unlinked";
|
|
213
|
+
const changed = parsed.action === "add" ? added : removed;
|
|
214
|
+
const text = changed.length > 0
|
|
215
|
+
? `${verb} ${parsed.id} ${parsed.action === "add" ? "→" : "⇁"} ${changed.join(", ")}`
|
|
216
|
+
: `No change: ${parsed.id} links unchanged (${next.length} total)`;
|
|
217
|
+
return { id: parsed.id, related: next, added, removed, text };
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Version history with line diffs (audit Phase 8: diff/history view).
|
|
221
|
+
* Newest first; each past version carries a unified diff against its
|
|
222
|
+
* predecessor (the current version diffs against the newest snapshot).
|
|
223
|
+
*/
|
|
224
|
+
async history(input) {
|
|
225
|
+
let parsed;
|
|
226
|
+
try {
|
|
227
|
+
parsed = HistoryInput.parse(input);
|
|
228
|
+
}
|
|
229
|
+
catch (err) {
|
|
230
|
+
throw inputError(err, "INVALID_INPUT");
|
|
231
|
+
}
|
|
232
|
+
const memory = await this.db.get(parsed.id);
|
|
233
|
+
if (!memory)
|
|
234
|
+
throw new RemembraError("NOT_FOUND", `No memory with id ${parsed.id}`);
|
|
235
|
+
const entries = (await this.db.history?.(parsed.id)) ?? [];
|
|
236
|
+
const kept = entries.slice(0, parsed.limit ?? entries.length);
|
|
237
|
+
const chain = [
|
|
238
|
+
{ base: { current: true, at: memory.updatedAt }, content: memory.content },
|
|
239
|
+
...kept.map((e) => ({
|
|
240
|
+
base: { file: e.file, at: e.at, snapshotAt: e.snapshotAt },
|
|
241
|
+
content: e.content,
|
|
242
|
+
})),
|
|
243
|
+
];
|
|
244
|
+
const versions = chain.map((v, i) => {
|
|
245
|
+
const older = chain[i + 1];
|
|
246
|
+
const diff = older
|
|
247
|
+
? unifiedDiff(older.content, v.content, older.base.at ?? "older", v.base.at ?? "current")
|
|
248
|
+
: "";
|
|
249
|
+
return { ...v.base, content: v.content, diff };
|
|
250
|
+
});
|
|
251
|
+
const lines = [`History for ${parsed.id} — ${versions.length} version(s), newest first:`];
|
|
252
|
+
for (const v of versions) {
|
|
253
|
+
const label = v.current
|
|
254
|
+
? "current"
|
|
255
|
+
: `superseded ${v.snapshotAt?.slice(0, 19) ?? "?"} (was current: ${v.at?.slice(0, 19) ?? "?"})`;
|
|
256
|
+
lines.push("", `# ${label} — ${v.content.split("\n")[0]}`);
|
|
257
|
+
if (v.diff)
|
|
258
|
+
lines.push(v.diff.trimEnd());
|
|
259
|
+
}
|
|
260
|
+
return { id: parsed.id, versions, text: lines.join("\n") };
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Redact a content+tags pair, emitting metrics + a log event when anything
|
|
264
|
+
* was found. Returns null when disabled or clean (callers keep originals).
|
|
265
|
+
*/
|
|
266
|
+
redactPair(content, tags) {
|
|
267
|
+
if (!this.redactOn)
|
|
268
|
+
return null;
|
|
269
|
+
const c = redact(content);
|
|
270
|
+
const t = redactTags(tags);
|
|
271
|
+
const counts = {};
|
|
272
|
+
for (const [k, v] of Object.entries(c.counts))
|
|
273
|
+
counts[k] = v;
|
|
274
|
+
for (const [k, v] of Object.entries(t.counts)) {
|
|
275
|
+
counts[k] = (counts[k] ?? 0) + v;
|
|
276
|
+
}
|
|
277
|
+
if (Object.keys(counts).length === 0)
|
|
278
|
+
return null;
|
|
279
|
+
for (const [kind, n] of Object.entries(counts)) {
|
|
280
|
+
if (n)
|
|
281
|
+
metrics.inc("remembra_redactions_total", { kind }, n);
|
|
282
|
+
}
|
|
283
|
+
logEvent("info", "redacted", { ...counts }, `Remembra: redacted PII at ingest (${JSON.stringify(counts)})`);
|
|
284
|
+
return { content: c.text, tags: t.tags };
|
|
285
|
+
}
|
|
286
|
+
/** PII redaction of a store input (audit Phase 8). */
|
|
287
|
+
applyRedaction(parsed) {
|
|
288
|
+
const clean = this.redactPair(parsed.content, parsed.tags);
|
|
289
|
+
if (!clean)
|
|
290
|
+
return parsed;
|
|
291
|
+
return { ...parsed, content: clean.content, tags: clean.tags };
|
|
292
|
+
}
|
|
293
|
+
/** Readiness probe (audit Phase 7): can the backend actually be read? */
|
|
294
|
+
async health() {
|
|
295
|
+
let storage = "ok";
|
|
296
|
+
try {
|
|
297
|
+
await this.db.all();
|
|
298
|
+
}
|
|
299
|
+
catch (err) {
|
|
300
|
+
storage = errorLabel(err);
|
|
301
|
+
}
|
|
302
|
+
const cache = this.storageStats();
|
|
303
|
+
return {
|
|
304
|
+
status: storage === "ok" ? "ok" : "unready",
|
|
305
|
+
version: VERSION,
|
|
306
|
+
uptime_s: Math.round(process.uptime()),
|
|
307
|
+
storage,
|
|
308
|
+
...(cache ? { cache } : {}),
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
/** Parse-cache stats when the backend exposes them (file backend does). */
|
|
312
|
+
storageStats() {
|
|
313
|
+
return this.db.cacheStats?.() ?? null;
|
|
314
|
+
}
|
|
120
315
|
/**
|
|
121
316
|
* Session digest (v2) with contradiction-merge (v3):
|
|
122
317
|
* extract worth-keeping memories, skip exact duplicates, and let the LLM
|
|
@@ -126,9 +321,23 @@ export class MemoryService {
|
|
|
126
321
|
* read the same active set and double-store duplicates.
|
|
127
322
|
*/
|
|
128
323
|
async digest(opts) {
|
|
324
|
+
const t0 = performance.now();
|
|
129
325
|
const run = this.digestLock.then(() => this.doDigest(opts));
|
|
130
326
|
this.digestLock = run.then(() => undefined, () => undefined);
|
|
131
|
-
|
|
327
|
+
const observe = () => {
|
|
328
|
+
metrics.observe("remembra_digest_duration_seconds", (performance.now() - t0) / 1000);
|
|
329
|
+
};
|
|
330
|
+
return run.then((res) => {
|
|
331
|
+
metrics.inc("remembra_digests_total");
|
|
332
|
+
metrics.inc("remembra_digest_items_total", { result: "stored" }, res.stored.length);
|
|
333
|
+
metrics.inc("remembra_digest_items_total", { result: "skipped" }, res.skippedDuplicates);
|
|
334
|
+
metrics.inc("remembra_digest_items_total", { result: "merged" }, res.merged);
|
|
335
|
+
observe();
|
|
336
|
+
return res;
|
|
337
|
+
}, (err) => {
|
|
338
|
+
observe();
|
|
339
|
+
throw err;
|
|
340
|
+
});
|
|
132
341
|
}
|
|
133
342
|
digestLock = Promise.resolve();
|
|
134
343
|
async doDigest(opts) {
|
|
@@ -146,7 +355,13 @@ export class MemoryService {
|
|
|
146
355
|
const stored = [];
|
|
147
356
|
let skippedDuplicates = 0;
|
|
148
357
|
let merged = 0;
|
|
149
|
-
for (const
|
|
358
|
+
for (const extractedItem of extracted) {
|
|
359
|
+
// PII redaction at the digest boundary: extraction LLM sees the raw
|
|
360
|
+
// transcript (it must, to understand it) — storage never does.
|
|
361
|
+
const clean = this.redactPair(extractedItem.content, extractedItem.tags);
|
|
362
|
+
const item = clean
|
|
363
|
+
? { ...extractedItem, content: clean.content, tags: clean.tags }
|
|
364
|
+
: extractedItem;
|
|
150
365
|
const scope = item.scope ?? opts.scope ?? "global";
|
|
151
366
|
const key = dedupKey(item.type, item.content, scope);
|
|
152
367
|
// Exact duplicate → skip (or revive if it had decayed).
|
|
@@ -195,7 +410,7 @@ export class MemoryService {
|
|
|
195
410
|
});
|
|
196
411
|
}
|
|
197
412
|
catch (err) {
|
|
198
|
-
|
|
413
|
+
logEvent("warn", "merge_llm_failed", { error: String(err).slice(0, 150) }, `Remembra: merge LLM failed (${String(err).slice(0, 150)}); storing fresh`);
|
|
199
414
|
decision = { action: "store" };
|
|
200
415
|
}
|
|
201
416
|
if (decision.action === "skip") {
|
|
@@ -205,13 +420,17 @@ export class MemoryService {
|
|
|
205
420
|
if (decision.action === "merge") {
|
|
206
421
|
const now = new Date().toISOString().slice(0, 10);
|
|
207
422
|
const old = candidate.content.split("\n")[0];
|
|
208
|
-
const
|
|
209
|
-
|
|
423
|
+
const cleanMerged = this.redactPair(decision.content, []) ?? {
|
|
424
|
+
content: decision.content,
|
|
425
|
+
tags: [],
|
|
426
|
+
};
|
|
427
|
+
const content = `${cleanMerged.content}\n\n> superseded (${now}): ${old}`;
|
|
428
|
+
const embedding = await this.maybeEmbed(cleanMerged.content);
|
|
210
429
|
await this.db.update({ ...candidate, content, embedding, source: opts.source ?? candidate.source });
|
|
211
430
|
merged++;
|
|
212
431
|
// Re-index dedup set against the new content.
|
|
213
432
|
seen.delete(dedupKey(candidate.type, candidate.content, candidate.scope));
|
|
214
|
-
seen.add(dedupKey(candidate.type,
|
|
433
|
+
seen.add(dedupKey(candidate.type, cleanMerged.content, candidate.scope));
|
|
215
434
|
continue;
|
|
216
435
|
}
|
|
217
436
|
// action: "store" → fall through and store fresh
|
|
@@ -224,6 +443,7 @@ export class MemoryService {
|
|
|
224
443
|
tags: item.tags,
|
|
225
444
|
importance: item.importance,
|
|
226
445
|
source: opts.source,
|
|
446
|
+
confidence: item.confidence,
|
|
227
447
|
}, { provenance: "auto" });
|
|
228
448
|
stored.push(memory);
|
|
229
449
|
active.push(memory); // so later items in this batch dedup against it
|
|
@@ -339,7 +559,7 @@ export class MemoryService {
|
|
|
339
559
|
this.lastDecayRun = Date.now();
|
|
340
560
|
this.decayRunning = true;
|
|
341
561
|
this.decayPass()
|
|
342
|
-
.catch((err) =>
|
|
562
|
+
.catch((err) => logEvent("warn", "decay_failed", { error: String(err).slice(0, 200) }, `Remembra: decay pass failed: ${err instanceof Error ? err.message : String(err)}`))
|
|
343
563
|
.finally(() => {
|
|
344
564
|
this.decayRunning = false;
|
|
345
565
|
});
|