@miller-tech/uap 1.5.4 → 1.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -0
- package/dist/bin/cli.js +6 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/bin/llama-server-optimize.js +0 -0
- package/dist/bin/policy.js +0 -0
- package/dist/bin/tool-calls.js +0 -0
- package/dist/cli/dashboard.d.ts +1 -0
- package/dist/cli/dashboard.d.ts.map +1 -1
- package/dist/cli/dashboard.js.map +1 -1
- package/dist/memory/active-context.d.ts +65 -0
- package/dist/memory/active-context.d.ts.map +1 -0
- package/dist/memory/active-context.js +126 -0
- package/dist/memory/active-context.js.map +1 -0
- package/dist/memory/dedup-detector.d.ts +57 -0
- package/dist/memory/dedup-detector.d.ts.map +1 -0
- package/dist/memory/dedup-detector.js +107 -0
- package/dist/memory/dedup-detector.js.map +1 -0
- package/dist/memory/dedup-memory.d.ts +89 -0
- package/dist/memory/dedup-memory.d.ts.map +1 -0
- package/dist/memory/dedup-memory.js +173 -0
- package/dist/memory/dedup-memory.js.map +1 -0
- package/dist/memory/merge-claude-md.d.ts +45 -0
- package/dist/memory/merge-claude-md.d.ts.map +1 -0
- package/dist/memory/merge-claude-md.js +118 -0
- package/dist/memory/merge-claude-md.js.map +1 -0
- package/dist/memory/patterns.d.ts +37 -0
- package/dist/memory/patterns.d.ts.map +1 -0
- package/dist/memory/patterns.js +81 -0
- package/dist/memory/patterns.js.map +1 -0
- package/dist/memory/semantic-edge-graph.d.ts +86 -0
- package/dist/memory/semantic-edge-graph.d.ts.map +1 -0
- package/dist/memory/semantic-edge-graph.js +168 -0
- package/dist/memory/semantic-edge-graph.js.map +1 -0
- package/dist/memory/semantic-retrieval.d.ts +70 -0
- package/dist/memory/semantic-retrieval.d.ts.map +1 -0
- package/dist/memory/semantic-retrieval.js +112 -0
- package/dist/memory/semantic-retrieval.js.map +1 -0
- package/dist/memory/smart-consolidator.d.ts +53 -0
- package/dist/memory/smart-consolidator.d.ts.map +1 -0
- package/dist/memory/smart-consolidator.js +144 -0
- package/dist/memory/smart-consolidator.js.map +1 -0
- package/dist/memory/view-memory.d.ts +80 -0
- package/dist/memory/view-memory.d.ts.map +1 -0
- package/dist/memory/view-memory.js +130 -0
- package/dist/memory/view-memory.js.map +1 -0
- package/dist/memory/wrapped-memory.d.ts +77 -0
- package/dist/memory/wrapped-memory.d.ts.map +1 -0
- package/dist/memory/wrapped-memory.js +127 -0
- package/dist/memory/wrapped-memory.js.map +1 -0
- package/dist/telemetry/session-telemetry.d.ts +5 -0
- package/dist/telemetry/session-telemetry.d.ts.map +1 -1
- package/dist/telemetry/session-telemetry.js +243 -0
- package/dist/telemetry/session-telemetry.js.map +1 -1
- package/dist/utils/adaptive-cache.d.ts +67 -0
- package/dist/utils/adaptive-cache.d.ts.map +1 -0
- package/dist/utils/adaptive-cache.js +149 -0
- package/dist/utils/adaptive-cache.js.map +1 -0
- package/dist/utils/concurrency.d.ts +33 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +88 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/file-discovery.d.ts +38 -0
- package/dist/utils/file-discovery.d.ts.map +1 -0
- package/dist/utils/file-discovery.js +100 -0
- package/dist/utils/file-discovery.js.map +1 -0
- package/dist/utils/performance-monitor.d.ts +52 -0
- package/dist/utils/performance-monitor.d.ts.map +1 -0
- package/dist/utils/performance-monitor.js +103 -0
- package/dist/utils/performance-monitor.js.map +1 -0
- package/package.json +5 -3
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Active Context Module for UAP
|
|
3
|
+
*
|
|
4
|
+
* Implements context-aware memory management with automatic activation/deactivation
|
|
5
|
+
* based on relevance scoring and temporal proximity.
|
|
6
|
+
*/
|
|
7
|
+
export interface ActiveContextConfig {
|
|
8
|
+
maxActiveEntries: number;
|
|
9
|
+
relevanceThreshold: number;
|
|
10
|
+
decayRate: number;
|
|
11
|
+
activationWindow: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ActiveEntry {
|
|
14
|
+
id: string;
|
|
15
|
+
content: string;
|
|
16
|
+
relevance: number;
|
|
17
|
+
lastActive: Date;
|
|
18
|
+
accessCount: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Active Context Manager
|
|
22
|
+
* Manages a set of active memory entries based on relevance and recency
|
|
23
|
+
*/
|
|
24
|
+
export declare class ActiveContextManager {
|
|
25
|
+
private config;
|
|
26
|
+
private entries;
|
|
27
|
+
constructor(config?: Partial<ActiveContextConfig>);
|
|
28
|
+
/**
|
|
29
|
+
* Add or update an entry in active context
|
|
30
|
+
*/
|
|
31
|
+
activate(id: string, content: string, relevance: number): ActiveEntry;
|
|
32
|
+
/**
|
|
33
|
+
* Deactivate an entry (remove from active context)
|
|
34
|
+
*/
|
|
35
|
+
deactivate(id: string): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Get all active entries sorted by relevance
|
|
38
|
+
*/
|
|
39
|
+
getActiveEntries(limit?: number): ActiveEntry[];
|
|
40
|
+
/**
|
|
41
|
+
* Check if an entry is active
|
|
42
|
+
*/
|
|
43
|
+
isActive(id: string): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Get relevance score for an entry
|
|
46
|
+
*/
|
|
47
|
+
getRelevance(id: string): number | null;
|
|
48
|
+
/**
|
|
49
|
+
* Decay relevance scores over time
|
|
50
|
+
*/
|
|
51
|
+
decay(): void;
|
|
52
|
+
/**
|
|
53
|
+
* Rebalance entries to maintain max size
|
|
54
|
+
*/
|
|
55
|
+
private rebalance;
|
|
56
|
+
/**
|
|
57
|
+
* Clear all active entries
|
|
58
|
+
*/
|
|
59
|
+
clear(): void;
|
|
60
|
+
/**
|
|
61
|
+
* Get count of active entries
|
|
62
|
+
*/
|
|
63
|
+
getCount(): number;
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=active-context.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active-context.d.ts","sourceRoot":"","sources":["../../src/memory/active-context.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AASD,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,IAAI,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,OAAO,CAAuC;gBAE1C,MAAM,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAIrD;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW;IA0BrE;;OAEG;IACH,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI/B;;OAEG;IACH,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE;IAM/C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI7B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAKvC;;OAEG;IACH,KAAK,IAAI,IAAI;IAmBb;;OAEG;IACH,OAAO,CAAC,SAAS;IAmBjB;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,QAAQ,IAAI,MAAM;CAGnB"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Active Context Module for UAP
|
|
3
|
+
*
|
|
4
|
+
* Implements context-aware memory management with automatic activation/deactivation
|
|
5
|
+
* based on relevance scoring and temporal proximity.
|
|
6
|
+
*/
|
|
7
|
+
const DEFAULT_CONFIG = {
|
|
8
|
+
maxActiveEntries: 20,
|
|
9
|
+
relevanceThreshold: 0.6,
|
|
10
|
+
decayRate: 0.95,
|
|
11
|
+
activationWindow: 24,
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Active Context Manager
|
|
15
|
+
* Manages a set of active memory entries based on relevance and recency
|
|
16
|
+
*/
|
|
17
|
+
export class ActiveContextManager {
|
|
18
|
+
config;
|
|
19
|
+
entries = new Map();
|
|
20
|
+
constructor(config = {}) {
|
|
21
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Add or update an entry in active context
|
|
25
|
+
*/
|
|
26
|
+
activate(id, content, relevance) {
|
|
27
|
+
const now = new Date();
|
|
28
|
+
let entry;
|
|
29
|
+
if (this.entries.has(id)) {
|
|
30
|
+
entry = this.entries.get(id);
|
|
31
|
+
entry.content = content;
|
|
32
|
+
entry.relevance = relevance;
|
|
33
|
+
entry.accessCount++;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
entry = {
|
|
37
|
+
id,
|
|
38
|
+
content,
|
|
39
|
+
relevance,
|
|
40
|
+
lastActive: now,
|
|
41
|
+
accessCount: 1,
|
|
42
|
+
};
|
|
43
|
+
this.entries.set(id, entry);
|
|
44
|
+
}
|
|
45
|
+
// Rebalance active entries
|
|
46
|
+
this.rebalance();
|
|
47
|
+
return entry;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Deactivate an entry (remove from active context)
|
|
51
|
+
*/
|
|
52
|
+
deactivate(id) {
|
|
53
|
+
return this.entries.delete(id);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Get all active entries sorted by relevance
|
|
57
|
+
*/
|
|
58
|
+
getActiveEntries(limit) {
|
|
59
|
+
const entries = Array.from(this.entries.values());
|
|
60
|
+
entries.sort((a, b) => b.relevance - a.relevance);
|
|
61
|
+
return limit ? entries.slice(0, limit) : entries;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Check if an entry is active
|
|
65
|
+
*/
|
|
66
|
+
isActive(id) {
|
|
67
|
+
return this.entries.has(id);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Get relevance score for an entry
|
|
71
|
+
*/
|
|
72
|
+
getRelevance(id) {
|
|
73
|
+
const entry = this.entries.get(id);
|
|
74
|
+
return entry ? entry.relevance : null;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Decay relevance scores over time
|
|
78
|
+
*/
|
|
79
|
+
decay() {
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
const windowMs = this.config.activationWindow * 60 * 60 * 1000;
|
|
82
|
+
for (const [id, entry] of this.entries) {
|
|
83
|
+
const age = now - entry.lastActive.getTime();
|
|
84
|
+
// Decay based on age
|
|
85
|
+
if (age > windowMs) {
|
|
86
|
+
entry.relevance *= Math.pow(this.config.decayRate, age / (1000 * 60 * 60));
|
|
87
|
+
// Auto-deactivate if relevance drops below threshold
|
|
88
|
+
if (entry.relevance < this.config.relevanceThreshold) {
|
|
89
|
+
this.entries.delete(id);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Rebalance entries to maintain max size
|
|
96
|
+
*/
|
|
97
|
+
rebalance() {
|
|
98
|
+
const entries = Array.from(this.entries.values());
|
|
99
|
+
if (entries.length > this.config.maxActiveEntries) {
|
|
100
|
+
// Sort by relevance and access count
|
|
101
|
+
entries.sort((a, b) => {
|
|
102
|
+
const scoreA = a.relevance * 0.7 + (a.accessCount / Math.max(1, a.accessCount)) * 0.3;
|
|
103
|
+
const scoreB = b.relevance * 0.7 + (b.accessCount / Math.max(1, b.accessCount)) * 0.3;
|
|
104
|
+
return scoreA - scoreB;
|
|
105
|
+
});
|
|
106
|
+
// Remove lowest relevance entries
|
|
107
|
+
const toRemove = entries.slice(0, entries.length - this.config.maxActiveEntries);
|
|
108
|
+
for (const entry of toRemove) {
|
|
109
|
+
this.entries.delete(entry.id);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Clear all active entries
|
|
115
|
+
*/
|
|
116
|
+
clear() {
|
|
117
|
+
this.entries.clear();
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Get count of active entries
|
|
121
|
+
*/
|
|
122
|
+
getCount() {
|
|
123
|
+
return this.entries.size;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=active-context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"active-context.js","sourceRoot":"","sources":["../../src/memory/active-context.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AASH,MAAM,cAAc,GAAwB;IAC1C,gBAAgB,EAAE,EAAE;IACpB,kBAAkB,EAAE,GAAG;IACvB,SAAS,EAAE,IAAI;IACf,gBAAgB,EAAE,EAAE;CACrB,CAAC;AAUF;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAsB;IAC5B,OAAO,GAA6B,IAAI,GAAG,EAAE,CAAC;IAEtD,YAAY,SAAuC,EAAE;QACnD,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,EAAU,EAAE,OAAe,EAAE,SAAiB;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,IAAI,KAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACzB,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;YAC9B,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;YACxB,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;YAC5B,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,GAAG;gBACN,EAAE;gBACF,OAAO;gBACP,SAAS;gBACT,UAAU,EAAE,GAAG;gBACf,WAAW,EAAE,CAAC;aACf,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,EAAU;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAc;QAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,EAAU;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,EAAU;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAE/D,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAE7C,qBAAqB;YACrB,IAAI,GAAG,GAAG,QAAQ,EAAE,CAAC;gBACnB,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAE3E,qDAAqD;gBACrD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;oBACrD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,SAAS;QACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAElD,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAClD,qCAAqC;YACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACpB,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC;gBACtF,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC;gBACtF,OAAO,MAAM,GAAG,MAAM,CAAC;YACzB,CAAC,CAAC,CAAC;YAEH,kCAAkC;YAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACjF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;CACF"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dedup Detector Module for UAP
|
|
3
|
+
*
|
|
4
|
+
* Detects duplicate and near-duplicate content in memory entries.
|
|
5
|
+
*/
|
|
6
|
+
export interface MemoryEntry {
|
|
7
|
+
id: string;
|
|
8
|
+
content: string;
|
|
9
|
+
type: string;
|
|
10
|
+
timestamp: string;
|
|
11
|
+
}
|
|
12
|
+
export interface DuplicateResult {
|
|
13
|
+
primaryId: string;
|
|
14
|
+
duplicateIds: string[];
|
|
15
|
+
similarity: number;
|
|
16
|
+
}
|
|
17
|
+
export interface DedupDetectorConfig {
|
|
18
|
+
similarityThreshold: number;
|
|
19
|
+
minContentLength: number;
|
|
20
|
+
exactMatch: boolean;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Dedup Detector
|
|
24
|
+
* Detects duplicate content in memory entries
|
|
25
|
+
*/
|
|
26
|
+
export declare class DedupDetector {
|
|
27
|
+
private config;
|
|
28
|
+
constructor(config?: Partial<DedupDetectorConfig>);
|
|
29
|
+
/**
|
|
30
|
+
* Detect duplicates in a batch of entries
|
|
31
|
+
*/
|
|
32
|
+
detectDuplicates(entries: MemoryEntry[]): DuplicateResult[];
|
|
33
|
+
/**
|
|
34
|
+
* Check if two pieces of content are duplicates
|
|
35
|
+
*/
|
|
36
|
+
isDuplicate(content1: string, content2: string): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Compute similarity between two contents
|
|
39
|
+
*/
|
|
40
|
+
computeSimilarity(content1: string, content2: string): number;
|
|
41
|
+
/**
|
|
42
|
+
* Find the most similar entry to a query
|
|
43
|
+
*/
|
|
44
|
+
findMostSimilar(query: string, entries: MemoryEntry[]): {
|
|
45
|
+
id: string;
|
|
46
|
+
similarity: number;
|
|
47
|
+
} | null;
|
|
48
|
+
/**
|
|
49
|
+
* Get duplicate statistics
|
|
50
|
+
*/
|
|
51
|
+
getStats(entries: MemoryEntry[]): {
|
|
52
|
+
totalEntries: number;
|
|
53
|
+
duplicatesFound: number;
|
|
54
|
+
uniqueEntries: number;
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=dedup-detector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup-detector.d.ts","sourceRoot":"","sources":["../../src/memory/dedup-detector.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,OAAO,CAAC;CACrB;AAQD;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAsB;gBAExB,MAAM,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAIrD;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,eAAe,EAAE;IAkC3D;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAmBxD;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAI7D;;OAEG;IACH,eAAe,CACb,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,WAAW,EAAE,GACrB;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAiB5C;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG;QAChC,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB;CAcF"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dedup Detector Module for UAP
|
|
3
|
+
*
|
|
4
|
+
* Detects duplicate and near-duplicate content in memory entries.
|
|
5
|
+
*/
|
|
6
|
+
import { jaccardSimilarity } from '../utils/string-similarity.js';
|
|
7
|
+
const DEFAULT_CONFIG = {
|
|
8
|
+
similarityThreshold: 0.85,
|
|
9
|
+
minContentLength: 20,
|
|
10
|
+
exactMatch: true,
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Dedup Detector
|
|
14
|
+
* Detects duplicate content in memory entries
|
|
15
|
+
*/
|
|
16
|
+
export class DedupDetector {
|
|
17
|
+
config;
|
|
18
|
+
constructor(config = {}) {
|
|
19
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Detect duplicates in a batch of entries
|
|
23
|
+
*/
|
|
24
|
+
detectDuplicates(entries) {
|
|
25
|
+
const results = [];
|
|
26
|
+
const processed = new Set();
|
|
27
|
+
for (let i = 0; i < entries.length; i++) {
|
|
28
|
+
if (processed.has(entries[i].id))
|
|
29
|
+
continue;
|
|
30
|
+
const primary = entries[i];
|
|
31
|
+
const duplicates = [];
|
|
32
|
+
for (let j = i + 1; j < entries.length; j++) {
|
|
33
|
+
const other = entries[j];
|
|
34
|
+
if (this.isDuplicate(primary.content, other.content)) {
|
|
35
|
+
duplicates.push(other.id);
|
|
36
|
+
processed.add(other.id);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (duplicates.length > 0) {
|
|
40
|
+
const dupEntry = entries.find((e) => e.id === duplicates[0]);
|
|
41
|
+
results.push({
|
|
42
|
+
primaryId: primary.id,
|
|
43
|
+
duplicateIds: duplicates,
|
|
44
|
+
similarity: this.computeSimilarity(primary.content, dupEntry.content),
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
processed.add(primary.id);
|
|
48
|
+
}
|
|
49
|
+
return results;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Check if two pieces of content are duplicates
|
|
53
|
+
*/
|
|
54
|
+
isDuplicate(content1, content2) {
|
|
55
|
+
// Exact match
|
|
56
|
+
if (this.config.exactMatch && content1 === content2) {
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
// Length check
|
|
60
|
+
if (content1.length < this.config.minContentLength ||
|
|
61
|
+
content2.length < this.config.minContentLength) {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
// Similarity check
|
|
65
|
+
const similarity = jaccardSimilarity(content1, content2);
|
|
66
|
+
return similarity >= this.config.similarityThreshold;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Compute similarity between two contents
|
|
70
|
+
*/
|
|
71
|
+
computeSimilarity(content1, content2) {
|
|
72
|
+
return jaccardSimilarity(content1, content2);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Find the most similar entry to a query
|
|
76
|
+
*/
|
|
77
|
+
findMostSimilar(query, entries) {
|
|
78
|
+
let bestId = null;
|
|
79
|
+
let bestSimilarity = 0;
|
|
80
|
+
for (const entry of entries) {
|
|
81
|
+
const similarity = this.computeSimilarity(query, entry.content);
|
|
82
|
+
if (similarity > bestSimilarity) {
|
|
83
|
+
bestSimilarity = similarity;
|
|
84
|
+
bestId = entry.id;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return bestSimilarity >= this.config.similarityThreshold
|
|
88
|
+
? { id: bestId, similarity: bestSimilarity }
|
|
89
|
+
: null;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Get duplicate statistics
|
|
93
|
+
*/
|
|
94
|
+
getStats(entries) {
|
|
95
|
+
const duplicates = this.detectDuplicates(entries);
|
|
96
|
+
let totalDuplicates = 0;
|
|
97
|
+
for (const dup of duplicates) {
|
|
98
|
+
totalDuplicates += dup.duplicateIds.length;
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
totalEntries: entries.length,
|
|
102
|
+
duplicatesFound: totalDuplicates,
|
|
103
|
+
uniqueEntries: entries.length - totalDuplicates,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=dedup-detector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup-detector.js","sourceRoot":"","sources":["../../src/memory/dedup-detector.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAqBlE,MAAM,cAAc,GAAwB;IAC1C,mBAAmB,EAAE,IAAI;IACzB,gBAAgB,EAAE,EAAE;IACpB,UAAU,EAAE,IAAI;CACjB,CAAC;AAEF;;;GAGG;AACH,MAAM,OAAO,aAAa;IAChB,MAAM,CAAsB;IAEpC,YAAY,SAAuC,EAAE;QACnD,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,OAAsB;QACrC,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,SAAS;YAE3C,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,UAAU,GAAa,EAAE,CAAC;YAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAEzB,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACrD,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1B,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC,CAAC,CAAE,CAAC;gBAC9D,OAAO,CAAC,IAAI,CAAC;oBACX,SAAS,EAAE,OAAO,CAAC,EAAE;oBACrB,YAAY,EAAE,UAAU;oBACxB,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC;iBACtE,CAAC,CAAC;YACL,CAAC;YAED,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,QAAgB;QAC5C,cAAc;QACd,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,eAAe;QACf,IACE,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB;YAC9C,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAC9C,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,mBAAmB;QACnB,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,OAAO,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,QAAgB,EAAE,QAAgB;QAClD,OAAO,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,eAAe,CACb,KAAa,EACb,OAAsB;QAEtB,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAChE,IAAI,UAAU,GAAG,cAAc,EAAE,CAAC;gBAChC,cAAc,GAAG,UAAU,CAAC;gBAC5B,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,cAAc,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB;YACtD,CAAC,CAAC,EAAE,EAAE,EAAE,MAAO,EAAE,UAAU,EAAE,cAAc,EAAE;YAC7C,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,OAAsB;QAK7B,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,eAAe,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,YAAY,EAAE,OAAO,CAAC,MAAM;YAC5B,eAAe,EAAE,eAAe;YAChC,aAAa,EAAE,OAAO,CAAC,MAAM,GAAG,eAAe;SAChD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deduplicated Memory Module for UAP
|
|
3
|
+
*
|
|
4
|
+
* Implements memory deduplication using content hashing and similarity detection.
|
|
5
|
+
* Prevents redundant memory entries and maintains memory efficiency.
|
|
6
|
+
*/
|
|
7
|
+
export interface MemoryEntry {
|
|
8
|
+
id: string;
|
|
9
|
+
content: string;
|
|
10
|
+
type: string;
|
|
11
|
+
timestamp: string;
|
|
12
|
+
importance?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface DedupConfig {
|
|
15
|
+
similarityThreshold: number;
|
|
16
|
+
minContentLength: number;
|
|
17
|
+
hashAlgorithm: 'jaccard' | 'cosine' | 'exact';
|
|
18
|
+
maxSimilarityWindow: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Deduplicated Memory Manager
|
|
22
|
+
* Prevents duplicate memory entries through content hashing and similarity detection
|
|
23
|
+
*/
|
|
24
|
+
export declare class DeduplicatedMemory {
|
|
25
|
+
private config;
|
|
26
|
+
private memory;
|
|
27
|
+
private contentIndex;
|
|
28
|
+
constructor(config?: Partial<DedupConfig>);
|
|
29
|
+
/**
|
|
30
|
+
* Add a memory entry, checking for duplicates
|
|
31
|
+
*/
|
|
32
|
+
add(entry: MemoryEntry): {
|
|
33
|
+
id: string;
|
|
34
|
+
isNew: boolean;
|
|
35
|
+
duplicateOf?: string;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Get a memory entry by ID
|
|
39
|
+
*/
|
|
40
|
+
get(id: string): MemoryEntry | null;
|
|
41
|
+
/**
|
|
42
|
+
* Update an existing entry
|
|
43
|
+
*/
|
|
44
|
+
update(id: string, content: string): boolean;
|
|
45
|
+
/**
|
|
46
|
+
* Delete a memory entry
|
|
47
|
+
*/
|
|
48
|
+
delete(id: string): boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Get all entries
|
|
51
|
+
*/
|
|
52
|
+
getAll(): MemoryEntry[];
|
|
53
|
+
/**
|
|
54
|
+
* Query entries by content (returns most similar)
|
|
55
|
+
*/
|
|
56
|
+
query(query: string, limit?: number): Array<{
|
|
57
|
+
entry: MemoryEntry;
|
|
58
|
+
similarity: number;
|
|
59
|
+
}>;
|
|
60
|
+
/**
|
|
61
|
+
* Get statistics
|
|
62
|
+
*/
|
|
63
|
+
getStats(): {
|
|
64
|
+
totalEntries: number;
|
|
65
|
+
uniqueContent: number;
|
|
66
|
+
duplicatesPrevented: number;
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Clear all memory
|
|
70
|
+
*/
|
|
71
|
+
clear(): void;
|
|
72
|
+
/**
|
|
73
|
+
* Compute content hash (simple approach)
|
|
74
|
+
*/
|
|
75
|
+
private computeHash;
|
|
76
|
+
/**
|
|
77
|
+
* Compute similarity between two texts
|
|
78
|
+
*/
|
|
79
|
+
private computeSimilarity;
|
|
80
|
+
/**
|
|
81
|
+
* Find similar content in memory
|
|
82
|
+
*/
|
|
83
|
+
private findSimilarContent;
|
|
84
|
+
/**
|
|
85
|
+
* Update content index
|
|
86
|
+
*/
|
|
87
|
+
private updateIndex;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=dedup-memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup-memory.d.ts","sourceRoot":"","sources":["../../src/memory/dedup-memory.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IAC9C,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AASD;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAAuC;IACrD,OAAO,CAAC,YAAY,CAAoC;gBAE5C,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM;IAI7C;;OAEG;IACH,GAAG,CAAC,KAAK,EAAE,WAAW,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE;IAkC7E;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAInC;;OAEG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO;IAgB5C;;OAEG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAQ3B;;OAEG;IACH,MAAM,IAAI,WAAW,EAAE;IAIvB;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAU,GAAG,KAAK,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAc1F;;OAEG;IACH,QAAQ,IAAI;QACV,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,mBAAmB,EAAE,MAAM,CAAC;KAC7B;IAQD;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,OAAO,CAAC,WAAW;IAWnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;OAEG;IACH,OAAO,CAAC,WAAW;CAMpB"}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deduplicated Memory Module for UAP
|
|
3
|
+
*
|
|
4
|
+
* Implements memory deduplication using content hashing and similarity detection.
|
|
5
|
+
* Prevents redundant memory entries and maintains memory efficiency.
|
|
6
|
+
*/
|
|
7
|
+
import { jaccardSimilarity } from '../utils/string-similarity.js';
|
|
8
|
+
const DEFAULT_CONFIG = {
|
|
9
|
+
similarityThreshold: 0.85,
|
|
10
|
+
minContentLength: 20,
|
|
11
|
+
hashAlgorithm: 'jaccard',
|
|
12
|
+
maxSimilarityWindow: 7,
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Deduplicated Memory Manager
|
|
16
|
+
* Prevents duplicate memory entries through content hashing and similarity detection
|
|
17
|
+
*/
|
|
18
|
+
export class DeduplicatedMemory {
|
|
19
|
+
config;
|
|
20
|
+
memory = new Map();
|
|
21
|
+
contentIndex = new Map(); // hash -> [ids]
|
|
22
|
+
constructor(config = {}) {
|
|
23
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Add a memory entry, checking for duplicates
|
|
27
|
+
*/
|
|
28
|
+
add(entry) {
|
|
29
|
+
const hash = this.computeHash(entry.content);
|
|
30
|
+
// Check if exact duplicate exists
|
|
31
|
+
if (this.memory.has(hash)) {
|
|
32
|
+
return {
|
|
33
|
+
id: hash,
|
|
34
|
+
isNew: false,
|
|
35
|
+
duplicateOf: hash,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
// Check for similar content
|
|
39
|
+
const similarId = this.findSimilarContent(entry.content);
|
|
40
|
+
if (similarId) {
|
|
41
|
+
return {
|
|
42
|
+
id: hash,
|
|
43
|
+
isNew: false,
|
|
44
|
+
duplicateOf: similarId,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
// Add new entry
|
|
48
|
+
const fullEntry = {
|
|
49
|
+
...entry,
|
|
50
|
+
id: hash,
|
|
51
|
+
};
|
|
52
|
+
this.memory.set(hash, fullEntry);
|
|
53
|
+
this.updateIndex(hash, entry.content);
|
|
54
|
+
return { id: hash, isNew: true };
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Get a memory entry by ID
|
|
58
|
+
*/
|
|
59
|
+
get(id) {
|
|
60
|
+
return this.memory.get(id) || null;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Update an existing entry
|
|
64
|
+
*/
|
|
65
|
+
update(id, content) {
|
|
66
|
+
if (!this.memory.has(id)) {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
const entry = this.memory.get(id);
|
|
70
|
+
entry.content = content;
|
|
71
|
+
entry.timestamp = new Date().toISOString();
|
|
72
|
+
// Update index
|
|
73
|
+
this.contentIndex.delete(entry.id);
|
|
74
|
+
this.updateIndex(id, content);
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Delete a memory entry
|
|
79
|
+
*/
|
|
80
|
+
delete(id) {
|
|
81
|
+
const deleted = this.memory.delete(id);
|
|
82
|
+
if (deleted) {
|
|
83
|
+
this.contentIndex.delete(id);
|
|
84
|
+
}
|
|
85
|
+
return deleted;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Get all entries
|
|
89
|
+
*/
|
|
90
|
+
getAll() {
|
|
91
|
+
return Array.from(this.memory.values());
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Query entries by content (returns most similar)
|
|
95
|
+
*/
|
|
96
|
+
query(query, limit = 5) {
|
|
97
|
+
const results = [];
|
|
98
|
+
for (const entry of this.memory.values()) {
|
|
99
|
+
const similarity = this.computeSimilarity(query, entry.content);
|
|
100
|
+
if (similarity > 0.5) {
|
|
101
|
+
results.push({ entry, similarity });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
results.sort((a, b) => b.similarity - a.similarity);
|
|
105
|
+
return results.slice(0, limit);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Get statistics
|
|
109
|
+
*/
|
|
110
|
+
getStats() {
|
|
111
|
+
return {
|
|
112
|
+
totalEntries: this.memory.size,
|
|
113
|
+
uniqueContent: this.memory.size,
|
|
114
|
+
duplicatesPrevented: 0, // Would need to track separately
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Clear all memory
|
|
119
|
+
*/
|
|
120
|
+
clear() {
|
|
121
|
+
this.memory.clear();
|
|
122
|
+
this.contentIndex.clear();
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Compute content hash (simple approach)
|
|
126
|
+
*/
|
|
127
|
+
computeHash(content) {
|
|
128
|
+
// Simple hash - in production, use a proper hashing algorithm
|
|
129
|
+
let hash = 0;
|
|
130
|
+
for (let i = 0; i < content.length; i++) {
|
|
131
|
+
const char = content.charCodeAt(i);
|
|
132
|
+
hash = (hash << 5) - hash + char;
|
|
133
|
+
hash = hash & hash; // Convert to 32bit integer
|
|
134
|
+
}
|
|
135
|
+
return `mem-${Math.abs(hash).toString(36)}-${Date.now()}`;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Compute similarity between two texts
|
|
139
|
+
*/
|
|
140
|
+
computeSimilarity(a, b) {
|
|
141
|
+
if (this.config.hashAlgorithm === 'exact') {
|
|
142
|
+
return a === b ? 1 : 0;
|
|
143
|
+
}
|
|
144
|
+
if (a.length < this.config.minContentLength || b.length < this.config.minContentLength) {
|
|
145
|
+
return 0;
|
|
146
|
+
}
|
|
147
|
+
return jaccardSimilarity(a, b);
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Find similar content in memory
|
|
151
|
+
*/
|
|
152
|
+
findSimilarContent(content) {
|
|
153
|
+
for (const [id, entry] of this.memory) {
|
|
154
|
+
if (entry.id === id)
|
|
155
|
+
continue;
|
|
156
|
+
const similarity = this.computeSimilarity(content, entry.content);
|
|
157
|
+
if (similarity >= this.config.similarityThreshold) {
|
|
158
|
+
return id;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Update content index
|
|
165
|
+
*/
|
|
166
|
+
updateIndex(id, content) {
|
|
167
|
+
if (!this.contentIndex.has(id)) {
|
|
168
|
+
this.contentIndex.set(id, []);
|
|
169
|
+
}
|
|
170
|
+
this.contentIndex.get(id).push(content);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=dedup-memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup-memory.js","sourceRoot":"","sources":["../../src/memory/dedup-memory.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAiBlE,MAAM,cAAc,GAAgB;IAClC,mBAAmB,EAAE,IAAI;IACzB,gBAAgB,EAAE,EAAE;IACpB,aAAa,EAAE,SAAS;IACxB,mBAAmB,EAAE,CAAC;CACvB,CAAC;AAEF;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAAc;IACpB,MAAM,GAA6B,IAAI,GAAG,EAAE,CAAC;IAC7C,YAAY,GAA0B,IAAI,GAAG,EAAE,CAAC,CAAC,gBAAgB;IAEzE,YAAY,SAA+B,EAAE;QAC3C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,KAAkB;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE7C,kCAAkC;QAClC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,KAAK,EAAE,KAAK;gBACZ,WAAW,EAAE,IAAI;aAClB,CAAC;QACJ,CAAC;QAED,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,EAAE,EAAE,IAAI;gBACR,KAAK,EAAE,KAAK;gBACZ,WAAW,EAAE,SAAS;aACvB,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,MAAM,SAAS,GAAgB;YAC7B,GAAG,KAAK;YACR,EAAE,EAAE,IAAI;SACT,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAEtC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,EAAU;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU,EAAE,OAAe;QAChC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACnC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;QACxB,KAAK,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,eAAe;QACf,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAE9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU;QACf,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAa,EAAE,QAAgB,CAAC;QACpC,MAAM,OAAO,GAAsD,EAAE,CAAC;QAEtE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAChE,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QACpD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,QAAQ;QAKN,OAAO;YACL,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YAC9B,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YAC/B,mBAAmB,EAAE,CAAC,EAAE,iCAAiC;SAC1D,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,OAAe;QACjC,8DAA8D;QAC9D,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;YACjC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,2BAA2B;QACjD,CAAC;QACD,OAAO,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC5D,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,CAAS,EAAE,CAAS;QAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACvF,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,OAAe;QACxC,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE;gBAAE,SAAS;YAE9B,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAClE,IAAI,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;gBAClD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,EAAU,EAAE,OAAe;QAC7C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;CACF"}
|