@dmsdc-ai/aigentry-brain 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/aigentry-brain-mcp.mjs +17 -0
- package/bin/aigentry-brain-setup.mjs +17 -0
- package/bin/aigentry-brain.mjs +17 -0
- package/dist/audit/AuditTrail.d.ts +15 -0
- package/dist/audit/AuditTrail.js +43 -0
- package/dist/audit/AuditTrail.js.map +1 -0
- package/dist/backend/GitBackend.d.ts +27 -0
- package/dist/backend/GitBackend.js +201 -0
- package/dist/backend/GitBackend.js.map +1 -0
- package/dist/backend/SyncBackend.d.ts +13 -0
- package/dist/backend/SyncBackend.js +2 -0
- package/dist/backend/SyncBackend.js.map +1 -0
- package/dist/cli/AutoUpdater.d.ts +7 -0
- package/dist/cli/AutoUpdater.js +275 -0
- package/dist/cli/AutoUpdater.js.map +1 -0
- package/dist/cli/ConfigStore.d.ts +17 -0
- package/dist/cli/ConfigStore.js +50 -0
- package/dist/cli/ConfigStore.js.map +1 -0
- package/dist/cli/GhIntegration.d.ts +55 -0
- package/dist/cli/GhIntegration.js +192 -0
- package/dist/cli/GhIntegration.js.map +1 -0
- package/dist/cli/Prerequisites.d.ts +48 -0
- package/dist/cli/Prerequisites.js +182 -0
- package/dist/cli/Prerequisites.js.map +1 -0
- package/dist/cli/SetupWizard.d.ts +26 -0
- package/dist/cli/SetupWizard.js +407 -0
- package/dist/cli/SetupWizard.js.map +1 -0
- package/dist/cli/braincli.d.ts +2 -0
- package/dist/cli/braincli.js +260 -0
- package/dist/cli/braincli.js.map +1 -0
- package/dist/cli/setup.d.ts +25 -0
- package/dist/cli/setup.js +238 -0
- package/dist/cli/setup.js.map +1 -0
- package/dist/context/ConfidenceGuard.d.ts +11 -0
- package/dist/context/ConfidenceGuard.js +32 -0
- package/dist/context/ConfidenceGuard.js.map +1 -0
- package/dist/context/ContextBudgetPolicy.d.ts +21 -0
- package/dist/context/ContextBudgetPolicy.js +136 -0
- package/dist/context/ContextBudgetPolicy.js.map +1 -0
- package/dist/context/ContextPacker.d.ts +30 -0
- package/dist/context/ContextPacker.js +219 -0
- package/dist/context/ContextPacker.js.map +1 -0
- package/dist/context/ContextRestoreService.d.ts +39 -0
- package/dist/context/ContextRestoreService.js +134 -0
- package/dist/context/ContextRestoreService.js.map +1 -0
- package/dist/contract/BrainContract.d.ts +93 -0
- package/dist/contract/BrainContract.js +245 -0
- package/dist/contract/BrainContract.js.map +1 -0
- package/dist/core/ContentDedup.d.ts +43 -0
- package/dist/core/ContentDedup.js +145 -0
- package/dist/core/ContentDedup.js.map +1 -0
- package/dist/core/EntityGraph.d.ts +66 -0
- package/dist/core/EntityGraph.js +196 -0
- package/dist/core/EntityGraph.js.map +1 -0
- package/dist/core/EntrySchema.d.ts +46 -0
- package/dist/core/EntrySchema.js +4 -0
- package/dist/core/EntrySchema.js.map +1 -0
- package/dist/core/GraphStore.d.ts +16 -0
- package/dist/core/GraphStore.js +29 -0
- package/dist/core/GraphStore.js.map +1 -0
- package/dist/core/MemoryDecay.d.ts +49 -0
- package/dist/core/MemoryDecay.js +113 -0
- package/dist/core/MemoryDecay.js.map +1 -0
- package/dist/core/ProfileFormat.d.ts +38 -0
- package/dist/core/ProfileFormat.js +254 -0
- package/dist/core/ProfileFormat.js.map +1 -0
- package/dist/core/ProfileManager.d.ts +80 -0
- package/dist/core/ProfileManager.js +269 -0
- package/dist/core/ProfileManager.js.map +1 -0
- package/dist/core/SchemaMigration.d.ts +12 -0
- package/dist/core/SchemaMigration.js +34 -0
- package/dist/core/SchemaMigration.js.map +1 -0
- package/dist/crypto/AgeCrypto.d.ts +26 -0
- package/dist/crypto/AgeCrypto.js +101 -0
- package/dist/crypto/AgeCrypto.js.map +1 -0
- package/dist/index.d.ts +38 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/BrainMcpServer.d.ts +13 -0
- package/dist/mcp/BrainMcpServer.js +223 -0
- package/dist/mcp/BrainMcpServer.js.map +1 -0
- package/dist/mcp/cli.d.ts +2 -0
- package/dist/mcp/cli.js +107 -0
- package/dist/mcp/cli.js.map +1 -0
- package/dist/metrics/ResumeMetrics.d.ts +20 -0
- package/dist/metrics/ResumeMetrics.js +46 -0
- package/dist/metrics/ResumeMetrics.js.map +1 -0
- package/dist/peers/PeerStore.d.ts +17 -0
- package/dist/peers/PeerStore.js +65 -0
- package/dist/peers/PeerStore.js.map +1 -0
- package/dist/policy/PolicyEngine.d.ts +16 -0
- package/dist/policy/PolicyEngine.js +70 -0
- package/dist/policy/PolicyEngine.js.map +1 -0
- package/dist/policy/PolicyEnvelope.d.ts +7 -0
- package/dist/policy/PolicyEnvelope.js +2 -0
- package/dist/policy/PolicyEnvelope.js.map +1 -0
- package/dist/policy/RetentionParser.d.ts +6 -0
- package/dist/policy/RetentionParser.js +48 -0
- package/dist/policy/RetentionParser.js.map +1 -0
- package/dist/shared/AtomicWrite.d.ts +1 -0
- package/dist/shared/AtomicWrite.js +11 -0
- package/dist/shared/AtomicWrite.js.map +1 -0
- package/dist/shared/Clock.d.ts +11 -0
- package/dist/shared/Clock.js +17 -0
- package/dist/shared/Clock.js.map +1 -0
- package/dist/shared/Config.d.ts +15 -0
- package/dist/shared/Config.js +67 -0
- package/dist/shared/Config.js.map +1 -0
- package/dist/shared/Encryption.d.ts +19 -0
- package/dist/shared/Encryption.js +47 -0
- package/dist/shared/Encryption.js.map +1 -0
- package/dist/shared/Logger.d.ts +12 -0
- package/dist/shared/Logger.js +40 -0
- package/dist/shared/Logger.js.map +1 -0
- package/dist/shared/Mutex.d.ts +7 -0
- package/dist/shared/Mutex.js +34 -0
- package/dist/shared/Mutex.js.map +1 -0
- package/dist/shared/SetupErrors.d.ts +31 -0
- package/dist/shared/SetupErrors.js +155 -0
- package/dist/shared/SetupErrors.js.map +1 -0
- package/dist/status/SyncStatusStore.d.ts +17 -0
- package/dist/status/SyncStatusStore.js +48 -0
- package/dist/status/SyncStatusStore.js.map +1 -0
- package/dist/sync/AutoCommitter.d.ts +17 -0
- package/dist/sync/AutoCommitter.js +68 -0
- package/dist/sync/AutoCommitter.js.map +1 -0
- package/dist/sync/ConflictResolver.d.ts +7 -0
- package/dist/sync/ConflictResolver.js +56 -0
- package/dist/sync/ConflictResolver.js.map +1 -0
- package/dist/sync/PushPullManager.d.ts +23 -0
- package/dist/sync/PushPullManager.js +155 -0
- package/dist/sync/PushPullManager.js.map +1 -0
- package/dist/sync/SyncEngine.d.ts +43 -0
- package/dist/sync/SyncEngine.js +145 -0
- package/dist/sync/SyncEngine.js.map +1 -0
- package/package.json +65 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { Entry } from '../core/EntrySchema.js';
|
|
2
|
+
import type { ProfileManager } from '../core/ProfileManager.js';
|
|
3
|
+
import type { SyncStatusStore, SyncStatusFile } from '../status/SyncStatusStore.js';
|
|
4
|
+
import type { ContextResumeOptions, ContextResumeResult } from '../context/ContextRestoreService.js';
|
|
5
|
+
import type { Clock } from '../shared/Clock.js';
|
|
6
|
+
import type { AuditTrail } from '../audit/AuditTrail.js';
|
|
7
|
+
import { type PolicyConfig } from '../policy/PolicyEngine.js';
|
|
8
|
+
/**
|
|
9
|
+
* Minimal interface required by BrainContract for sync operations.
|
|
10
|
+
* The real SyncEngine must be wrapped to match this shape (see cli.ts adapter).
|
|
11
|
+
*/
|
|
12
|
+
export interface SyncEngine {
|
|
13
|
+
pull(): Promise<{
|
|
14
|
+
pulled: boolean;
|
|
15
|
+
status: string;
|
|
16
|
+
}>;
|
|
17
|
+
push(): Promise<{
|
|
18
|
+
pushed: boolean;
|
|
19
|
+
status: string;
|
|
20
|
+
}>;
|
|
21
|
+
}
|
|
22
|
+
export interface QueryFilter {
|
|
23
|
+
category?: string;
|
|
24
|
+
source?: string;
|
|
25
|
+
since?: string;
|
|
26
|
+
contentQuery?: string;
|
|
27
|
+
tags?: string[];
|
|
28
|
+
}
|
|
29
|
+
export interface BrainContractConfig {
|
|
30
|
+
profileManager: ProfileManager;
|
|
31
|
+
statusStore: SyncStatusStore;
|
|
32
|
+
clock?: Clock;
|
|
33
|
+
syncEngine?: SyncEngine | null;
|
|
34
|
+
auditTrail?: AuditTrail;
|
|
35
|
+
policyConfig?: PolicyConfig;
|
|
36
|
+
}
|
|
37
|
+
export declare const MAX_ACTIVE_MEDIA_PROVIDERS = 3;
|
|
38
|
+
/**
|
|
39
|
+
* Thin Contract v0.1 — Public API for aigentry-brain.
|
|
40
|
+
*
|
|
41
|
+
* 8 operations: append, query, syncStatus, erase, contextResume, health, appendBatch, eraseBatch
|
|
42
|
+
*/
|
|
43
|
+
export declare class BrainContract {
|
|
44
|
+
private profileManager;
|
|
45
|
+
private statusStore;
|
|
46
|
+
private restoreService;
|
|
47
|
+
private syncEngine;
|
|
48
|
+
private auditTrail;
|
|
49
|
+
private policyEngine;
|
|
50
|
+
constructor(config: BrainContractConfig);
|
|
51
|
+
/** Append a new entry (or update existing by id). */
|
|
52
|
+
append(partial: Partial<Entry> & {
|
|
53
|
+
category: string;
|
|
54
|
+
content: string;
|
|
55
|
+
source: string;
|
|
56
|
+
}): Promise<Entry>;
|
|
57
|
+
/** Query entries with optional filters. Returns active (non-deleted) entries. */
|
|
58
|
+
query(filter?: QueryFilter): Promise<Entry[]>;
|
|
59
|
+
/** Get current sync status. */
|
|
60
|
+
syncStatus(): Promise<SyncStatusFile>;
|
|
61
|
+
/** Soft-delete an entry by id. Returns true if deleted, false if not found. */
|
|
62
|
+
erase(id: string): Promise<boolean>;
|
|
63
|
+
/** Restore context from synced profile for AI consumption. */
|
|
64
|
+
contextResume(options: ContextResumeOptions): Promise<ContextResumeResult>;
|
|
65
|
+
/** Get system health report. */
|
|
66
|
+
health(): Promise<{
|
|
67
|
+
profileAccessible: boolean;
|
|
68
|
+
entryCount: number;
|
|
69
|
+
syncStatus: string;
|
|
70
|
+
uptimeMs: number;
|
|
71
|
+
}>;
|
|
72
|
+
/** Append multiple entries in a single operation (single read + N mutations + single write). */
|
|
73
|
+
appendBatch(items: Array<Partial<Entry> & {
|
|
74
|
+
category: string;
|
|
75
|
+
content: string;
|
|
76
|
+
source: string;
|
|
77
|
+
}>): Promise<Entry[]>;
|
|
78
|
+
/** Pull changes from remote repository. */
|
|
79
|
+
syncPull(): Promise<{
|
|
80
|
+
pulled: boolean;
|
|
81
|
+
status: string;
|
|
82
|
+
}>;
|
|
83
|
+
/** Push local changes to remote repository. */
|
|
84
|
+
syncPush(): Promise<{
|
|
85
|
+
pushed: boolean;
|
|
86
|
+
status: string;
|
|
87
|
+
}>;
|
|
88
|
+
/** Soft-delete multiple entries by id. Returns count of successfully deleted. */
|
|
89
|
+
eraseBatch(ids: string[]): Promise<{
|
|
90
|
+
deleted: number;
|
|
91
|
+
total: number;
|
|
92
|
+
}>;
|
|
93
|
+
}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { ContextRestoreService } from '../context/ContextRestoreService.js';
|
|
2
|
+
import { PolicyEngine } from '../policy/PolicyEngine.js';
|
|
3
|
+
const MESSENGER_PROVIDERS = new Set([
|
|
4
|
+
'telegram',
|
|
5
|
+
'whatsapp',
|
|
6
|
+
'messenger',
|
|
7
|
+
'discord',
|
|
8
|
+
'slack',
|
|
9
|
+
'line',
|
|
10
|
+
'kakao',
|
|
11
|
+
]);
|
|
12
|
+
const MAX_MEDIA_POINTERS_PER_ENTRY = 20;
|
|
13
|
+
export const MAX_ACTIVE_MEDIA_PROVIDERS = 3;
|
|
14
|
+
function normalizeProvider(provider) {
|
|
15
|
+
return provider.trim().toLowerCase();
|
|
16
|
+
}
|
|
17
|
+
function inferMediaChannel(provider) {
|
|
18
|
+
return MESSENGER_PROVIDERS.has(normalizeProvider(provider)) ? 'messenger' : 'platform';
|
|
19
|
+
}
|
|
20
|
+
function normalizeMediaPointers(pointers) {
|
|
21
|
+
if (pointers === undefined)
|
|
22
|
+
return undefined;
|
|
23
|
+
if (pointers.length === 0)
|
|
24
|
+
return [];
|
|
25
|
+
if (pointers.length > MAX_MEDIA_POINTERS_PER_ENTRY) {
|
|
26
|
+
throw new Error(`Too many media pointers: max ${MAX_MEDIA_POINTERS_PER_ENTRY} per entry`);
|
|
27
|
+
}
|
|
28
|
+
const normalized = pointers.map((pointer, idx) => {
|
|
29
|
+
const provider = normalizeProvider(pointer.provider ?? '');
|
|
30
|
+
const url = (pointer.url ?? '').trim();
|
|
31
|
+
if (!provider)
|
|
32
|
+
throw new Error(`media_pointers[${idx}] provider is required`);
|
|
33
|
+
if (!url)
|
|
34
|
+
throw new Error(`media_pointers[${idx}] url is required`);
|
|
35
|
+
const channel = pointer.channel ?? inferMediaChannel(provider);
|
|
36
|
+
const purpose = pointer.purpose ?? (channel === 'messenger' ? 'share' : 'storage');
|
|
37
|
+
if (channel === 'messenger' && purpose === 'storage') {
|
|
38
|
+
throw new Error('Messenger channels are share-only and cannot be used as canonical storage');
|
|
39
|
+
}
|
|
40
|
+
if (pointer.duration_sec !== undefined && pointer.duration_sec < 0) {
|
|
41
|
+
throw new Error(`media_pointers[${idx}] duration_sec must be >= 0`);
|
|
42
|
+
}
|
|
43
|
+
return {
|
|
44
|
+
...pointer,
|
|
45
|
+
provider,
|
|
46
|
+
url,
|
|
47
|
+
channel,
|
|
48
|
+
purpose,
|
|
49
|
+
...(pointer.title !== undefined ? { title: pointer.title.trim() } : {}),
|
|
50
|
+
};
|
|
51
|
+
});
|
|
52
|
+
const activeStorageProviders = new Set(normalized
|
|
53
|
+
.filter(p => p.channel === 'platform' && p.purpose === 'storage')
|
|
54
|
+
.map(p => p.provider));
|
|
55
|
+
if (activeStorageProviders.size > MAX_ACTIVE_MEDIA_PROVIDERS) {
|
|
56
|
+
throw new Error(`Too many active media providers: max ${MAX_ACTIVE_MEDIA_PROVIDERS}`);
|
|
57
|
+
}
|
|
58
|
+
return normalized;
|
|
59
|
+
}
|
|
60
|
+
function applyMediaPolicy(partial) {
|
|
61
|
+
if (partial.media_pointers === undefined)
|
|
62
|
+
return partial;
|
|
63
|
+
return {
|
|
64
|
+
...partial,
|
|
65
|
+
media_pointers: normalizeMediaPointers(partial.media_pointers),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Thin Contract v0.1 — Public API for aigentry-brain.
|
|
70
|
+
*
|
|
71
|
+
* 8 operations: append, query, syncStatus, erase, contextResume, health, appendBatch, eraseBatch
|
|
72
|
+
*/
|
|
73
|
+
export class BrainContract {
|
|
74
|
+
profileManager;
|
|
75
|
+
statusStore;
|
|
76
|
+
restoreService;
|
|
77
|
+
syncEngine;
|
|
78
|
+
auditTrail;
|
|
79
|
+
policyEngine;
|
|
80
|
+
constructor(config) {
|
|
81
|
+
this.profileManager = config.profileManager;
|
|
82
|
+
this.statusStore = config.statusStore;
|
|
83
|
+
this.syncEngine = config.syncEngine ?? null;
|
|
84
|
+
this.auditTrail = config.auditTrail ?? null;
|
|
85
|
+
this.policyEngine = config.policyConfig
|
|
86
|
+
? new PolicyEngine(config.policyConfig)
|
|
87
|
+
: null;
|
|
88
|
+
this.restoreService = new ContextRestoreService(config.profileManager, config.clock);
|
|
89
|
+
}
|
|
90
|
+
/** Append a new entry (or update existing by id). */
|
|
91
|
+
async append(partial) {
|
|
92
|
+
let processed = applyMediaPolicy(partial);
|
|
93
|
+
// CMP: validate policy on write
|
|
94
|
+
if (this.policyEngine) {
|
|
95
|
+
this.policyEngine.validateWrite(processed.policy);
|
|
96
|
+
if (processed.policy || processed.policy === undefined) {
|
|
97
|
+
processed = { ...processed, policy: this.policyEngine.applyDefaults(processed.policy) };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const entry = await this.profileManager.upsert(processed);
|
|
101
|
+
if (this.auditTrail) {
|
|
102
|
+
this.auditTrail.log({
|
|
103
|
+
ts: new Date().toISOString(),
|
|
104
|
+
action: 'append',
|
|
105
|
+
entry_id: entry.id,
|
|
106
|
+
device_id: this.profileManager.getDeviceId(),
|
|
107
|
+
}).catch(() => { });
|
|
108
|
+
}
|
|
109
|
+
return entry;
|
|
110
|
+
}
|
|
111
|
+
/** Query entries with optional filters. Returns active (non-deleted) entries. */
|
|
112
|
+
async query(filter) {
|
|
113
|
+
let entries = await this.profileManager.getAll();
|
|
114
|
+
// CMP: filter by policy (retention_ttl expiration + scope/projectId)
|
|
115
|
+
if (this.policyEngine) {
|
|
116
|
+
entries = this.policyEngine.filterRead(entries);
|
|
117
|
+
}
|
|
118
|
+
if (filter?.category) {
|
|
119
|
+
entries = entries.filter(e => e.category === filter.category);
|
|
120
|
+
}
|
|
121
|
+
if (filter?.source) {
|
|
122
|
+
entries = entries.filter(e => e.source === filter.source);
|
|
123
|
+
}
|
|
124
|
+
if (filter?.since) {
|
|
125
|
+
const sinceMs = new Date(filter.since).getTime();
|
|
126
|
+
if (!Number.isNaN(sinceMs)) {
|
|
127
|
+
entries = entries.filter(e => new Date(e.updated).getTime() >= sinceMs);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (filter?.contentQuery) {
|
|
131
|
+
const q = filter.contentQuery.toLowerCase();
|
|
132
|
+
entries = entries.filter(e => e.content.toLowerCase().includes(q));
|
|
133
|
+
}
|
|
134
|
+
if (filter?.tags && filter.tags.length > 0) {
|
|
135
|
+
entries = entries.filter(e => {
|
|
136
|
+
const entryTags = e.tags ?? [];
|
|
137
|
+
return filter.tags.every(t => entryTags.includes(t));
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
return entries;
|
|
141
|
+
}
|
|
142
|
+
/** Get current sync status. */
|
|
143
|
+
async syncStatus() {
|
|
144
|
+
return this.statusStore.get();
|
|
145
|
+
}
|
|
146
|
+
/** Soft-delete an entry by id. Returns true if deleted, false if not found. */
|
|
147
|
+
async erase(id) {
|
|
148
|
+
const deleted = await this.profileManager.softDelete(id);
|
|
149
|
+
if (deleted && this.auditTrail) {
|
|
150
|
+
this.auditTrail.log({
|
|
151
|
+
ts: new Date().toISOString(),
|
|
152
|
+
action: 'erase',
|
|
153
|
+
entry_id: id,
|
|
154
|
+
device_id: this.profileManager.getDeviceId(),
|
|
155
|
+
}).catch(() => { });
|
|
156
|
+
}
|
|
157
|
+
return deleted;
|
|
158
|
+
}
|
|
159
|
+
/** Restore context from synced profile for AI consumption. */
|
|
160
|
+
async contextResume(options) {
|
|
161
|
+
return this.restoreService.restore(options);
|
|
162
|
+
}
|
|
163
|
+
/** Get system health report. */
|
|
164
|
+
async health() {
|
|
165
|
+
let profileAccessible = true;
|
|
166
|
+
let entryCount = 0;
|
|
167
|
+
try {
|
|
168
|
+
const entries = await this.profileManager.getAll();
|
|
169
|
+
entryCount = entries.length;
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
profileAccessible = false;
|
|
173
|
+
}
|
|
174
|
+
const status = await this.statusStore.get();
|
|
175
|
+
return {
|
|
176
|
+
profileAccessible,
|
|
177
|
+
entryCount,
|
|
178
|
+
syncStatus: status.status,
|
|
179
|
+
uptimeMs: Math.floor(process.uptime() * 1000),
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
/** Append multiple entries in a single operation (single read + N mutations + single write). */
|
|
183
|
+
async appendBatch(items) {
|
|
184
|
+
const processed = items.map(item => {
|
|
185
|
+
const normalizedItem = applyMediaPolicy(item);
|
|
186
|
+
if (this.policyEngine) {
|
|
187
|
+
this.policyEngine.validateWrite(normalizedItem.policy);
|
|
188
|
+
return { ...normalizedItem, policy: this.policyEngine.applyDefaults(normalizedItem.policy) };
|
|
189
|
+
}
|
|
190
|
+
return normalizedItem;
|
|
191
|
+
});
|
|
192
|
+
const results = await this.profileManager.upsertBatch(processed);
|
|
193
|
+
if (this.auditTrail) {
|
|
194
|
+
for (const entry of results) {
|
|
195
|
+
this.auditTrail.log({
|
|
196
|
+
ts: new Date().toISOString(),
|
|
197
|
+
action: 'append',
|
|
198
|
+
entry_id: entry.id,
|
|
199
|
+
device_id: this.profileManager.getDeviceId(),
|
|
200
|
+
}).catch(() => { });
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return results;
|
|
204
|
+
}
|
|
205
|
+
/** Pull changes from remote repository. */
|
|
206
|
+
async syncPull() {
|
|
207
|
+
if (!this.syncEngine) {
|
|
208
|
+
return { pulled: false, status: 'no_sync_engine' };
|
|
209
|
+
}
|
|
210
|
+
const result = await this.syncEngine.pull();
|
|
211
|
+
if (result.pulled && this.auditTrail) {
|
|
212
|
+
this.auditTrail.log({
|
|
213
|
+
ts: new Date().toISOString(),
|
|
214
|
+
action: 'sync.pull',
|
|
215
|
+
device_id: this.profileManager.getDeviceId(),
|
|
216
|
+
}).catch(() => { });
|
|
217
|
+
}
|
|
218
|
+
return result;
|
|
219
|
+
}
|
|
220
|
+
/** Push local changes to remote repository. */
|
|
221
|
+
async syncPush() {
|
|
222
|
+
if (!this.syncEngine) {
|
|
223
|
+
return { pushed: false, status: 'no_sync_engine' };
|
|
224
|
+
}
|
|
225
|
+
const result = await this.syncEngine.push();
|
|
226
|
+
if (result.pushed && this.auditTrail) {
|
|
227
|
+
this.auditTrail.log({
|
|
228
|
+
ts: new Date().toISOString(),
|
|
229
|
+
action: 'sync.push',
|
|
230
|
+
device_id: this.profileManager.getDeviceId(),
|
|
231
|
+
}).catch(() => { });
|
|
232
|
+
}
|
|
233
|
+
return result;
|
|
234
|
+
}
|
|
235
|
+
/** Soft-delete multiple entries by id. Returns count of successfully deleted. */
|
|
236
|
+
async eraseBatch(ids) {
|
|
237
|
+
let deleted = 0;
|
|
238
|
+
for (const id of ids) {
|
|
239
|
+
if (await this.erase(id))
|
|
240
|
+
deleted++;
|
|
241
|
+
}
|
|
242
|
+
return { deleted, total: ids.length };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
//# sourceMappingURL=BrainContract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BrainContract.js","sourceRoot":"","sources":["../../src/contract/BrainContract.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAI5E,OAAO,EAAE,YAAY,EAAqB,MAAM,2BAA2B,CAAC;AA4B5E,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,UAAU;IACV,UAAU;IACV,WAAW;IACX,SAAS;IACT,OAAO;IACP,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,4BAA4B,GAAG,EAAE,CAAC;AACxC,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAE5C,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,mBAAmB,CAAC,GAAG,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;AACzF,CAAC;AAED,SAAS,sBAAsB,CAAC,QAAoC;IAClE,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,IAAI,QAAQ,CAAC,MAAM,GAAG,4BAA4B,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,gCAAgC,4BAA4B,YAAY,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;QAC/C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,wBAAwB,CAAC,CAAC;QAC9E,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,mBAAmB,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAiB,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAiB,OAAO,CAAC,OAAO,IAAI,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjG,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAC;QACtE,CAAC;QAED,OAAO;YACL,GAAG,OAAO;YACV,QAAQ;YACR,GAAG;YACH,OAAO;YACP,OAAO;YACP,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CACpC,UAAU;SACP,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC;SAChE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CACxB,CAAC;IAEF,IAAI,sBAAsB,CAAC,IAAI,GAAG,0BAA0B,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,wCAAwC,0BAA0B,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,gBAAgB,CAA2B,OAAU;IAC5D,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IACzD,OAAO;QACL,GAAG,OAAO;QACV,cAAc,EAAE,sBAAsB,CAAC,OAAO,CAAC,cAAc,CAAC;KAC/D,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,aAAa;IAChB,cAAc,CAAiB;IAC/B,WAAW,CAAkB;IAC7B,cAAc,CAAwB;IACtC,UAAU,CAAoB;IAC9B,UAAU,CAAoB;IAC9B,YAAY,CAAsB;IAE1C,YAAY,MAA2B;QACrC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;YACrC,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC;YACvC,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,CAAC,cAAc,GAAG,IAAI,qBAAqB,CAC7C,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,KAAK,CACb,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,MAAM,CACV,OAA+E;QAE/E,IAAI,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAE1C,gCAAgC;QAChC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvD,SAAS,GAAG,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1F,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAClB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE,KAAK,CAAC,EAAE;gBAClB,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE;aAC7C,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyB,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,KAAK,CAAC,MAAoB;QAC9B,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;QAEjD,qEAAqE;QACrE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;YACrB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;YACnB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QACD,IAAI,MAAM,EAAE,YAAY,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;YAC5C,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAC3B,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC/B,OAAO,MAAM,CAAC,IAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,KAAK,CAAC,EAAU;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzD,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAClB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,MAAM,EAAE,OAAO;gBACf,QAAQ,EAAE,EAAE;gBACZ,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE;aAC7C,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyB,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,8DAA8D;IAC9D,KAAK,CAAC,aAAa,CAAC,OAA6B;QAC/C,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,gCAAgC;IAChC,KAAK,CAAC,MAAM;QACV,IAAI,iBAAiB,GAAG,IAAI,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;YACnD,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,OAAO;YACL,iBAAiB;YACjB,UAAU;YACV,UAAU,EAAE,MAAM,CAAC,MAAM;YACzB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;SAC9C,CAAC;IACJ,CAAC;IAED,gGAAgG;IAChG,KAAK,CAAC,WAAW,CACf,KAAoF;QAEpF,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACjC,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBACvD,OAAO,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/F,CAAC;YACD,OAAO,cAAc,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAClB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,MAAM,EAAE,QAAQ;oBAChB,QAAQ,EAAE,KAAK,CAAC,EAAE;oBAClB,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE;iBAC7C,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyB,CAAC,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,2CAA2C;IAC3C,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAClB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE;aAC7C,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyB,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,+CAA+C;IAC/C,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAClB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE;aAC7C,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyB,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,UAAU,CAAC,GAAa;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,OAAO,EAAE,CAAC;QACtC,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;CACF"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Entry } from './EntrySchema.js';
|
|
2
|
+
import type { DecayConfig } from './MemoryDecay.js';
|
|
3
|
+
export interface DedupConfig {
|
|
4
|
+
similarityThreshold: number;
|
|
5
|
+
minTokens: number;
|
|
6
|
+
}
|
|
7
|
+
export declare const DEFAULT_DEDUP_CONFIG: DedupConfig;
|
|
8
|
+
export interface DedupResult {
|
|
9
|
+
entries: Entry[];
|
|
10
|
+
merged: number;
|
|
11
|
+
pairs: Array<{
|
|
12
|
+
kept: string;
|
|
13
|
+
removed: string;
|
|
14
|
+
similarity: number;
|
|
15
|
+
}>;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Tokenize text: lowercase, split on whitespace/punctuation, filter short tokens.
|
|
19
|
+
*/
|
|
20
|
+
export declare function tokenize(text: string): string[];
|
|
21
|
+
/**
|
|
22
|
+
* Build TF (term frequency) vector for a document.
|
|
23
|
+
* Values are normalized by document length.
|
|
24
|
+
*/
|
|
25
|
+
export declare function buildTF(tokens: string[]): Map<string, number>;
|
|
26
|
+
/**
|
|
27
|
+
* Build IDF (inverse document frequency) from multiple TF vectors.
|
|
28
|
+
*/
|
|
29
|
+
export declare function buildIDF(documents: Map<string, number>[]): Map<string, number>;
|
|
30
|
+
/**
|
|
31
|
+
* Compute TF-IDF vector from a TF vector and an IDF map.
|
|
32
|
+
*/
|
|
33
|
+
export declare function tfidfVector(tf: Map<string, number>, idf: Map<string, number>): Map<string, number>;
|
|
34
|
+
/**
|
|
35
|
+
* Cosine similarity between two sparse vectors.
|
|
36
|
+
*/
|
|
37
|
+
export declare function cosineSimilarity(a: Map<string, number>, b: Map<string, number>): number;
|
|
38
|
+
/**
|
|
39
|
+
* Deduplicate entries within each category using TF-IDF cosine similarity.
|
|
40
|
+
* When duplicates found: keep the one with higher effectiveConfidence,
|
|
41
|
+
* soft-delete the other.
|
|
42
|
+
*/
|
|
43
|
+
export declare function deduplicateEntries(entries: Entry[], now: Date, decayConfig: DecayConfig, dedupConfig?: DedupConfig): DedupResult;
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { effectiveConfidence } from './MemoryDecay.js';
|
|
2
|
+
export const DEFAULT_DEDUP_CONFIG = {
|
|
3
|
+
similarityThreshold: 0.85,
|
|
4
|
+
minTokens: 5,
|
|
5
|
+
};
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
// Pure functions
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
/**
|
|
10
|
+
* Tokenize text: lowercase, split on whitespace/punctuation, filter short tokens.
|
|
11
|
+
*/
|
|
12
|
+
export function tokenize(text) {
|
|
13
|
+
return text
|
|
14
|
+
.toLowerCase()
|
|
15
|
+
.split(/[\s\p{P}]+/u)
|
|
16
|
+
.filter(t => t.length > 1);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Build TF (term frequency) vector for a document.
|
|
20
|
+
* Values are normalized by document length.
|
|
21
|
+
*/
|
|
22
|
+
export function buildTF(tokens) {
|
|
23
|
+
const tf = new Map();
|
|
24
|
+
for (const token of tokens) {
|
|
25
|
+
tf.set(token, (tf.get(token) || 0) + 1);
|
|
26
|
+
}
|
|
27
|
+
const len = tokens.length;
|
|
28
|
+
for (const [key, val] of tf) {
|
|
29
|
+
tf.set(key, val / len);
|
|
30
|
+
}
|
|
31
|
+
return tf;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Build IDF (inverse document frequency) from multiple TF vectors.
|
|
35
|
+
*/
|
|
36
|
+
export function buildIDF(documents) {
|
|
37
|
+
const docCount = documents.length;
|
|
38
|
+
const idf = new Map();
|
|
39
|
+
const docFreq = new Map();
|
|
40
|
+
for (const doc of documents) {
|
|
41
|
+
for (const term of doc.keys()) {
|
|
42
|
+
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
for (const [term, freq] of docFreq) {
|
|
46
|
+
idf.set(term, Math.log(1 + docCount / freq));
|
|
47
|
+
}
|
|
48
|
+
return idf;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Compute TF-IDF vector from a TF vector and an IDF map.
|
|
52
|
+
*/
|
|
53
|
+
export function tfidfVector(tf, idf) {
|
|
54
|
+
const vec = new Map();
|
|
55
|
+
for (const [term, tfVal] of tf) {
|
|
56
|
+
const idfVal = idf.get(term) || 0;
|
|
57
|
+
vec.set(term, tfVal * idfVal);
|
|
58
|
+
}
|
|
59
|
+
return vec;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Cosine similarity between two sparse vectors.
|
|
63
|
+
*/
|
|
64
|
+
export function cosineSimilarity(a, b) {
|
|
65
|
+
let dot = 0;
|
|
66
|
+
let normA = 0;
|
|
67
|
+
let normB = 0;
|
|
68
|
+
for (const [key, val] of a) {
|
|
69
|
+
normA += val * val;
|
|
70
|
+
const bVal = b.get(key);
|
|
71
|
+
if (bVal !== undefined) {
|
|
72
|
+
dot += val * bVal;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
for (const [, val] of b) {
|
|
76
|
+
normB += val * val;
|
|
77
|
+
}
|
|
78
|
+
if (normA === 0 || normB === 0)
|
|
79
|
+
return 0;
|
|
80
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Deduplicate entries within each category using TF-IDF cosine similarity.
|
|
84
|
+
* When duplicates found: keep the one with higher effectiveConfidence,
|
|
85
|
+
* soft-delete the other.
|
|
86
|
+
*/
|
|
87
|
+
export function deduplicateEntries(entries, now, decayConfig, dedupConfig = DEFAULT_DEDUP_CONFIG) {
|
|
88
|
+
const pairs = [];
|
|
89
|
+
const toDelete = new Set();
|
|
90
|
+
// Group active (non-deleted, non-pinned) entries by category
|
|
91
|
+
const byCategory = new Map();
|
|
92
|
+
for (const entry of entries) {
|
|
93
|
+
if (entry.deleted || entry.pinned)
|
|
94
|
+
continue;
|
|
95
|
+
const cat = entry.category;
|
|
96
|
+
if (!byCategory.has(cat))
|
|
97
|
+
byCategory.set(cat, []);
|
|
98
|
+
byCategory.get(cat).push(entry);
|
|
99
|
+
}
|
|
100
|
+
// Process each category independently
|
|
101
|
+
for (const [, catEntries] of byCategory) {
|
|
102
|
+
if (catEntries.length < 2)
|
|
103
|
+
continue;
|
|
104
|
+
// Tokenize all entries, filter those below minTokens
|
|
105
|
+
const tokenized = catEntries
|
|
106
|
+
.map(e => ({ entry: e, tokens: tokenize(e.content) }))
|
|
107
|
+
.filter(t => t.tokens.length >= dedupConfig.minTokens);
|
|
108
|
+
if (tokenized.length < 2)
|
|
109
|
+
continue;
|
|
110
|
+
// Build TF vectors
|
|
111
|
+
const tfVectors = tokenized.map(t => buildTF(t.tokens));
|
|
112
|
+
// Build IDF across all documents in this category
|
|
113
|
+
const idf = buildIDF(tfVectors);
|
|
114
|
+
// Build TF-IDF vectors
|
|
115
|
+
const tfidfVectors = tfVectors.map(tf => tfidfVector(tf, idf));
|
|
116
|
+
// Compare all pairs within this category
|
|
117
|
+
for (let i = 0; i < tokenized.length; i++) {
|
|
118
|
+
if (toDelete.has(tokenized[i].entry.id))
|
|
119
|
+
continue;
|
|
120
|
+
for (let j = i + 1; j < tokenized.length; j++) {
|
|
121
|
+
if (toDelete.has(tokenized[j].entry.id))
|
|
122
|
+
continue;
|
|
123
|
+
const sim = cosineSimilarity(tfidfVectors[i], tfidfVectors[j]);
|
|
124
|
+
if (sim >= dedupConfig.similarityThreshold) {
|
|
125
|
+
const scoreI = effectiveConfidence(tokenized[i].entry, now, decayConfig);
|
|
126
|
+
const scoreJ = effectiveConfidence(tokenized[j].entry, now, decayConfig);
|
|
127
|
+
const [kept, removed] = scoreI >= scoreJ
|
|
128
|
+
? [tokenized[i].entry, tokenized[j].entry]
|
|
129
|
+
: [tokenized[j].entry, tokenized[i].entry];
|
|
130
|
+
toDelete.add(removed.id);
|
|
131
|
+
pairs.push({ kept: kept.id, removed: removed.id, similarity: sim });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
// Apply soft-deletes
|
|
137
|
+
const result = entries.map(entry => {
|
|
138
|
+
if (toDelete.has(entry.id)) {
|
|
139
|
+
return { ...entry, deleted: true, deleted_at: now.toISOString() };
|
|
140
|
+
}
|
|
141
|
+
return entry;
|
|
142
|
+
});
|
|
143
|
+
return { entries: result, merged: toDelete.size, pairs };
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=ContentDedup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ContentDedup.js","sourceRoot":"","sources":["../../src/core/ContentDedup.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAWvD,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAC/C,mBAAmB,EAAE,IAAI;IACzB,SAAS,EAAE,CAAC;CACb,CAAC;AAYF,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,aAAa,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,MAAgB;IACtC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAC1B,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QAC5B,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,SAAgC;IACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,EAAuB,EACvB,GAAwB;IAExB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,CAAsB,EACtB,CAAsB;IAEtB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,KAAK,IAAI,GAAG,GAAG,GAAG,CAAC;QACnB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,IAAI,GAAG,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,KAAK,IAAI,GAAG,GAAG,GAAG,CAAC;IACrB,CAAC;IAED,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACzC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAgB,EAChB,GAAS,EACT,WAAwB,EACxB,cAA2B,oBAAoB;IAE/C,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,6DAA6D;IAC7D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM;YAAE,SAAS;QAC5C,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAClD,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,sCAAsC;IACtC,KAAK,MAAM,CAAC,EAAE,UAAU,CAAC,IAAI,UAAU,EAAE,CAAC;QACxC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAEpC,qDAAqD;QACrD,MAAM,SAAS,GAAG,UAAU;aACzB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;aACrD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC;QAEzD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAEnC,mBAAmB;QACnB,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAExD,kDAAkD;QAClD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;QAEhC,uBAAuB;QACvB,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QAE/D,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,SAAS;YAElD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;oBAAE,SAAS;gBAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE/D,IAAI,GAAG,IAAI,WAAW,CAAC,mBAAmB,EAAE,CAAC;oBAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;oBACzE,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;oBAEzE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,MAAM;wBACtC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;wBAC1C,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAE7C,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBACzB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACjC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;QACpE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { Entry } from './EntrySchema.js';
|
|
2
|
+
export interface EntityNode {
|
|
3
|
+
name: string;
|
|
4
|
+
type: EntityType;
|
|
5
|
+
entryIds: string[];
|
|
6
|
+
}
|
|
7
|
+
export type EntityType = 'person' | 'project' | 'technology' | 'path' | 'url' | 'concept';
|
|
8
|
+
export interface EntityEdge {
|
|
9
|
+
source: string;
|
|
10
|
+
target: string;
|
|
11
|
+
entryId: string;
|
|
12
|
+
}
|
|
13
|
+
export interface EntityGraph {
|
|
14
|
+
nodes: Record<string, EntityNode>;
|
|
15
|
+
edges: EntityEdge[];
|
|
16
|
+
metadata: {
|
|
17
|
+
builtAt: string;
|
|
18
|
+
entryCount: number;
|
|
19
|
+
nodeCount: number;
|
|
20
|
+
edgeCount: number;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export interface EntityScore {
|
|
24
|
+
entryId: string;
|
|
25
|
+
hubScore: number;
|
|
26
|
+
connectedness: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Extract entities from text using pattern matching.
|
|
30
|
+
* Patterns:
|
|
31
|
+
* - URLs → url
|
|
32
|
+
* - File paths (/, .ts, .js, etc.) → path
|
|
33
|
+
* - @mentions → person
|
|
34
|
+
* - Kebab-case multi-part names → project
|
|
35
|
+
* - Known tech keywords → technology
|
|
36
|
+
* - Capitalized multi-word phrases → concept
|
|
37
|
+
*
|
|
38
|
+
* Deterministic: same input always produces same output.
|
|
39
|
+
*/
|
|
40
|
+
export declare function extractEntities(text: string): Array<{
|
|
41
|
+
name: string;
|
|
42
|
+
type: EntityType;
|
|
43
|
+
}>;
|
|
44
|
+
/**
|
|
45
|
+
* Build an entity graph from entries.
|
|
46
|
+
* Only considers active (non-deleted) entries.
|
|
47
|
+
*/
|
|
48
|
+
export declare function buildEntityGraph(entries: Entry[]): EntityGraph;
|
|
49
|
+
/**
|
|
50
|
+
* Score entries by their entity connectedness.
|
|
51
|
+
* Hub entries (connected to many other entries via shared entities) get higher scores.
|
|
52
|
+
*/
|
|
53
|
+
export declare function scoreEntries(entries: Entry[], graph: EntityGraph): EntityScore[];
|
|
54
|
+
/**
|
|
55
|
+
* Get confidence boost based on entity connectedness.
|
|
56
|
+
* Entries connected to 3+ others get a boost.
|
|
57
|
+
*/
|
|
58
|
+
export declare function connectednessBoost(score: EntityScore): number;
|
|
59
|
+
/**
|
|
60
|
+
* Serialize graph to JSON (for graph.json file, git-diffable).
|
|
61
|
+
*/
|
|
62
|
+
export declare function serializeGraph(graph: EntityGraph): string;
|
|
63
|
+
/**
|
|
64
|
+
* Deserialize graph from JSON string.
|
|
65
|
+
*/
|
|
66
|
+
export declare function deserializeGraph(json: string): EntityGraph;
|