@holdyourvoice/hyv 3.2.0 → 3.3.0

package/dist/learning.js

@@ -1,5 +1,5 @@
-import { createHash } from 'node:crypto';
-import { appendFileSync, closeSync, mkdirSync, openSync, readSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { createHash, randomUUID } from 'node:crypto';
+import { closeSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 const MAX_PREFERENCES = 10;
@@ -20,118 +20,185 @@ function canonicalJson(value) {
 export function profileFingerprint(profile) {
     return createHash('sha256').update(canonicalJson(profile)).digest('hex');
 }
-function learningDirectory(options = {}) {
-    return join(options.root ?? process.env.HYV_HOME ?? join(homedir(), '.hyv'), 'learning');
-}
-function eventFile(profile, options = {}) {
-    return join(learningDirectory(options), `${profileFingerprint(profile)}.jsonl`);
-}
-function normalizeInstruction(instruction) {
-    return instruction.replace(/\s+/g, ' ').trim();
-}
+function identity(profile) { return profile.version === '3' ? profile.id : profileFingerprint(profile); }
+function revision(profile) { return profile.version === '3' ? profile.revision : 1; }
+function revisionDigest(profile) { return profile.version === '3' ? profile.revisionDigest : profileFingerprint(profile); }
+function learningDirectory(options = {}) { return join(options.root ?? process.env.HYV_HOME ?? join(homedir(), '.hyv'), 'learning'); }
+function eventFile(profile, options = {}) { return join(learningDirectory(options), `${identity(profile)}.jsonl`); }
+function normalizeInstruction(instruction) { return instruction.replace(/\s+/g, ' ').trim(); }
+function digest(value) { return createHash('sha256').update(canonicalJson(value)).digest('hex'); }
 function isResolvedFinding(value) {
     if (!value || typeof value !== 'object')
         return false;
     const finding = value;
-    return (finding.engine === 'voice_dna' || finding.engine === 'ai_editor')
-        && typeof finding.id === 'string' && finding.id.length > 0
-        && (finding.severity === 'red' || finding.severity === 'yellow')
-        && Number.isInteger(finding.count) && (finding.count ?? 0) > 0;
+    return (finding.engine === 'voice_dna' || finding.engine === 'ai_editor') && typeof finding.id === 'string' && finding.id.length > 0
+        && (finding.severity === 'red' || finding.severity === 'yellow') && Number.isInteger(finding.count) && (finding.count ?? 0) > 0;
 }
-function parseEvent(line) {
+function parseEvent(line, lineNumber = 0) {
     try {
-        const event = JSON.parse(line);
-        if (!event || event.version !== '1' || typeof event.timestamp !== 'string')
+        const parsed = JSON.parse(line);
+        if (parsed.version === '1' && typeof parsed.timestamp === 'string') {
+            const legacyId = digest({ version: '1', timestamp: parsed.timestamp, kind: parsed.kind, lineNumber });
+            const base = {
+                version: '2', eventId: legacyId, mutationId: `legacy:${legacyId}`, timestamp: parsed.timestamp,
+                profileRevision: 1, revisionDigest: 'legacy-v1', authority: 'system', provenance: 'hyv-3.2.0', weight: 1,
+                compatibility: 'same-or-newer', kind: parsed.kind, requestDigest: legacyId,
+            };
+            if (parsed.kind === 'instruction') {
+                const instruction = normalizeInstruction(typeof parsed.instruction === 'string' ? parsed.instruction : '');
+                return instruction && instruction.length <= MAX_INSTRUCTION_CHARACTERS ? { ...base, instruction } : undefined;
+            }
+            if (parsed.kind === 'verified_candidate' && Array.isArray(parsed.resolved) && parsed.resolved.length > 0 && parsed.resolved.every(isResolvedFinding) && typeof parsed.outcome === 'string') {
+                return { ...base, resolved: parsed.resolved, outcome: parsed.outcome };
+            }
             return undefined;
-        if (event.kind === 'instruction' && typeof event.instruction === 'string') {
-            const instruction = normalizeInstruction(event.instruction);
-            if (instruction.length > 0 && instruction.length <= MAX_INSTRUCTION_CHARACTERS)
-                return { version: '1', timestamp: event.timestamp, kind: 'instruction', instruction };
         }
-        const resolved = event.resolved;
-        if (event.kind === 'verified_candidate' && Array.isArray(resolved) && resolved.length > 0 && resolved.every(isResolvedFinding) && typeof event.outcome === 'string') {
-            return { version: '1', timestamp: event.timestamp, kind: 'verified_candidate', resolved, outcome: event.outcome };
+        const value = parsed;
+        if (value.version !== '2' || typeof value.eventId !== 'string' || typeof value.mutationId !== 'string' || typeof value.requestDigest !== 'string' || typeof value.timestamp !== 'string'
+            || !Number.isSafeInteger(value.profileRevision) || (value.profileRevision ?? 0) < 1 || typeof value.revisionDigest !== 'string'
+            || !['founder', 'team', 'system'].includes(value.authority ?? '') || typeof value.provenance !== 'string'
+            || typeof value.weight !== 'number' || !Number.isFinite(value.weight) || value.weight <= 0
+            || !['same-or-newer', 'exact'].includes(value.compatibility ?? '')
+            || !['verified_candidate', 'instruction', 'ratification', 'supersession', 'migration'].includes(value.kind ?? ''))
+            return undefined;
+        if (value.kind === 'instruction') {
+            const instruction = normalizeInstruction(value.instruction ?? '');
+            if (!instruction || instruction.length > MAX_INSTRUCTION_CHARACTERS)
+                return undefined;
+            value.instruction = instruction;
         }
+        if (value.kind === 'verified_candidate' && (!Array.isArray(value.resolved) || !value.resolved.length || !value.resolved.every(isResolvedFinding) || typeof value.outcome !== 'string'))
+            return undefined;
+        if ((value.kind === 'ratification' || value.kind === 'supersession') && typeof value.targetEventId !== 'string')
+            return undefined;
+        return value;
     }
     catch {
         return undefined;
     }
-    return undefined;
 }
-function readRecentText(file) {
+function readEventsFromFile(file) {
+    let contents;
     try {
-        const size = statSync(file).size;
-        const length = Math.min(size, MAX_STORAGE_BYTES);
-        const descriptor = openSync(file, 'r');
-        try {
-            const buffer = Buffer.alloc(length);
-            readSync(descriptor, buffer, 0, length, Math.max(0, size - length));
-            return buffer.toString('utf8');
-        }
-        finally {
-            closeSync(descriptor);
-        }
+        contents = readFileSync(file, 'utf8');
     }
-    catch {
-        return '';
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return [];
+        throw error;
     }
+    return contents.split('\n').flatMap((line, index) => {
+        if (!line.trim())
+            return [];
+        const event = parseEvent(line, index + 1);
+        if (!event)
+            throw new Error(`Learning store is corrupt at line ${index + 1}.`);
+        return [event];
+    });
 }
-function readEvents(profile, options = {}) {
-    const file = eventFile(profile, options);
-    return readRecentText(file).split('\n').flatMap((line) => {
-        const event = parseEvent(line);
-        return event ? [event] : [];
-    }).slice(-MAX_EVENTS);
+function readEvents(profile, options = {}) { return readEventsFromFile(eventFile(profile, options)); }
+function serialize(events) { return events.length ? `${events.map((event) => JSON.stringify(event)).join('\n')}\n` : ''; }
+function learningState(events, profile) {
+    return {
+        superseded: new Set(events.filter((event) => event.kind === 'supersession').map((event) => event.targetEventId)),
+        ratified: new Set(events.filter((event) => event.kind === 'ratification' && event.profileRevision <= revision(profile)).map((event) => event.targetEventId)),
+    };
 }
-function serialize(events) {
-    return `${events.map((event) => JSON.stringify(event)).join('\n')}\n`;
+function isCompatible(event, profile) {
+    return event.profileRevision <= revision(profile)
+        && (event.compatibility === 'same-or-newer' || event.profileRevision === revision(profile));
 }
-function withCompactionLock(file, operation) {
+function withLock(file, operation) {
     const lock = `${file}.lock`;
     for (let attempt = 0; attempt < 100; attempt += 1) {
+        let descriptor;
         try {
-            const descriptor = openSync(lock, 'wx', 0o600);
-            try {
-                return operation();
-            }
-            finally {
-                closeSync(descriptor);
-                unlinkSync(lock);
-            }
+            descriptor = openSync(lock, 'wx', 0o600);
         }
         catch (error) {
             if (error.code !== 'EEXIST')
-                return undefined;
+                throw error;
             Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+            continue;
+        }
+        try {
+            return operation();
+        }
+        finally {
+            closeSync(descriptor);
+            unlinkSync(lock);
         }
     }
     return undefined;
 }
-function appendEvent(profile, event, options = {}) {
+function withLocks(files, operation) {
+    const unique = [...new Set(files)].sort();
+    const acquire = (index) => index === unique.length ? operation() : withLock(unique[index], () => acquire(index + 1));
+    return acquire(0);
+}
+const authorityRank = { founder: 3, team: 2, system: 1 };
+function compact(events) {
+    const content = events.filter((event) => event.kind === 'instruction' || event.kind === 'verified_candidate')
+        .sort((a, b) => authorityRank[b.authority] - authorityRank[a.authority] || b.weight - a.weight || b.timestamp.localeCompare(a.timestamp) || a.eventId.localeCompare(b.eventId));
+    const selected = new Set(content.slice(0, MAX_EVENTS).map((event) => event.eventId));
+    const controls = events.filter((event) => event.kind === 'migration' || ((event.kind === 'ratification' || event.kind === 'supersession') && selected.has(event.targetEventId ?? '')));
+    while (selected.size + controls.length > MAX_EVENTS)
+        selected.delete(content[[...selected].length - 1]?.eventId ?? '');
+    const retained = events.filter((event) => selected.has(event.eventId) || (event.kind === 'migration') || ((event.kind === 'ratification' || event.kind === 'supersession') && selected.has(event.targetEventId ?? '')))
+        .sort((a, b) => a.timestamp.localeCompare(b.timestamp) || a.eventId.localeCompare(b.eventId));
+    while (retained.length && Buffer.byteLength(serialize(retained)) > MAX_STORAGE_BYTES) {
+        const removable = retained.findIndex((event) => event.kind === 'instruction' || event.kind === 'verified_candidate');
+        retained.splice(removable < 0 ? 0 : removable, 1);
+    }
+    return retained;
+}
+function writeEventsAtomically(file, events) {
+    const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
+    try {
+        writeFileSync(temporary, serialize(compact(events)), { encoding: 'utf8', mode: 0o600 });
+        renameSync(temporary, file);
+    }
+    catch (error) {
+        try {
+            unlinkSync(temporary);
+        }
+        catch { /* best-effort cleanup */ }
+        throw error;
+    }
+}
+function receipt(profile, event, status) {
+    const publicMetadata = { mutationId: event.mutationId, eventId: event.eventId, profileId: identity(profile), profileRevision: revision(profile), status, timestamp: event.timestamp };
+    return { version: '1', ...publicMetadata, digest: digest(publicMetadata) };
+}
+function eventBase(profile, kind, options) {
+    const mutationId = options.mutationId ?? randomUUID();
+    const base = {
+        version: '2', eventId: digest({ profile: identity(profile), mutationId }), mutationId, timestamp: new Date().toISOString(),
+        profileRevision: revision(profile), revisionDigest: revisionDigest(profile), authority: options.authority ?? 'team',
+        provenance: options.provenance ?? 'local', weight: options.weight ?? 1, compatibility: options.compatibility ?? 'same-or-newer', kind,
+    };
+    return { ...base, requestDigest: digest({ ...base, eventId: undefined, timestamp: undefined, requestDigest: undefined }) };
+}
+function mutate(profile, event, options, validate) {
     try {
         const directory = learningDirectory(options);
         mkdirSync(directory, { recursive: true, mode: 0o700 });
         const file = eventFile(profile, options);
-        const line = `${JSON.stringify(event)}\n`;
-        const events = readEvents(profile, options);
-        const needsCompaction = events.length >= MAX_EVENTS || (statSync(file, { throwIfNoEntry: false })?.size ?? 0) + Buffer.byteLength(line) > MAX_STORAGE_BYTES;
-        if (needsCompaction) {
-            const compacted = withCompactionLock(file, () => {
-                const retained = [...readEvents(profile, options), event].slice(-MAX_EVENTS);
-                while (Buffer.byteLength(serialize(retained)) > MAX_STORAGE_BYTES)
-                    retained.shift();
-                writeFileSync(file, serialize(retained), { encoding: 'utf8', mode: 0o600 });
-            });
-            if (compacted === undefined)
-                return false;
-        }
-        else {
-            appendFileSync(file, line, { encoding: 'utf8', mode: 0o600 });
-        }
-        return true;
+        const result = withLock(file, () => {
+            const events = readEventsFromFile(file);
+            const existing = events.find((item) => item.mutationId === event.mutationId);
+            if (existing)
+                return receipt(profile, existing, existing.requestDigest === event.requestDigest ? 'already_recorded' : 'conflict');
+            const invalid = validate?.(events);
+            if (invalid)
+                return receipt(profile, event, invalid);
+            writeEventsAtomically(file, [...events, event]);
+            return receipt(profile, event, 'recorded');
+        });
+        return result ?? receipt(profile, event, 'lock_timeout');
     }
-    catch {
-        return false;
+    catch (error) {
+        return receipt(profile, event, error.message.includes('corrupt') ? 'corrupt' : 'write_failed');
     }
 }
 function countFindings(findings) {
@@ -151,35 +218,100 @@ export function recordVerifiedCandidate(profile, verification, candidate, option
         return 'nothing_to_learn';
     const original = countFindings([...verification.original.voiceDna.findings, ...verification.original.aiEditor.findings]);
     const candidateFindings = countFindings([...verification.candidate.voiceDna.findings, ...verification.candidate.aiEditor.findings]);
-    const resolved = [...original].flatMap(([key, entry]) => {
-        const count = entry.count - (candidateFindings.get(key)?.count ?? 0);
-        return count > 0 ? [{ engine: entry.finding.engine, id: entry.finding.id, severity: entry.finding.severity, count }] : [];
-    });
+    const resolved = [...original].flatMap(([key, entry]) => { const count = entry.count - (candidateFindings.get(key)?.count ?? 0); return count > 0 ? [{ engine: entry.finding.engine, id: entry.finding.id, severity: entry.finding.severity, count }] : []; });
     if (!resolved.length)
         return 'nothing_to_learn';
-    const bounded = resolved.slice(0, MAX_RESOLVED_FINDINGS);
-    const outcome = createHash('sha256').update(`${profileFingerprint(profile)}\0${candidate}`).digest('hex');
+    const outcome = createHash('sha256').update(`${identity(profile)}\0${candidate}`).digest('hex');
     if (readEvents(profile, options).some((event) => event.kind === 'verified_candidate' && event.outcome === outcome))
         return 'nothing_to_learn';
-    return appendEvent(profile, { version: '1', timestamp: new Date().toISOString(), kind: 'verified_candidate', resolved: bounded, outcome }, options) ? 'recorded' : 'write_failed';
+    const event = { ...eventBase(profile, 'verified_candidate', { ...options, mutationId: options.mutationId ?? outcome }), resolved: resolved.slice(0, MAX_RESOLVED_FINDINGS), outcome };
+    event.requestDigest = digest({ ...event, eventId: undefined, timestamp: undefined, requestDigest: undefined });
+    const result = mutate(profile, event, options);
+    return result.status === 'recorded' ? 'recorded' : result.status === 'already_recorded' ? 'nothing_to_learn' : 'write_failed';
 }
-export function addLearningInstruction(profile, instruction, options = {}) {
-    const trimmed = normalizeInstruction(instruction);
-    if (!trimmed)
+export function recordLearningInstruction(profile, instruction, options = {}) {
+    const normalized = normalizeInstruction(instruction);
+    if (!normalized)
         throw new Error('Learning instructions cannot be empty.');
-    if (trimmed.length > MAX_INSTRUCTION_CHARACTERS)
+    if (normalized.length > MAX_INSTRUCTION_CHARACTERS)
         throw new Error(`Learning instructions must be ${MAX_INSTRUCTION_CHARACTERS} characters or fewer.`);
-    return appendEvent(profile, { version: '1', timestamp: new Date().toISOString(), kind: 'instruction', instruction: trimmed }, options);
+    const event = { ...eventBase(profile, 'instruction', options), instruction: normalized };
+    event.requestDigest = digest({ ...event, eventId: undefined, timestamp: undefined, requestDigest: undefined });
+    return mutate(profile, event, options);
+}
+export function addLearningInstruction(profile, instruction, options = {}) { return recordLearningInstruction(profile, instruction, options).status === 'recorded'; }
+export function ratifyLearningEvent(profile, targetEventId, options = {}) {
+    const event = { ...eventBase(profile, 'ratification', options), targetEventId };
+    event.requestDigest = digest({ ...event, eventId: undefined, timestamp: undefined, requestDigest: undefined });
+    return mutate(profile, event, options, (events) => { const target = events.find((item) => item.eventId === targetEventId); return !target ? 'not_found' : authorityRank[event.authority] < authorityRank[target.authority] ? 'unauthorized' : undefined; });
+}
+export function supersedeLearningEvent(profile, targetEventId, options = {}) {
+    const event = { ...eventBase(profile, 'supersession', options), targetEventId };
+    event.requestDigest = digest({ ...event, eventId: undefined, timestamp: undefined, requestDigest: undefined });
+    return mutate(profile, event, options, (events) => { const target = events.find((item) => item.eventId === targetEventId); return !target ? 'not_found' : authorityRank[event.authority] < authorityRank[target.authority] ? 'unauthorized' : undefined; });
+}
+export function migrateLearningV2ToV3(source, target, options = {}) {
+    const sourceFingerprint = profileFingerprint(source);
+    const migrationId = options.mutationId ?? `migration:${digest({ source: sourceFingerprint, target: target.id, revision: target.revision })}`;
+    const migrationOptions = { ...options, mutationId: migrationId };
+    const marker = { ...eventBase(target, 'migration', migrationOptions), sourceProfile: sourceFingerprint };
+    try {
+        const directory = learningDirectory(options);
+        mkdirSync(directory, { recursive: true, mode: 0o700 });
+        const file = eventFile(target, options);
+        const sourceFile = eventFile(source, options);
+        const result = withLocks([sourceFile, file], () => {
+            const targetEvents = readEventsFromFile(file);
+            const existing = targetEvents.find((event) => event.mutationId === marker.mutationId);
+            if (existing)
+                return receipt(target, existing, 'already_recorded');
+            const migrated = readEventsFromFile(sourceFile).filter((event) => event.kind === 'instruction' || event.kind === 'verified_candidate').map((event) => ({
+                ...event, eventId: digest({ target: target.id, source: event.eventId }), mutationId: `migration:${marker.mutationId}:${event.mutationId}`,
+                profileRevision: target.revision, revisionDigest: target.revisionDigest, provenance: `migration:${sourceFingerprint}`,
+            }));
+            const intended = [...targetEvents, ...migrated, marker];
+            const compacted = compact(intended);
+            if (migrated.some((event) => !compacted.some((item) => item.eventId === event.eventId)) || !compacted.some((item) => item.eventId === marker.eventId))
+                return receipt(target, marker, 'capacity_exceeded');
+            writeEventsAtomically(file, intended);
+            return receipt(target, marker, 'recorded');
+        });
+        return result ?? receipt(target, marker, 'lock_timeout');
+    }
+    catch (error) {
+        return receipt(target, marker, error.message.includes('corrupt') ? 'corrupt' : 'write_failed');
+    }
+}
+export function inspectLearning(profile, options = {}) {
+    const events = readEvents(profile, options);
+    const { superseded, ratified } = learningState(events, profile);
+    return events.slice(-MAX_EVENTS).map((event) => {
+        const control = event.kind === 'ratification' || event.kind === 'supersession' || event.kind === 'migration';
+        const status = control ? 'control' : superseded.has(event.eventId) ? 'superseded' : isCompatible(event, profile) || ratified.has(event.eventId) ? 'active' : 'incompatible';
+        return {
+            version: '1', eventId: event.eventId, mutationId: event.mutationId, eventType: event.kind, timestamp: event.timestamp,
+            profileRevision: event.profileRevision, authority: event.authority, weight: event.weight,
+            compatibility: event.compatibility, status, ...(event.targetEventId ? { targetEventId: event.targetEventId } : {}),
+        };
+    });
 }
 export function composeLearning(profile, options = {}) {
+    const events = readEvents(profile, options);
+    const { superseded, ratified } = learningState(events, profile);
     const preferences = new Map();
-    for (const event of readEvents(profile, options)) {
-        const texts = event.kind === 'instruction' && event.instruction
-            ? [{ text: event.instruction, count: 1 }]
-            : (event.resolved ?? []).map((finding) => ({ text: `Previously verified repair: ${finding.engine}/${finding.id}.`, count: finding.count }));
+    for (const event of events) {
+        if (event.kind !== 'instruction' && event.kind !== 'verified_candidate')
+            continue;
+        if (superseded.has(event.eventId))
+            continue;
+        if (!isCompatible(event, profile) && !ratified.has(event.eventId))
+            continue;
+        const texts = event.kind === 'instruction' && event.instruction ? [{ text: event.instruction, count: event.weight }]
+            : (event.resolved ?? []).map((finding) => ({ text: `Previously verified repair: ${finding.engine}/${finding.id}.`, count: finding.count * event.weight }));
         for (const { text, count } of texts) {
-            const current = preferences.get(text) ?? { count: 0, lastSeen: event.timestamp };
+            const current = preferences.get(text) ?? { count: 0, lastSeen: event.timestamp, authority: authorityRank[event.authority] };
             current.count += count;
+            current.authority = Math.max(current.authority, authorityRank[event.authority]);
             if (event.timestamp > current.lastSeen)
                 current.lastSeen = event.timestamp;
             preferences.set(text, current);
@@ -187,9 +319,7 @@ export function composeLearning(profile, options = {}) {
     }
     const result = [];
     let characters = 0;
-    for (const [text, value] of [...preferences.entries()]
-        .sort(([firstText, first], [secondText, second]) => second.count - first.count || second.lastSeen.localeCompare(first.lastSeen) || firstText.localeCompare(secondText))
-        .slice(0, MAX_PREFERENCES)) {
+    for (const [text, value] of [...preferences.entries()].sort(([aText, a], [bText, b]) => b.authority - a.authority || b.count - a.count || b.lastSeen.localeCompare(a.lastSeen) || aText.localeCompare(bText)).slice(0, MAX_PREFERENCES)) {
         if (characters + text.length > MAX_COMPOSED_CHARACTERS)
             break;
         result.push({ text, count: value.count });
@@ -200,8 +330,18 @@ export function composeLearning(profile, options = {}) {
 export function clearLearning(profile, options = {}) {
     const file = eventFile(profile, options);
     try {
-        unlinkSync(file);
-        return true;
+        const result = withLock(file, () => { try {
+            unlinkSync(file);
+            return true;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return false;
+            throw error;
+        } });
+        if (result === undefined)
+            throw new Error('Learning store lock timeout.');
+        return result;
     }
     catch (error) {
         if (error.code === 'ENOENT')