@tenphi/akno-core 0.4.0 → 0.6.0

package/dist/ops/retain.js

@@ -1,6 +1,7 @@
 import fsp from 'node:fs/promises';
 import path from 'node:path';
 import { RetainInput, } from '@tenphi/akno-protocol';
+import { ModelClient } from "../models/client.js";
 import { folderCatalog } from "../kb/folders.js";
 import { parseFrontmatter, serializeYamlString } from "../kb/frontmatter.js";
 import { parsePage } from "../kb/page.js";
@@ -10,8 +11,11 @@ import { restoreFile, writeFileAtomic } from "../write/atomic.js";
 import { detectConflict } from "../write/conflict.js";
 import { fileEntry } from "../write/journal.js";
 import { managedMemoryBlock, managedMemoryFingerprint, markerFromProvidedCandidate, parseManagedMemoryMarker, renderManagedMemoryMarker, renderManagedMemoryPayload, sameManagedMemorySemantics, } from "../write/managed-memory.js";
-import { appendManagedBlock, managedSourceReference } from "../write/placement.js";
+import { appendManagedBlock, managedSourceReference, placeManagedItems } from "../write/placement.js";
+import { modelCallReceipt, runRetain, } from "../write/retain.js";
+import { resolveRememberFallback } from "../write/remember-fallback.js";
 import { reconcileRetainManagedSources } from "../write/retain-supports.js";
+import { routeAutomaticCandidate } from "./remember.js";
 import { normalizeSlug } from "./write.js";
 export async function retain(ctx, rawInput) {
     const input = RetainInput.parse(rawInput);
@@ -19,7 +23,15 @@ export async function retain(ctx, rawInput) {
     for (const source of input.sources) {
         try {
             results.push('input' in source
-                ? await retainProvidedExact(ctx, source, input.dry_run ?? false)
+                ? source.retention.mode === 'extract'
+                    ? await retainExtracted(ctx, source, input.dry_run ?? false)
+                    : await retainCandidates(ctx, source, source.retention.candidates, {
+                        dryRun: input.dry_run ?? false,
+                        selection: 'provided',
+                        placement: source.retention.placement,
+                        initialResults: [],
+                        modelUsage: { extraction: null, verification: null, placement: [] },
+                    })
                 : await retractSource(ctx, source, input.dry_run ?? false));
         }
         catch (error) {
@@ -27,43 +39,121 @@ export async function retain(ctx, rawInput) {
                 source_id: source.source_id,
                 revision: source.revision,
                 outcome: 'held',
+                status: 'degraded',
+                degraded: ['retain_apply_failed'],
+                reason_code: 'apply_failed',
                 candidates: [],
                 note: `source apply failed: ${error instanceof Error ? error.message : String(error)}`,
             });
         }
     }
-    const held = results.filter((result) => result.outcome === 'held' || result.outcome === 'revision_conflict').length;
+    const incomplete = results.some((result) => result.outcome === 'held' ||
+        result.outcome === 'revision_conflict' ||
+        result.candidates.some((candidate) => candidate.outcome === 'held' || candidate.outcome === 'not_found'));
     const effective = results.filter((result) => result.outcome === 'ok').length;
+    const degraded = [...new Set(results.flatMap((result) => result.degraded ?? []))];
+    const allEmpty = results.length > 0 && results.every((result) => result.status === 'empty');
     return {
-        status: 'ok',
-        outcome: held > 0 && effective > 0 ? 'partial' : effective > 0 ? 'ok' : 'noop',
+        status: degraded.length > 0 ? 'degraded' : allEmpty ? 'empty' : 'ok',
+        ...(degraded.length > 0 ? { degraded } : {}),
+        outcome: incomplete && effective > 0 ? 'partial' : effective > 0 ? 'ok' : 'noop',
         sources: results,
     };
 }
-async function retainProvidedExact(ctx, source, dryRun) {
+async function retainExtracted(ctx, source, dryRun) {
+    if (source.retention.mode !== 'extract')
+        throw new Error('retain extract received a provided source');
     const sourceHash = sha256(JSON.stringify(source.input));
     const requestHash = sha256(JSON.stringify(source));
     const replay = replayResult(ctx, source.source_id, source.revision, requestHash, sourceHash);
     if (replay)
         return replay;
     const groupIssue = sourceGroupIssue(ctx, source.source_id, source.source_group);
-    if (groupIssue) {
+    if (groupIssue)
         return conflictResult(source.source_id, source.revision, groupIssue);
+    const curator = retentionModel(ctx);
+    const text = 'text' in source.input ? source.input.text : source.input.items.map((item) => item.text).join('\n');
+    const extracted = await runRetain(text, curator, {
+        ...(source.mentioned_at ? { mentionedAt: source.mentioned_at } : {}),
+        ...(source.timezone ? { timezone: source.timezone } : {}),
+        ...(source.retention.mission ? { mission: source.retention.mission } : {}),
+        ...('items' in source.input ? { sourceItems: source.input.items } : {}),
+        folders: folderCatalog(ctx.config, ctx.store),
+        sourceId: source.source_id,
+        revision: source.revision,
+    });
+    if (extracted.sourceHold) {
+        return retainCandidates(ctx, source, [], {
+            dryRun,
+            selection: 'extracted',
+            placement: 'automatic',
+            initialResults: [],
+            modelUsage: { ...extracted.modelUsage, placement: [] },
+            sourceHold: extracted.sourceHold,
+        });
+    }
+    const initialResults = extracted.held.map(heldCandidateResult);
+    if (extracted.error) {
+        return {
+            source_id: source.source_id,
+            revision: source.revision,
+            outcome: 'held',
+            status: 'degraded',
+            degraded: [extracted.degradedReason ?? 'derive_failed'],
+            candidates: initialResults,
+            model_usage: { ...extracted.modelUsage, placement: [] },
+            note: `automatic retention could not complete: ${extracted.error}`,
+        };
     }
+    return retainCandidates(ctx, source, extracted.candidates, {
+        dryRun,
+        selection: 'extracted',
+        placement: 'automatic',
+        initialResults,
+        modelUsage: { ...extracted.modelUsage, placement: [] },
+    });
+}
+async function retainCandidates(ctx, source, suppliedCandidates, options) {
+    const sourceHash = sha256(JSON.stringify(source.input));
+    const requestHash = sha256(JSON.stringify(source));
+    const replay = replayResult(ctx, source.source_id, source.revision, requestHash, sourceHash);
+    if (replay)
+        return replay;
+    const groupIssue = sourceGroupIssue(ctx, source.source_id, source.source_group);
+    if (groupIssue)
+        return conflictResult(source.source_id, source.revision, groupIssue);
+    const resolved = options.placement === 'automatic'
+        ? await resolveAutomaticCandidates(ctx, suppliedCandidates)
+        : {
+            candidates: suppliedCandidates.filter(hasDestination),
+            held: suppliedCandidates
+                .filter((candidate) => !candidate.destination)
+                .map((candidate) => ({
+                candidate_id: candidate.candidate_id,
+                outcome: 'held',
+                reason_code: 'validation_failed',
+                reason: 'provided exact candidates require a destination',
+            })),
+        };
+    const candidates = resolved.candidates;
     const receiptFingerprint = managedMemoryFingerprint(`receipt:${source.source_id}:${source.revision}:${requestHash}`);
     const proofGroup = managedMemoryFingerprint(`proof:${source.source_group ?? source.source_id}`);
     const stages = new Map();
     const pendingSupports = [];
-    const candidateResults = [];
-    const candidateIds = new Set(source.retention.candidates.map((candidate) => candidate.candidate_id));
-    const candidateOrder = new Map(source.retention.candidates.map((candidate, index) => [candidate.candidate_id, index]));
+    const candidateResults = [...options.initialResults, ...resolved.held];
+    const candidateIds = new Set(suppliedCandidates.map((candidate) => candidate.candidate_id));
+    const candidateOrder = new Map([
+        ...options.initialResults.map((result) => result.candidate_id),
+        ...suppliedCandidates.map((candidate) => candidate.candidate_id),
+    ].map((candidateId, index) => [candidateId, index]));
     const candidateMemoryIds = new Map();
-    const ordered = dependencyOrder(source.retention.candidates);
-    for (const candidate of source.retention.candidates) {
+    const ordered = dependencyOrder(candidates);
+    for (const candidate of candidates) {
         if (ordered.blocked.has(candidate.candidate_id)) {
             candidateResults.push({
                 candidate_id: candidate.candidate_id,
                 outcome: 'held',
+                reason_code: 'validation_failed',
                 reason: 'candidate relation cycle or blocked dependency',
             });
         }
@@ -71,7 +161,12 @@ async function retainProvidedExact(ctx, source, dryRun) {
     for (const candidate of ordered.candidates) {
         const issue = candidateIssue(ctx, source, candidate, candidateIds);
         if (issue) {
-            candidateResults.push({ candidate_id: candidate.candidate_id, outcome: 'held', reason: issue });
+            candidateResults.push({
+                candidate_id: candidate.candidate_id,
+                outcome: 'held',
+                reason_code: candidateIssueReasonCode(issue),
+                reason: issue,
+            });
             continue;
         }
         const missingTarget = (candidate.relations ?? []).find((relation) => 'candidate_id' in relation.target && !candidateMemoryIds.has(relation.target.candidate_id));
@@ -79,6 +174,7 @@ async function retainProvidedExact(ctx, source, dryRun) {
             candidateResults.push({
                 candidate_id: candidate.candidate_id,
                 outcome: 'held',
+                reason_code: 'validation_failed',
                 reason: `relation target candidate ${missingTarget.target.candidate_id} was not retained`,
             });
             continue;
@@ -90,6 +186,7 @@ async function retainProvidedExact(ctx, source, dryRun) {
                 candidate_id: candidate.candidate_id,
                 outcome: 'held',
                 slug,
+                reason_code: stage.issue.includes('unreadable') ? 'source_unavailable' : 'no_writable_destination',
                 reason: stage.issue,
             });
             continue;
@@ -104,7 +201,7 @@ async function retainProvidedExact(ctx, source, dryRun) {
             receipt: receiptFingerprint,
             candidate: candidateFingerprint,
             proofGroup,
-            selection: 'provided',
+            selection: options.selection,
         };
         const proposedId = newPrefixedId('mem');
         const marker = markerFromProvidedCandidate(proposedId, candidate, support, candidateMemoryIds);
@@ -124,6 +221,7 @@ async function retainProvidedExact(ctx, source, dryRun) {
                     outcome: 'held',
                     memory_id: memoryId,
                     slug,
+                    reason_code: 'support_limit',
                     reason: 'support_limit',
                 });
                 continue;
@@ -136,13 +234,15 @@ async function retainProvidedExact(ctx, source, dryRun) {
         }
         else {
             const row = ctx.store.db.prepare('SELECT id FROM pages WHERE slug = ?').get(slug);
-            if (stage.before !== null &&
+            if (options.placement === 'exact' &&
+                stage.before !== null &&
                 candidate.destination.section &&
                 countSection(stage.after, candidate.destination.section) !== 1) {
                 candidateResults.push({
                     candidate_id: candidate.candidate_id,
                     outcome: 'held',
                     slug,
+                    reason_code: 'validation_failed',
                     reason: 'exact destination section is absent or ambiguous',
                 });
                 continue;
@@ -162,12 +262,32 @@ async function retainProvidedExact(ctx, source, dryRun) {
                         candidate_id: candidate.candidate_id,
                         outcome: 'held',
                         slug,
+                        reason_code: 'conflict',
                         reason: 'conflict',
                     });
                     continue;
                 }
             }
-            stage.after = appendBlock(stage.after, candidate.destination.section ?? 'Unsorted', managedMemoryBlock(marker, payload));
+            if (options.placement === 'automatic') {
+                const curator = retentionModel(ctx);
+                const placed = await placeManagedItems(stage.after, [{ id: memoryId, marker, text: payload }], curator);
+                stage.after = placed.content;
+                if (placed.modelOutcome)
+                    options.modelUsage.placement.push(modelCallReceipt(curator, placed.modelOutcome));
+                if (placed.error) {
+                    candidateResults.push({
+                        candidate_id: candidate.candidate_id,
+                        outcome,
+                        memory_id: memoryId,
+                        slug,
+                        reason_code: 'placement_degraded',
+                        reason: `written under Unsorted because semantic section placement degraded: ${placed.error}`,
+                    });
+                }
+            }
+            else {
+                stage.after = appendBlock(stage.after, candidate.destination.section ?? 'Unsorted', managedMemoryBlock(marker, payload));
+            }
         }
         const evidence = sourceEvidence(candidate);
         pendingSupports.push({
@@ -177,31 +297,56 @@ async function retainProvidedExact(ctx, source, dryRun) {
             proof_group: proofGroup,
             memory_id: memoryId,
             slug,
-            selection: 'provided',
+            selection: options.selection,
             evidence,
             evidence_hash: sha256(evidence),
             source_ref: managedSourceReference(source.locator ?? source.source_id),
             origin: sourceOrigin(candidate.attribution.source_role),
             input_hash: sourceHash,
         });
-        candidateResults.push({
-            candidate_id: candidate.candidate_id,
-            outcome,
-            memory_id: memoryId,
-            slug,
-        });
+        if (!candidateResults.some((result) => result.candidate_id === candidate.candidate_id)) {
+            candidateResults.push({ candidate_id: candidate.candidate_id, outcome, memory_id: memoryId, slug });
+        }
         candidateMemoryIds.set(candidate.candidate_id, memoryId);
     }
-    candidateResults.sort((left, right) => candidateOrder.get(left.candidate_id) - candidateOrder.get(right.candidate_id));
+    candidateResults.sort((left, right) => (candidateOrder.get(left.candidate_id) ?? Number.MAX_SAFE_INTEGER) -
+        (candidateOrder.get(right.candidate_id) ?? Number.MAX_SAFE_INTEGER));
     const changed = [...stages.values()].filter((stage) => stage.before !== stage.after);
+    const placementDegraded = candidateResults.some((result) => result.reason_code === 'placement_degraded');
+    const degraded = placementDegraded
+        ? [retentionModel(ctx).available ? 'derive_failed' : 'no_derive_model']
+        : [];
     const preview = {
         source_id: source.source_id,
         revision: source.revision,
-        outcome: changed.length > 0 ? 'ok' : candidateResults.some((item) => item.outcome === 'held') ? 'held' : 'noop',
+        outcome: options.sourceHold
+            ? 'held'
+            : changed.length > 0
+                ? 'ok'
+                : candidateResults.some((item) => item.outcome === 'held')
+                    ? 'held'
+                    : 'noop',
         candidates: candidateResults,
-        ...(dryRun ? { note: 'dry run — nothing was written and no replay receipt was created' } : {}),
+        ...(options.sourceHold ? { reason_code: options.sourceHold.reason_code } : {}),
+        status: degraded.length > 0
+            ? 'degraded'
+            : options.sourceHold
+                ? 'ok'
+                : candidateResults.length === 0
+                    ? 'empty'
+                    : 'ok',
+        ...(degraded.length > 0 ? { degraded } : {}),
+        ...(options.placement === 'automatic' ? { model_usage: options.modelUsage } : {}),
+        ...(options.sourceHold ? { note: options.sourceHold.reason } : {}),
+        ...(options.dryRun
+            ? {
+                note: options.sourceHold
+                    ? `${options.sourceHold.reason}; dry run — no replay receipt was created`
+                    : 'dry run — nothing was written and no replay receipt was created',
+            }
+            : {}),
     };
-    if (dryRun)
+    if (options.dryRun)
         return preview;
     const committed = await commitStages(ctx, changed, `retained ${candidateResults.filter((item) => ['written', 'support_added'].includes(item.outcome)).length} item(s) from one source revision`);
     const changeId = committed?.changeId ?? null;
@@ -212,7 +357,11 @@ async function retainProvidedExact(ctx, source, dryRun) {
             sourceHash,
             requestHash,
             receiptFingerprint,
-            mode: 'provided_exact',
+            mode: options.selection === 'extracted'
+                ? 'extract_automatic'
+                : options.placement === 'automatic'
+                    ? 'provided_automatic'
+                    : 'provided_exact',
             result,
             changeId,
             supports: pendingSupports,
@@ -225,6 +374,67 @@ async function retainProvidedExact(ctx, source, dryRun) {
     await reindexStages(ctx, changed);
     return result;
 }
+async function resolveAutomaticCandidates(ctx, candidates) {
+    const resolved = [];
+    const held = [];
+    const catalog = folderCatalog(ctx.config, ctx.store);
+    const fallback = await resolveRememberFallback(ctx, catalog);
+    for (const candidate of candidates) {
+        const suggested = candidate.destination?.slug;
+        const routed = await routeAutomaticCandidate(ctx, {
+            ...candidate,
+            ...(suggested ? { page: suggested } : {}),
+        }, { constrainToSuggestedFolder: suggested !== undefined });
+        let slug = routed.slug;
+        if (!slug && suggested && !pageExists(ctx, suggested) && admittedAutomaticSlug(catalog, suggested)) {
+            slug = suggested;
+        }
+        if (!slug && fallback && fallback.status !== 'unavailable')
+            slug = fallback.slug;
+        if (!slug) {
+            const reasonCode = routed.blocked || (!suggested && routed.nearest.length === 0)
+                ? 'no_writable_destination'
+                : 'routing_uncertain';
+            held.push({
+                candidate_id: candidate.candidate_id,
+                outcome: 'held',
+                reason_code: reasonCode,
+                reason: routed.blocked
+                    ? `the strongest semantic destination is read-only: ${routed.blocked}`
+                    : suggested && pageExists(ctx, suggested)
+                        ? 'the suggested existing page was not judged a safe semantic destination'
+                        : 'no admitted writable destination could be established',
+            });
+            continue;
+        }
+        resolved.push({ ...candidate, destination: { slug } });
+    }
+    return { candidates: resolved, held };
+}
+function hasDestination(candidate) {
+    return candidate.destination !== undefined;
+}
+function pageExists(ctx, slug) {
+    return ctx.store.db.prepare('SELECT 1 FROM pages WHERE slug = ?').get(slug) !== undefined;
+}
+function admittedAutomaticSlug(catalog, slug) {
+    const parent = slug.slice(0, slug.lastIndexOf('/'));
+    return catalog.some((folder) => folder.path === parent &&
+        folder.role === 'knowledge' &&
+        folder.remember === 'integrate' &&
+        folder.creatable);
+}
+function retentionModel(ctx) {
+    return ctx.config.maintenance.model ? new ModelClient(ctx.config.maintenance.model) : ctx.models.derive;
+}
+function heldCandidateResult(candidate) {
+    return {
+        candidate_id: candidate.candidate_id,
+        outcome: 'held',
+        reason_code: candidate.reason_code,
+        reason: candidate.reason,
+    };
+}
 async function retractSource(ctx, source, dryRun) {
     const sourceHash = sha256('retraction');
     const requestHash = sha256(JSON.stringify(source));
@@ -239,6 +449,7 @@ async function retractSource(ctx, source, dryRun) {
             source_id: source.source_id,
             revision: source.revision,
             outcome: 'held',
+            reason_code: 'source_unavailable',
             candidates: [],
             note: 'target_revision was not retained',
         };
@@ -262,17 +473,32 @@ async function retractSource(ctx, source, dryRun) {
         }
         const stage = await existingPageStage(ctx, stages, support.slug);
         if ('issue' in stage) {
-            results.push({ candidate_id: candidateId, outcome: 'held', reason: stage.issue });
+            results.push({
+                candidate_id: candidateId,
+                outcome: 'held',
+                reason_code: 'source_unavailable',
+                reason: stage.issue,
+            });
             continue;
         }
         const block = managedBlocks(stage.after).find((entry) => entry.marker.id === support.memory_id);
         if (!block) {
-            results.push({ candidate_id: candidateId, outcome: 'held', reason: 'owned memory block changed' });
+            results.push({
+                candidate_id: candidateId,
+                outcome: 'held',
+                reason_code: 'conflict',
+                reason: 'owned memory block changed',
+            });
             continue;
         }
         const nextSupports = block.marker.supports.filter((entry) => !(entry.receipt === support.receipt_fingerprint && entry.candidate === support.candidate_fingerprint));
         if (nextSupports.length === block.marker.supports.length) {
-            results.push({ candidate_id: candidateId, outcome: 'held', reason: 'support marker changed' });
+            results.push({
+                candidate_id: candidateId,
+                outcome: 'held',
+                reason_code: 'conflict',
+                reason: 'support marker changed',
+            });
             continue;
         }
         stage.after =
@@ -389,6 +615,22 @@ function candidateIssue(ctx, source, candidate, candidateIds) {
         return attributionIssue;
     if (sourceEvidence(candidate).length > 1200)
         return 'discourse frame exceeds the evidence limit';
+    if (RELATIVE_TIME.test(sourceEvidence(candidate))) {
+        const sourceItems = 'items' in source.input ? source.input.items : [];
+        const supportedMentionTimes = new Set([
+            ...(source.mentioned_at ? [source.mentioned_at] : []),
+            ...candidate.discourse_frame.flatMap((span) => {
+                const item = sourceItems.find((entry) => entry.item_id === span.item_id);
+                return item?.mentioned_at ? [item.mentioned_at] : [];
+            }),
+        ]);
+        if (!source.timezone ||
+            !candidate.time?.mentioned_at ||
+            candidate.time.timezone !== source.timezone ||
+            !supportedMentionTimes.has(candidate.time.mentioned_at)) {
+            return 'relative time is unresolved without the exact source mention time and timezone';
+        }
+    }
     if (candidate.subject_ref &&
         !ctx.store.db.prepare('SELECT 1 FROM graph_entities WHERE id = ?').get(candidate.subject_ref.entity_id)) {
         return 'subject_ref does not name an existing canonical entity';
@@ -419,6 +661,14 @@ function candidateIssue(ctx, source, candidate, candidateIds) {
     }
     return null;
 }
+function candidateIssueReasonCode(issue) {
+    if (issue.startsWith('relative time is unresolved'))
+        return 'time_unresolved';
+    return /(?:support|discourse_frame|source|evidence).*(?:missing|ambiguous|unavailable)/i.test(issue)
+        ? 'source_unavailable'
+        : 'validation_failed';
+}
+const RELATIVE_TIME = /\b(today|tomorrow|yesterday|tonight|next\s+(?:day|week|month|year|monday|tuesday|wednesday|thursday|friday|saturday|sunday)|last\s+(?:night|week|month|year|monday|tuesday|wednesday|thursday|friday|saturday|sunday)|this\s+(?:morning|afternoon|evening|week|month|year))\b/i;
 function structuredAttributionIssue(source, candidate) {
     if (!('items' in source.input))
         return null;