@aitne/daemon 0.1.9 → 0.1.10

package/dist/core/dispatcher-scheduled-tasks.js

@@ -44,11 +44,12 @@
  *     does not own the manager's lifecycle.
  */
 import { EventPriority, createEvent, formatSqliteDatetime, getAgentDayDateStr, isBackendId, isKnowledgeImportEvent, isRoutineEvent, resolveProcessKey, } from "@aitne/shared";
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
 import { join } from "node:path";
 import { randomUUID } from "node:crypto";
-import { CONTEXT_RELATIVE_PATHS } from "./context-paths.js";
+import { CONTEXT_RELATIVE_PATHS, agentLessonsPath } from "./context-paths.js";
 import { getContextDir } from "../config.js";
+import { ensureTodaySkeleton } from "./today-direct-writer.js";
 import { cleanupSessionWorkdir, ensureBackendMaterialized, } from "./workdir.js";
 import { readIntegrations, readIntegrationState } from "../db/integrations-store.js";
 import { refreshInterestsReflection } from "../services/browser-history/refresh-interests-reflection.js";
@@ -60,6 +61,10 @@ import { getRepository, getRepositoryByLocalPath, recordManagementInitDone, reco
 import { runRepositoryManagementInit, runRepositoryManagementScan, } from "./repository-management-docs.js";
 import { routineWindowKeyFromEvent } from "./routine-fetch-window-runner.js";
 import { routineHasWindows } from "./routine-windows.js";
+import { buildFeedbackWorksheet, gatherFeedbackWorksheetScopes, lessonCapsForScope, GLOBAL_LESSON_ENTRY_CAP, PER_AGENT_LESSON_ENTRY_CAP, } from "./feedback/consolidation-prep.js";
+import { buildRegeneralizationWorksheet, } from "./feedback/regeneralization-prep.js";
+import { isSafeAgentSlug, scopeStoreFile, } from "./feedback/scope-parser.js";
+import { feedbackRetentionCutoff, sweepConsumedFeedbackSignals, } from "../db/feedback-signals-store.js";
 import { morningRoutineRanToday } from "../bootstrap/schedule-helpers.js";
 import { resolveAgentId } from "./agents/agent-id-resolver.js";
 import { loadEffectiveDefinition } from "./agents/effective-definition.js";
@@ -227,6 +232,7 @@ export class ScheduledTaskRunner {
     morningRoutine;
     fetchWindowRunner;
     roadmapWriteLock;
+    todayWriteLock;
     writeTracker;
     getConfiguredServices;
     getActiveMailAccounts;
@@ -243,6 +249,7 @@ export class ScheduledTaskRunner {
         this.morningRoutine = deps.morningRoutine;
         this.fetchWindowRunner = deps.fetchWindowRunner;
         this.roadmapWriteLock = deps.roadmapWriteLock;
+        this.todayWriteLock = deps.todayWriteLock;
         this.writeTracker = deps.writeTracker;
         this.getConfiguredServices = deps.getConfiguredServices;
         this.getActiveMailAccounts = deps.getActiveMailAccounts;
@@ -1085,6 +1092,81 @@ export class ScheduledTaskRunner {
                     };
                 }
             }
+            // Deterministic pre-step for `routine.today_refresh`. rotateDayFiles()
+            // intentionally leaves today.md absent at the day boundary for the
+            // morning routine to recreate; when that routine has not yet run (or
+            // failed — e.g. a quota/budget death with no fallback), today.md is
+            // missing and this section-only refresh would 404 on its PATCH and
+            // fall back to budget-burning full-file PUTs against the strict
+            // validateTodayContent schema (the "Refresh Today does nothing"
+            // symptom). Guarantee the canonical skeleton exists (lock-aware,
+            // absent-only) so the refresh session has a valid PATCH target; full
+            // creation/repair stays the morning routine's job. See
+            // ensureTodaySkeleton. Skipped when no today-write-lock was wired.
+            if (this.todayWriteLock
+                && isRoutineEvent(effectiveEvent)
+                && effectiveEvent.routine === "today_refresh") {
+                // Failure-isolated like the sibling pre-steps below: this seed is a
+                // best-effort convenience so the refresh has a valid PATCH target.
+                // A throw here (e.g. getContextDir / serializer-lock internals) must
+                // NOT abort the whole today_refresh — that would reproduce the exact
+                // "Refresh Today does nothing" symptom the skeleton was added to
+                // prevent. The morning routine remains responsible for real repair.
+                try {
+                    await ensureTodaySkeleton({
+                        contextDir: getContextDir(this.config, this.db),
+                        todayWriteLock: this.todayWriteLock,
+                    });
+                }
+                catch (err) {
+                    logger.warn({ err }, "Failed to seed today.md skeleton for today_refresh");
+                }
+            }
+            // FEEDBACK_LEARNING_LOOP_DESIGN.md §4 — deterministic consolidation
+            // pre-step for `routine.evening_review`. Runs synchronously BEFORE
+            // `contextBuilder.build` so the `<feedback_worksheet>` (unconsumed
+            // signals grouped by scope + per-candidate promotion verdicts +
+            // eviction ranking + consume ids) lands in the review session's
+            // context. Failure-isolated inside `prepareFeedbackWorksheet`: a throw
+            // there is swallowed and the review still ships. Skipped when feedback
+            // learning is off or no signals pend (no empty block in the prompt).
+            if (isRoutineEvent(effectiveEvent)
+                && effectiveEvent.routine === "evening_review") {
+                const worksheetBlock = this.prepareFeedbackWorksheet();
+                if (worksheetBlock) {
+                    effectiveEvent = {
+                        ...effectiveEvent,
+                        data: {
+                            ...effectiveEvent.data,
+                            feedbackWorksheetBlock: worksheetBlock,
+                        },
+                    };
+                }
+            }
+            // FEEDBACK_LEARNING_LOOP_DESIGN.md §4 "Monthly re-generalization" /
+            // Phase 5 — deterministic pre-step for `routine.monthly_review`. Re-reads
+            // the consolidated lesson stores (global + per-agent) and stamps a
+            // `<feedback_regeneralization>` block so the monthly session collapses
+            // same-theme lessons into higher-level principles. Distinct from the
+            // nightly evening-review pass: it carries no signals and consumes
+            // nothing — it only surfaces existing lessons ranked for collapse.
+            // Failure-isolated inside `prepareRegeneralizationWorksheet`; skipped
+            // when feedback learning is off or no scope has enough lessons. Rides the
+            // same `monthlyReviewEnabled` kill switch as the rest of the monthly
+            // routine (the routine never dispatches while it is off).
+            if (isRoutineEvent(effectiveEvent)
+                && effectiveEvent.routine === "monthly_review") {
+                const regeneralizationBlock = this.prepareRegeneralizationWorksheet();
+                if (regeneralizationBlock) {
+                    effectiveEvent = {
+                        ...effectiveEvent,
+                        data: {
+                            ...effectiveEvent.data,
+                            regeneralizationBlock,
+                        },
+                    };
+                }
+            }
             // WEEKLY_INTERESTS_REFLECTION_PLAN.md §10.4 — deterministic
             // pre-hook for `routine.weekly_review`. Runs synchronously
             // BEFORE `contextBuilder.build` so the freshly-refreshed
@@ -1277,6 +1359,189 @@ export class ScheduledTaskRunner {
             this.appendWeeklyInterestsJournalLine(`interest reflection failed: ${message}`);
         }
     }
+    /**
+     * FEEDBACK_LEARNING_LOOP_DESIGN.md §4 — the deterministic consolidation
+     * pre-step. Reads unconsumed `feedback_signals` (user + agent + `agent:<slug>`
+     * scope as of Phase 4), reads each lessons store's current contents, and
+     * composes the `<feedback_worksheet>` block via the pure `core/feedback/*`
+     * modules. Returns the block string, or `null` when feedback learning is off,
+     * nothing pends, or anything throws — so the caller simply skips stamping and
+     * the evening review proceeds unchanged.
+     *
+     * The DB read + per-scope file read live here (the FS-/DB-heavy dispatcher
+     * is coverage-excluded); the byte-deterministic worksheet composition lives
+     * in `buildFeedbackWorksheet`, which is 100% unit-tested.
+     */
+    prepareFeedbackWorksheet() {
+        if (this.config.feedbackLearningEnabled === false)
+            return null;
+        try {
+            // Nightly retention close-out (§4 step 6 / §6): drop signal rows that
+            // were consumed past the retention horizon so the raw table stays
+            // bounded. Only touches already-consumed rows — an unconsolidated
+            // signal is never lost to retention. Runs once per Evening Review.
+            // Guarded against a missing/NaN retention knob so a config gap degrades
+            // to "skip the sweep", not "throw and abandon the whole consolidation".
+            // The guard + cutoff math live in the pure, 100%-covered
+            // `feedbackRetentionCutoff` helper (returns null → skip).
+            const retentionCutoff = feedbackRetentionCutoff(this.config.feedbackSignalRetentionDays, Date.now());
+            if (retentionCutoff) {
+                sweepConsumedFeedbackSignals(this.db, retentionCutoff);
+            }
+            const groups = gatherFeedbackWorksheetScopes(this.db, {
+                // Phase 4 adds `agent_slug` — per-agent (`agent:<slug>`) signals captured
+                // by the explicit route + behavioral sink now fold into
+                // `policies/agents/<slug>/lessons.md`. Per-scope-type fetch (not a global
+                // LIMIT) keeps an `agent_slug` backlog from starving user/agent.
+                scopeTypes: ["user", "agent", "agent_slug"],
+            });
+            if (groups.length === 0)
+                return null;
+            const contextDir = getContextDir(this.config, this.db);
+            const byteCaps = {
+                global: this.config.feedbackLessonMaxBytesGlobal,
+                perAgent: this.config.feedbackLessonMaxBytesPerAgent,
+            };
+            const scopes = [];
+            for (const group of groups) {
+                const caps = lessonCapsForScope(group.scope, byteCaps);
+                let existingFileMd = null;
+                if (caps) {
+                    const rel = scopeStoreFile(group.scope);
+                    if (rel) {
+                        const full = join(contextDir, rel);
+                        // Per-scope read isolation (FEEDBACK_LEARNING_LOOP_DESIGN.md §11
+                        // v1.9 nuance #3, now closed). A *missing* file is the normal
+                        // first-write case (existingFileMd stays null → the LLM PUTs a
+                        // fresh store); but a *present-but-unreadable* file (EACCES, a
+                        // mid-write truncation) must NOT forfeit the whole night's
+                        // consolidation. Skip just this scope: its signals stay unconsumed
+                        // and retry next Evening Review, the file is left untouched, and the
+                        // user/agent scopes that read cleanly still consolidate. Crucially
+                        // we do NOT fall back to existingFileMd=null on a read error — that
+                        // would make the LLM PUT a fresh file and destroy the existing
+                        // lessons.
+                        try {
+                            existingFileMd = existsSync(full)
+                                ? readFileSync(full, "utf-8")
+                                : null;
+                        }
+                        catch (err) {
+                            logger.warn({ err, path: rel }, "Skipping feedback scope — lessons file present but unreadable; its signals will retry next Evening Review");
+                            continue;
+                        }
+                    }
+                }
+                scopes.push({
+                    scope: group.scope,
+                    signals: group.signals,
+                    existingFileMd,
+                    caps,
+                });
+            }
+            if (scopes.length === 0)
+                return null;
+            const result = buildFeedbackWorksheet(scopes, {
+                promotionThreshold: this.config.feedbackPromotionThreshold,
+                nowIso: new Date().toISOString(),
+                staleDays: this.config.feedbackLessonStaleDays,
+            });
+            return result?.block ?? null;
+        }
+        catch (err) {
+            logger.warn({ err }, "Failed to prepare feedback worksheet");
+            return null;
+        }
+    }
+    /**
+     * FEEDBACK_LEARNING_LOOP_DESIGN.md §4 "Monthly re-generalization" / Phase 5 —
+     * the deterministic monthly pre-step. Enumerates the consolidated lesson
+     * stores on disk (the global `policies/agent-lessons.md` plus every per-agent
+     * `policies/agents/<slug>/lessons.md`), reads their contents, and composes a
+     * `<feedback_regeneralization>` block via the pure
+     * `buildRegeneralizationWorksheet`. Returns the block, or `null` when feedback
+     * learning is off, no store holds enough lessons to collapse, or anything
+     * throws — so the caller simply skips stamping and the monthly review
+     * proceeds unchanged.
+     *
+     * The FS enumeration lives here (the dispatcher is coverage-excluded); the
+     * byte-deterministic composition lives in `buildRegeneralizationWorksheet`,
+     * which is 100% unit-tested.
+     */
+    prepareRegeneralizationWorksheet() {
+        if (this.config.feedbackLearningEnabled === false)
+            return null;
+        try {
+            const contextDir = getContextDir(this.config, this.db);
+            const scopes = [];
+            // Per-file read isolation (FEEDBACK_LEARNING_LOOP_DESIGN.md §11 v1.9
+            // nuance #3, now closed): one present-but-unreadable lessons file must
+            // skip only that scope, not abandon the whole monthly collapse. A scope
+            // that fails to read is simply not surfaced for re-generalization this
+            // month; the file is left untouched.
+            const readScopeFile = (rel) => {
+                const full = join(contextDir, rel);
+                if (!existsSync(full))
+                    return null;
+                try {
+                    return readFileSync(full, "utf-8");
+                }
+                catch (err) {
+                    logger.warn({ err, path: rel }, "Skipping re-generalization scope — lessons file present but unreadable");
+                    return null;
+                }
+            };
+            // Global `agent`-scope store.
+            const globalRel = CONTEXT_RELATIVE_PATHS.agentLessons;
+            const globalMd = readScopeFile(globalRel);
+            if (globalMd !== null) {
+                scopes.push({
+                    scope: { kind: "agent" },
+                    storeFile: globalRel,
+                    existingFileMd: globalMd,
+                    caps: {
+                        capBytes: this.config.feedbackLessonMaxBytesGlobal,
+                        maxEntries: GLOBAL_LESSON_ENTRY_CAP,
+                    },
+                });
+            }
+            // Per-agent `agent:<slug>` stores under `policies/agents/<slug>/lessons.md`.
+            // The slug is the directory name; the same `isSafeAgentSlug` guard the
+            // inject + consolidate sides apply rejects any unsafe directory name so a
+            // hand-created `..`-style dir never reaches the worksheet.
+            const agentsDir = join(contextDir, "policies", "agents");
+            if (existsSync(agentsDir)) {
+                for (const entry of readdirSync(agentsDir, { withFileTypes: true })) {
+                    if (!entry.isDirectory() || !isSafeAgentSlug(entry.name))
+                        continue;
+                    const rel = agentLessonsPath(entry.name);
+                    const md = readScopeFile(rel);
+                    if (md === null)
+                        continue;
+                    scopes.push({
+                        scope: { kind: "agent_slug", ref: entry.name },
+                        storeFile: rel,
+                        existingFileMd: md,
+                        caps: {
+                            capBytes: this.config.feedbackLessonMaxBytesPerAgent,
+                            maxEntries: PER_AGENT_LESSON_ENTRY_CAP,
+                        },
+                    });
+                }
+            }
+            if (scopes.length === 0)
+                return null;
+            const result = buildRegeneralizationWorksheet(scopes, {
+                nowIso: new Date().toISOString(),
+                staleDays: this.config.feedbackLessonStaleDays,
+            });
+            return result?.block ?? null;
+        }
+        catch (err) {
+            logger.warn({ err }, "Failed to prepare feedback re-generalization worksheet");
+            return null;
+        }
+    }
     /**
      * Append a single bullet under `## Weekly interests reflection`
      * inside `context/journal/agent.md`, creating the section (and the

package/dist/core/dispatcher.js

@@ -587,6 +587,7 @@ export class EventDispatcher {
             morningRoutine: this.morningRoutine,
             fetchWindowRunner: this.fetchWindowRunner,
             roadmapWriteLock: this.roadmapWriteLock,
+            todayWriteLock: this.todayWriteLock,
             writeTracker: this.writeTracker,
             getConfiguredServices: () => this.getConfiguredServices(),
             getActiveMailAccounts: () => this.getActiveMailAccounts(),
@@ -803,10 +804,21 @@ export class EventDispatcher {
         const scheduleRowId = isScheduledEvent(event) && event.scheduleId !== undefined
             ? event.scheduleId
             : null;
-        this.agentExecutionTracker.begin(event.correlationId, resolution, {
+        // `begin` opens the rollup row AND returns the resolved Agent slug (or null
+        // for a firing that resolves to no Agent — reactive DMs, legacy tasks).
+        const agentId = this.agentExecutionTracker.begin(event.correlationId, resolution, {
             scheduleRowId,
             trigger: this.resolveExecutionTrigger(event, taskContext),
         });
+        // FEEDBACK_LEARNING_LOOP_DESIGN.md §5 Phase 4 — thread the resolved slug onto
+        // the event so `ContextBuilder` can inject this Agent's own
+        // `policies/agents/<slug>/lessons.md` (scope `agent:<slug>`). This runs in
+        // `dispatchSafe` before `dispatch(event)`, so the stamp is visible to the
+        // builder downstream; the morning routine propagates it to Stage A via the
+        // `{...parent.data}` spread in `composeStageAEvent`. No-op for unbound runs.
+        if (agentId !== null) {
+            event.data.agentId = agentId;
+        }
     }
     /** Classify an execution's trigger for the rollup row (§5.2). */
     resolveExecutionTrigger(event, taskContext) {

package/dist/core/feedback/consolidation-prep.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Feedback Learning Loop — consolidation pre-step (FEEDBACK_LEARNING_LOOP_DESIGN.md §4).
+ *
+ * The daemon-side, deterministic half of Stage 2. On the evening-review tick it
+ * reads unconsumed `feedback_signals`, groups them by `(scope_type, scope_ref)`,
+ * pre-computes each candidate's weighted evidence + promotion verdict and each
+ * lessons file's eviction ranking + headroom, and emits a `<feedback_worksheet>`
+ * block — exactly as `<journal_skeleton>` / `harvestForGate` blocks are
+ * daemon-prepared today. The LLM step then does only the *semantic* work
+ * (intent-match merge, contradiction detection, phrasing) and writes via
+ * `PATCH /api/context/policies/agent-lessons`, then consumes the worksheet's ids.
+ *
+ * Two layers, mirroring `journal-skeleton-builder.ts`:
+ *   - `gatherFeedbackWorksheetScopes(db, …)` — the single DB read (side-effect
+ *     free); groups pending signals by scope. Cost scales with feedback volume,
+ *     not agent count (the `idx_feedback_unconsumed` partial index).
+ *   - `buildFeedbackWorksheet(scopes, …)` — pure markdown/XML composer. Every
+ *     output byte is a deterministic function of its inputs; the caller supplies
+ *     each lessons file's current contents so this stays I/O-free and 100%
+ *     coverable.
+ *
+ * Phase 2 stored `user` + `agent`; Phase 4 added `agent:<slug>` (the evening-review
+ * pre-step now requests it). This module already rendered any lessons scope
+ * generically, so Phase 4 was wiring (`scopeTypes` + task-flow), not new logic here.
+ */
+import type Database from "better-sqlite3";
+import { type FeedbackScopeType, type FeedbackSignalRow } from "../../db/feedback-signals-store.js";
+import { type CanonicalScope } from "./scope-parser.js";
+/** Fixed entry caps (§6 table) — config carries only the byte caps. */
+export declare const GLOBAL_LESSON_ENTRY_CAP = 40;
+export declare const PER_AGENT_LESSON_ENTRY_CAP = 20;
+export interface WorksheetScopeGroup {
+    scope: CanonicalScope;
+    signals: FeedbackSignalRow[];
+}
+/**
+ * Read unconsumed signals for the requested scope types and group them by
+ * canonical scope. Each scope type is queried independently (oldest-first
+ * within the type) so the per-pass row budget applies *per type*. A single
+ * global `LIMIT` over `created_at ASC` would let a backlog of unconsumed
+ * `agent_slug` rows occupy the oldest-N window and silently starve the
+ * `user`/`agent` scopes — the per-type fetch caps each scope type
+ * independently so a busy agent can't crowd out the others. Groups come back in
+ * `scopeTypes` order; rows whose `(scope_type, scope_ref)` can't be parsed
+ * (defensive — the route + behavioral sink only write valid pairs) are skipped
+ * so a bad row never breaks the pass.
+ */
+export declare function gatherFeedbackWorksheetScopes(db: Database.Database, opts: {
+    scopeTypes: ReadonlyArray<FeedbackScopeType>;
+    limit?: number;
+}): WorksheetScopeGroup[];
+/** Resolve per-scope byte/entry caps; `null` for raw (user) + unstored scopes. */
+export declare function lessonCapsForScope(scope: CanonicalScope, byteCaps: {
+    global: number;
+    perAgent: number;
+}): {
+    capBytes: number;
+    maxEntries: number;
+} | null;
+export interface WorksheetScopeInput {
+    scope: CanonicalScope;
+    signals: FeedbackSignalRow[];
+    /** Current lessons-store file contents (lessons scopes), else null. */
+    existingFileMd: string | null;
+    /** Byte/entry caps for lessons scopes; null for raw (user) scopes. */
+    caps: {
+        capBytes: number;
+        maxEntries: number;
+    } | null;
+}
+export interface BuildWorksheetOptions {
+    promotionThreshold: number;
+    nowIso: string;
+    recencyHalfLifeDays?: number;
+    /**
+     * Staleness horizon in days (`feedbackLessonStaleDays`, §4 step 7). An
+     * existing lesson whose `last=` predates `now − staleDays` and is not a
+     * `constraint` is flagged `stale="true"` so the LLM prunes it in the rebuild.
+     * Omitted ⇒ nothing is flagged stale (no time-based prune this pass).
+     */
+    staleDays?: number;
+}
+export interface WorksheetResult {
+    /** `<feedback_worksheet>…</feedback_worksheet>` block for verbatim injection. */
+    block: string;
+    /** Every surfaced signal id — the exact consume set (§4 step 6). */
+    signalIds: number[];
+    scopeCount: number;
+}
+/**
+ * Compose the `<feedback_worksheet>` block. Returns `null` when there are no
+ * signals at all (the caller then stamps nothing — no empty block in the prompt).
+ */
+export declare function buildFeedbackWorksheet(scopes: ReadonlyArray<WorksheetScopeInput>, opts: BuildWorksheetOptions): WorksheetResult | null;