@vue-skuilder/db 0.1.23 → 0.1.25

package/docs/navigators-architecture.md

@@ -5,6 +5,9 @@
 The navigation strategy system selects and scores cards for study sessions. It uses a
 **Pipeline architecture** where generators produce candidates and filters transform scores.
 
+An **Evolutionary Orchestration** layer enables strategies to carry learnable weights that
+automatically tune toward optimal values based on observed learning outcomes.
+
 ## Core Concepts
 
 ### WeightedCard
@@ -27,6 +30,7 @@ interface StrategyContribution {
   action: 'generated' | 'passed' | 'boosted' | 'penalized';
   score: number;              // Score after this strategy
   reason: string;             // Human-readable explanation
+  deviation?: number;         // User's deviation from peak weight (evolutionary)
 }
 ```
 
@@ -42,11 +46,28 @@ interface CardGenerator {
 ```
 
 **Implementations:**
-- `ELONavigator` — New cards scored by ELO proximity to user skill
-- `SRSNavigator` — Review cards scored by overdueness and interval recency
+- `ELONavigator` — New cards scored by ELO proximity to user skill (scores 0.0-1.0)
+- `SRSNavigator` — Review cards scored by overdueness, interval recency, and **backlog pressure** (scores 0.5-1.0)
 - `HardcodedOrderNavigator` — Fixed sequence defined by course author
 - `CompositeGenerator` — Merges multiple generators with frequency boost
 
+#### SRS Backlog Pressure
+
+The SRS generator implements a self-regulating **backlog pressure** mechanism that prevents review pile-up while maintaining healthy new/review balance:
+
+- **Healthy backlog** (≤20 due reviews): No pressure boost, scores 0.5-0.95. New content (ELO) naturally dominates.
+- **Elevated backlog** (40 due): +0.25 boost, scores 0.75-1.0. Reviews compete with new cards.
+- **High backlog** (60+ due): +0.50 boost (max), scores 0.95-1.0. Reviews take priority.
+
+This treats SRS scheduling times as **eligibility dates** rather than hard due dates—reviewing slightly later may be optimal. The system maintains a healthy backlog rather than always clearing to zero (avoiding "Anki death spiral").
+
+Configuration via strategy `serializedData`:
+```json
+{ "healthyBacklog": 20 }
+```
+
+See `todo-review-adaptation.md` for planned per-user adaptation extensions.
+
 ### CardFilter
 
 Transforms card scores (pure function, no side effects):
@@ -82,7 +103,7 @@ class Pipeline {
   )
 
   async getWeightedCards(limit: number): Promise<WeightedCard[]> {
-    // Build shared context (user ELO, etc.)
+    // Build shared context (user ELO, orchestration context, etc.)
     const context = await this.buildContext();
 
     // Generate candidates
@@ -104,7 +125,7 @@ class Pipeline {
 ```
 
 **Responsibilities:**
-- **Context building** — Fetches shared data (user ELO) once for all strategies
+- **Context building** — Fetches shared data (user ELO, orchestration context) once for all strategies
 - **Data hydration** — Pre-fetches commonly needed data (tags) in batch queries
 - **Filter orchestration** — Applies filters in sequence, accumulating provenance
 - **Result selection** — Removes zero-scores, sorts, and returns top N
@@ -170,7 +191,8 @@ provenance: [
     strategyId: 'NAVIGATION_STRATEGY-hierarchy-phonics',
     action: 'passed',
     score: 0.85,
-    reason: 'Prerequisites met, tags: letter-sounds'
+    reason: 'Prerequisites met, tags: letter-sounds',
+    deviation: 0.23  // User's position on bell curve for this strategy
   },
   {
     strategy: 'eloDistance',
@@ -185,6 +207,171 @@ provenance: [
 
 Use `getCardOrigin(card)` to extract 'new', 'review', or 'failed' from provenance.
 
+---
+
+## Evolutionary Orchestration
+
+The orchestration layer enables strategies to **learn optimal weights** from observed
+learning outcomes. Instead of fixed configuration, strategies carry **learnable weights**
+that automatically tune toward effectiveness.
+
+### LearnableWeight
+
+Every strategy can carry a learnable weight:
+
+```typescript
+interface LearnableWeight {
+  weight: number;       // Peak value, 1.0 = neutral, range [0.1, 3.0]
+  confidence: number;   // 0-1, controls exploration width
+  sampleSize: number;   // Total observations for this strategy
+
+  history?: Array<{     // Optional: for visualization
+    timestamp: string;
+    weight: number;
+    confidence: number;
+    gradient: number;
+  }>;
+}
+```
+
+Strategies extend with:
+
+```typescript
+interface ContentNavigationStrategyData {
+  // ... existing fields ...
+  learnable?: LearnableWeight;   // Omitted = default weight 1.0
+  staticWeight?: boolean;        // If true, not subject to learning
+}
+```
+
+### Deviation-Based Weight Distribution
+
+Each user experiences a **stable deviation** from the peak weight:
+
+```
+effectiveWeight(user, strategy) = peakWeight + deviation * spread
+
+where:
+  deviation = hash(userId, strategyId, salt) → [-1, 1]  // stable per user
+  spread = max(MIN_SPREAD, (1 - confidence) * MAX_SPREAD)
+```
+
+**Key insight:** Deviation is constant for a user. As confidence grows and spread
+shrinks, all users are pulled toward the optimal peak.
+
+```typescript
+// Example: Low confidence = wide exploration
+{ weight: 1.0, confidence: 0.2 }
+→ spread = 0.8 * MAX_SPREAD
+→ users range from weight 0.6 to 1.4
+
+// Example: High confidence = narrow convergence
+{ weight: 1.2, confidence: 0.9 }
+→ spread = 0.1 * MAX_SPREAD
+→ users cluster around weight 1.15 to 1.25
+```
+
+### Outcome Recording
+
+At the end of each learning period, user outcomes are recorded:
+
+```typescript
+interface UserOutcomeRecord {
+  _id: `USER_OUTCOME::${courseId}::${periodId}`;
+  docType: DocType.USER_OUTCOME;
+  userId: string;
+  courseId: string;
+  periodStart: string;
+  periodEnd: string;
+  strategyExposures: Array<{
+    strategyId: string;
+    deviation: number;      // User's stable deviation for this strategy
+  }>;
+  outcomeValue: number;     // 0-1 learning outcome signal
+}
+```
+
+The outcome signal combines multiple factors:
+- Accuracy within target zone (not too easy, not too hard)
+- ELO progression
+- Session completion
+
+### Gradient Learning
+
+The system discovers optimal weights by correlating **deviation with outcomes**
+across users:
+
+```
++deviation → +outcome = positive gradient → increase peak weight
++deviation → -outcome = negative gradient → decrease peak weight
+flat gradient = at optimum, increase confidence
+```
+
+Linear regression on (deviation, outcome) pairs produces:
+- **Gradient**: Direction to adjust peak
+- **R²**: Signal quality (high = consistent effect)
+- **Sample size**: Update confidence
+
+### Weight Update Cycle
+
+Periodically (or on-demand), the system updates strategy weights:
+
+```typescript
+async function runPeriodUpdate(courseId: string): Promise<void> {
+  for (const strategy of await getLearnableStrategies(courseId)) {
+    const observations = await aggregateOutcomesForGradient(strategy);
+    const { gradient, rSquared } = computeStrategyGradient(observations);
+    await updateStrategyWeight(strategy, gradient, rSquared);
+  }
+}
+```
+
+**Update rules:**
+- Positive gradient → increase peak weight
+- Negative gradient → decrease peak weight
+- Consistent observations → increase confidence
+- Noisy observations → decrease confidence
+
+### Observability
+
+The orchestration layer exposes API endpoints for monitoring:
+
+| Endpoint | Purpose |
+|----------|---------|
+| `GET /orchestration/:courseId/state` | All learning states |
+| `GET /orchestration/:courseId/weights` | Current weights summary |
+| `GET /orchestration/:courseId/strategy/:id/history` | Weight trajectory over time |
+| `GET /orchestration/:courseId/strategy/:id/scatter` | Deviation vs outcome data |
+| `GET /orchestration/:courseId/strategy/:id/distribution` | Bell curve visualization |
+| `POST /orchestration/:courseId/update` | Trigger period update |
+
+An admin dashboard in platform-ui visualizes strategy weights, confidence,
+gradient direction, and historical trajectories.
+
+### Lifecycle
+
+```
+New strategy:    Low confidence → wide spread → noisy gradient → big adjustments
+Learning:        Gradient visible → peak drifts → confidence grows → spread shrinks
+Converged:       High confidence → minimum spread → flat gradient → stable
+Disturbed:       Gradient reappears → peak drifts → adapts to new optimal
+```
+
+### Static vs Learnable
+
+Set `staticWeight: true` for foundational strategies that should not be tuned:
+
+```typescript
+{
+  strategyType: 'hierarchyDefinition',
+  name: 'Core Prerequisites',
+  staticWeight: true,  // Never tuned by orchestration
+  // learnable field ignored
+}
+```
+
+---
+
 ## Creating New Strategies
 
 ### Generator
@@ -249,6 +436,8 @@ class MyFilter extends ContentNavigator implements CardFilter {
 
 Register in `NavigatorRoles` as `NavigatorRole.FILTER`.
 
+---
+
 ## Strategy State Storage
 
 Strategies can persist user-scoped state (preferences, learned patterns, temporal tracking)
@@ -294,7 +483,7 @@ interface StrategyStateDoc<T> {
   courseId: string;
   strategyKey: string;
   data: T;               // Strategy-specific payload
-  updatedAt: string;     // ISO timestamp
+  updatedAt: string;
 }
 ```
 
@@ -332,6 +521,8 @@ return { ...card, score: card.score * multiplier };
 - All sliders share global max for consistent visual comparison
 - Writes to strategy state via `userDB.putStrategyState()`
 
+---
+
 ## File Reference
 
 | File | Purpose |
@@ -342,8 +533,8 @@ return { ...card, score: card.score * multiplier };
 | `core/navigators/Pipeline.ts` | Pipeline orchestration |
 | `core/navigators/PipelineAssembler.ts` | Builds Pipeline from strategy docs |
 | `core/navigators/CompositeGenerator.ts` | Merges multiple generators |
-| `core/navigators/elo.ts` | ELO generator |
-| `core/navigators/srs.ts` | SRS generator |
+| `core/navigators/generators/elo.ts` | ELO generator |
+| `core/navigators/generators/srs.ts` | SRS generator (with backlog pressure) |
 | `core/navigators/hardcodedOrder.ts` | Fixed-order generator |
 | `core/navigators/hierarchyDefinition.ts` | Prerequisite filter |
 | `core/navigators/interferenceMitigator.ts` | Interference filter |
@@ -355,9 +546,19 @@ return { ...card, score: card.score * multiplier };
 | `core/navigators/inferredPreference.ts` | Inferred preference navigator (stub) |
 | `core/types/strategyState.ts` | `StrategyStateDoc`, `StrategyStateId` |
 | `impl/couch/courseDB.ts` | `createNavigator()` entry point |
+| `core/orchestration/index.ts` | OrchestrationContext, deviation logic |
+| `core/orchestration/gradient.ts` | Gradient computation |
+| `core/orchestration/learning.ts` | Weight updates, period orchestration |
+| `core/orchestration/signal.ts` | Outcome signal computation |
+| `core/orchestration/recording.ts` | User outcome recording |
+| `core/types/learningState.ts` | `StrategyLearningState` |
+| `core/types/userOutcome.ts` | `UserOutcomeRecord` |
+| `express/routes/orchestration.ts` | Observability API endpoints |
 
 ## Related Documentation
 
 - `todo-strategy-authoring.md` — UX and DX for authoring strategies
-- `todo-evolutionary-orchestration.md` — Long-term adaptive strategy vision
+- `todo-review-adaptation.md` — Planned per-user review urgency adaptation
+- `future-orchestration-vision.md` — Long-term adaptive strategy vision (beyond current implementation)
 - `devlog/1004` — Implementation details for tag hydration optimization
+- `devlog/1032-orchestrator` — Evolutionary orchestration implementation details

package/docs/todo-review-urgency-adaptation.md

@@ -0,0 +1,205 @@
+# Future Work: Dynamic Review Urgency Adaptation
+
+This document outlines planned extensions to the SRS backlog pressure system for adaptive review urgency tuning.
+
+## Background
+
+The SRS generator now implements **backlog pressure** — a mechanism that boosts review urgency when the number of due reviews exceeds a "healthy" threshold (default: 20). This ensures reviews don't pile up indefinitely while maintaining a healthy mix of new and review content.
+
+See implementation: `core/navigators/generators/srs.ts`
+
+This document describes two extension directions:
+1. **Global/Orchestration Layer** — Course-wide tuning via learnable weights
+2. **Per-User Adaptation** — Individual urgency multipliers based on review outcomes
+
+---
+
+## Extension 1: Orchestrator-Tuned Generator Weights
+
+### Current State
+
+The orchestration layer (`core/orchestration/`) already supports learnable weights:
+
+```typescript
+interface LearnableWeight {
+  weight: number;       // Peak value, 1.0 = neutral
+  confidence: number;   // 0-1, controls exploration width
+  sampleSize: number;   // Observations collected
+}
+```
+
+When a strategy has `learnable` set and `staticWeight: false`, the orchestrator:
+1. Distributes users across a weight range (bell curve around peak)
+2. Records learning outcomes per user
+3. Correlates user deviation with outcomes via gradient estimation
+4. Updates peak weight toward observed optimum
+
+### Application to Review Balance
+
+The orchestrator can tune the ELO vs SRS balance by experimenting with their weights.
+
+In `CompositeGenerator.getWeightedCards()`, weights are already applied:
+
+```typescript
+let weight = gen.learnable?.weight ?? 1.0;
+if (gen.learnable && !gen.staticWeight && context.orchestration) {
+  weight = context.orchestration.getEffectiveWeight(strategyId, gen.learnable);
+}
+```
+
+If SRS has `learnable: { weight: 1.0, confidence: 0.5, sampleSize: 0 }`, the orchestrator will try variations (0.8, 1.2, etc.) and correlate with learning outcomes.
+
+### Required Work
+
+- [ ] Ensure default ELO and SRS strategy documents include `learnable` config
+- [ ] Verify outcome correlation tracks success/failure per card source (generator provenance)
+- [ ] Consider weight bounds (e.g., SRS weight ∈ [0.5, 2.0]) to prevent extreme tuning
+- [ ] Test interaction with backlog pressure — may need to mark backlog-pressure-derived scores as non-learnable
+
+### Risks
+
+- Orchestrator experiments could override backlog pressure intent
+- May need to separate "base urgency" (learnable) from "backlog pressure" (not learnable)
+
+### Priority
+
+**Low** — The backlog pressure mechanism should handle balance well. Orchestrator tuning is a refinement layer that can be enabled later.
+
+---
+
+## Extension 2: Per-User Review Urgency Adaptation
+
+### Concept
+
+Users have different memory characteristics. A "forgetful" user needs more review pressure; a "retentive" user needs less. Rather than one-size-fits-all, maintain a per-user `reviewUrgencyMultiplier` that adapts based on review outcomes.
+
+### Proposed Data Model
+
+```typescript
+interface ReviewAdaptationState {
+  urgencyMultiplier: number;  // Default 1.0, range [0.5, 2.0]
+  windowStart: string;        // ISO timestamp for rolling window
+  windowFailures: number;     // Failures in current window
+  windowSuccesses: number;    // Successes in current window
+  lastUpdated: string;        // ISO timestamp
+}
+```
+
+### Update Rules
+
+- On failed review: `multiplier = min(2.0, multiplier + 0.05)`
+- On successful review: `multiplier = max(0.5, multiplier - 0.01)`
+
+**Asymmetry rationale**:
+- Failures increase pressure quickly (5x faster than decrease)
+- Successes slowly reduce pressure (avoid death spiral from one good session)
+
+### Equilibrium Analysis
+
+At equilibrium: `0.05 * failRate = 0.01 * (1 - failRate)`
+
+Solving: `failRate = 0.167` (16.7%)
+
+This means:
+- User with 16.7% failure rate → multiplier stable
+- User with >16.7% failure rate → multiplier increases (more review pressure)
+- User with <16.7% failure rate → multiplier decreases (less review pressure)
+
+The equilibrium threshold can be tuned by adjusting increment/decrement rates.
+
+### Application in Scoring
+
+In SRS urgency calculation:
+
+```typescript
+const userAdaptation = await this.getReviewAdaptation();
+const adaptedUrgency = baseUrgency * userAdaptation.urgencyMultiplier;
+const score = Math.min(1.0, 0.5 + adaptedUrgency * 0.45 + backlogPressure);
+```
+
+Note: multiplier affects base urgency, not backlog pressure. This preserves the global backlog escape mechanism.
+
+### Storage
+
+Use strategy state storage (per-user, per-course, per-strategy):
+
+```typescript
+const state = await this.getStrategyState<ReviewAdaptationState>();
+await this.putStrategyState({ ...state, urgencyMultiplier: newValue });
+```
+
+### Implementation Sketch
+
+```typescript
+// In srs.ts
+
+private async getReviewAdaptation(): Promise<ReviewAdaptationState> {
+  const state = await this.getStrategyState<ReviewAdaptationState>();
+  return state || {
+    urgencyMultiplier: 1.0,
+    windowStart: new Date().toISOString(),
+    windowFailures: 0,
+    windowSuccesses: 0,
+    lastUpdated: new Date().toISOString(),
+  };
+}
+
+// Called after review outcome recorded (needs integration hook)
+async updateReviewAdaptation(success: boolean): Promise<void> {
+  const state = await this.getReviewAdaptation();
+  
+  if (success) {
+    state.urgencyMultiplier = Math.max(0.5, state.urgencyMultiplier - 0.01);
+    state.windowSuccesses++;
+  } else {
+    state.urgencyMultiplier = Math.min(2.0, state.urgencyMultiplier + 0.05);
+    state.windowFailures++;
+  }
+  
+  state.lastUpdated = new Date().toISOString();
+  await this.putStrategyState(state);
+}
+```
+
+### Required Work
+
+- [ ] Add `getReviewAdaptation()` method to SRSNavigator
+- [ ] Integrate `updateReviewAdaptation()` with card response recording flow
+- [ ] Decide: Should multiplier affect backlog pressure or just base urgency?
+- [ ] Consider reset behavior on long breaks (user returning after 6 months)
+- [ ] Consider course-specific vs global multipliers
+- [ ] Add provenance entry explaining multiplier contribution
+
+### Open Questions
+
+1. **Who calls `updateReviewAdaptation`?** — Needs hook into card response recording (currently in `UserDB.recordInteraction()` or similar)
+
+2. **Rolling window reset?** — Should the window reset periodically to allow recovery from bad streaks?
+
+3. **Visibility?** — Should users see their current multiplier? Could be motivating or discouraging.
+
+4. **Initial calibration?** — New users start at 1.0. Should there be an initial calibration period with more aggressive updates?
+
+### Priority
+
+**Medium** — High value for learning outcomes, moderate implementation complexity. Requires integration with card response recording flow.
+
+---
+
+## Implementation Order
+
+1. ✅ **Backlog Pressure** (implemented) — Fixes immediate balance issue
+2. **Per-User Adaptation** — High value, moderate complexity
+3. **Orchestrator Tuning** — Lower priority if backlog pressure works well
+
+---
+
+## Related Files
+
+| File | Purpose |
+|------|---------|
+| `core/navigators/generators/srs.ts` | SRS urgency calculation, backlog pressure |
+| `core/orchestration/index.ts` | Orchestration context, deviation logic |
+| `core/types/contentNavigationStrategy.ts` | Strategy config, learnable weights |
+| `core/types/strategyState.ts` | Per-user strategy state storage |
+| `impl/couch/userDB.ts` | User interaction recording |

package/docs/todo-strategy-authoring.md

@@ -53,7 +53,6 @@ interface HierarchyConfig {
   prerequisites: {
     [tagId: string]: TagPrerequisite[];
   };
-  delegateStrategy?: string;  // default: "elo"
 }
 
 interface TagPrerequisite {
@@ -75,8 +74,7 @@ interface TagPrerequisite {
     "blends": [
       { "tag": "cvc-words", "masteryThreshold": { "minCount": 20 } }
     ]
-  },
-  "delegateStrategy": "elo"
+  }
 }
 ```
 
@@ -99,7 +97,6 @@ interface InterferenceConfig {
     minElapsedDays?: number;
   };
   defaultDecay?: number;
-  delegateStrategy?: string;
 }
 
 interface InterferenceGroup {
@@ -137,7 +134,6 @@ interface RelativePriorityConfig {
   defaultPriority?: number;
   combineMode?: 'max' | 'average' | 'min';
   priorityInfluence?: number;
-  delegateStrategy?: string;
 }
 ```
 
@@ -398,4 +394,10 @@ The NavigationStrategy editor is accessible in:
 - `packages/db/src/core/navigators/hierarchyDefinition.ts` — Config interface reference
 - `packages/db/src/core/navigators/interferenceMitigator.ts` — Config interface reference
 - `packages/db/src/core/navigators/relativePriority.ts` — Config interface reference
-- `packages/edit-ui/src/components/NavigationStrategy/` — UI components (all forms)
+- `packages/edit-ui/src/components/NavigationStrategy/` — UI components (all forms)
+
+## Related Documentation
+
+- `navigators-architecture.md` — Pipeline architecture and strategy framework
+- `future-orchestration-vision.md` — Long-term adaptive strategy vision
+- `devlog/1032-orchestrator` — Evolutionary orchestration implementation details

package/package.json

@@ -4,7 +4,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.1.23",
+  "version": "0.1.25",
   "description": "Database layer for vue-skuilder",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -48,7 +48,7 @@
   },
   "dependencies": {
     "@nilock2/pouchdb-authentication": "^1.0.2",
-    "@vue-skuilder/common": "0.1.23",
+    "@vue-skuilder/common": "0.1.25",
     "cross-fetch": "^4.1.0",
     "moment": "^2.29.4",
     "pouchdb": "^9.0.0",
@@ -62,5 +62,5 @@
     "vite": "^7.0.0",
     "vitest": "^4.0.15"
   },
-  "stableVersion": "0.1.23"
+  "stableVersion": "0.1.25"
 }

package/src/core/index.ts

@@ -4,7 +4,9 @@ export * from './interfaces';
 export * from './types/types-legacy';
 export * from './types/user';
 export * from './types/strategyState';
+export * from './types/userOutcome';
 export * from '../util/Loggable';
 export * from './util';
 export * from './navigators';
 export * from './bulkImport';
+export * from './orchestration';

package/src/core/interfaces/contentSource.ts

@@ -4,6 +4,7 @@ import { StudentClassroomDB } from '../../impl/couch/classroomDB';
 import { WeightedCard } from '../navigators';
 import { TagFilter, hasActiveFilter } from '@vue-skuilder/common';
 import { TagFilteredContentSource } from '../../study/TagFilteredContentSource';
+import { OrchestrationContext } from '../orchestration';
 
 export type StudySessionFailedItem = StudySessionFailedNewItem | StudySessionFailedReviewItem;
 
@@ -72,6 +73,12 @@ export interface StudyContentSource {
    * @returns Cards sorted by score descending
    */
   getWeightedCards(limit: number): Promise<WeightedCard[]>;
+
+  /**
+   * Get the orchestration context for this source.
+   * Used for recording learning outcomes.
+   */
+  getOrchestrationContext?(): Promise<OrchestrationContext>;
 }
 // #endregion docs_StudyContentSource