@sparkleideas/neural 3.5.2-patch.1

package/src/algorithms/sarsa.js

@@ -0,0 +1,297 @@
+/**
+ * SARSA (State-Action-Reward-State-Action)
+ *
+ * On-policy TD learning algorithm with:
+ * - Epsilon-greedy exploration
+ * - State hashing for continuous states
+ * - Expected SARSA variant (optional)
+ * - Eligibility traces (SARSA-lambda)
+ *
+ * Performance Target: <1ms per update
+ */
+/**
+ * Default SARSA configuration
+ */
+export const DEFAULT_SARSA_CONFIG = {
+    algorithm: 'sarsa',
+    learningRate: 0.1,
+    gamma: 0.99,
+    entropyCoef: 0,
+    valueLossCoef: 1,
+    maxGradNorm: 1,
+    epochs: 1,
+    miniBatchSize: 1,
+    explorationInitial: 1.0,
+    explorationFinal: 0.01,
+    explorationDecay: 10000,
+    maxStates: 10000,
+    useExpectedSARSA: false,
+    useEligibilityTraces: false,
+    traceDecay: 0.9,
+};
+/**
+ * SARSA Algorithm Implementation
+ */
+export class SARSAAlgorithm {
+    config;
+    // Q-table
+    qTable = new Map();
+    // Exploration
+    epsilon;
+    stepCount = 0;
+    // Number of actions
+    numActions = 4;
+    // Eligibility traces
+    traces = new Map();
+    // Statistics
+    updateCount = 0;
+    avgTDError = 0;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_SARSA_CONFIG, ...config };
+        this.epsilon = this.config.explorationInitial;
+    }
+    /**
+     * Update Q-values from trajectory using SARSA
+     */
+    update(trajectory) {
+        const startTime = performance.now();
+        if (trajectory.steps.length < 2) {
+            return { tdError: 0 };
+        }
+        let totalTDError = 0;
+        // Reset eligibility traces
+        if (this.config.useEligibilityTraces) {
+            this.traces.clear();
+        }
+        for (let i = 0; i < trajectory.steps.length - 1; i++) {
+            const step = trajectory.steps[i];
+            const nextStep = trajectory.steps[i + 1];
+            const stateKey = this.hashState(step.stateBefore);
+            const action = this.hashAction(step.action);
+            const nextStateKey = this.hashState(step.stateAfter);
+            const nextAction = this.hashAction(nextStep.action);
+            // Get or create entries
+            const qEntry = this.getOrCreateEntry(stateKey);
+            const nextEntry = this.getOrCreateEntry(nextStateKey);
+            // Current Q-value
+            const currentQ = qEntry.qValues[action];
+            // Compute target Q-value using SARSA update rule
+            let targetQ;
+            if (this.config.useExpectedSARSA) {
+                // Expected SARSA: use expected value under current policy
+                targetQ = step.reward + this.config.gamma * this.expectedValue(nextEntry.qValues);
+            }
+            else {
+                // Standard SARSA: use actual next action
+                targetQ = step.reward + this.config.gamma * nextEntry.qValues[nextAction];
+            }
+            // TD error
+            const tdError = targetQ - currentQ;
+            totalTDError += Math.abs(tdError);
+            if (this.config.useEligibilityTraces) {
+                this.updateTrace(stateKey, action);
+                this.updateWithTraces(tdError);
+            }
+            else {
+                qEntry.qValues[action] += this.config.learningRate * tdError;
+                qEntry.visits++;
+                qEntry.lastUpdate = Date.now();
+            }
+        }
+        // Handle terminal state
+        const lastStep = trajectory.steps[trajectory.steps.length - 1];
+        const lastStateKey = this.hashState(lastStep.stateBefore);
+        const lastAction = this.hashAction(lastStep.action);
+        const lastEntry = this.getOrCreateEntry(lastStateKey);
+        const terminalTDError = lastStep.reward - lastEntry.qValues[lastAction];
+        lastEntry.qValues[lastAction] += this.config.learningRate * terminalTDError;
+        totalTDError += Math.abs(terminalTDError);
+        // Decay exploration
+        this.stepCount += trajectory.steps.length;
+        this.epsilon = Math.max(this.config.explorationFinal, this.config.explorationInitial - this.stepCount / this.config.explorationDecay);
+        // Prune if needed
+        if (this.qTable.size > this.config.maxStates) {
+            this.pruneQTable();
+        }
+        this.updateCount++;
+        this.avgTDError = totalTDError / trajectory.steps.length;
+        const elapsed = performance.now() - startTime;
+        if (elapsed > 1) {
+            console.warn(`SARSA update exceeded target: ${elapsed.toFixed(2)}ms > 1ms`);
+        }
+        return { tdError: this.avgTDError };
+    }
+    /**
+     * Get action using epsilon-greedy policy
+     */
+    getAction(state, explore = true) {
+        if (explore && Math.random() < this.epsilon) {
+            return Math.floor(Math.random() * this.numActions);
+        }
+        const stateKey = this.hashState(state);
+        const entry = this.qTable.get(stateKey);
+        if (!entry) {
+            return Math.floor(Math.random() * this.numActions);
+        }
+        return this.argmax(entry.qValues);
+    }
+    /**
+     * Get action probabilities for a state
+     */
+    getActionProbabilities(state) {
+        const probs = new Float32Array(this.numActions);
+        const stateKey = this.hashState(state);
+        const entry = this.qTable.get(stateKey);
+        if (!entry) {
+            // Uniform distribution
+            const uniform = 1 / this.numActions;
+            for (let a = 0; a < this.numActions; a++) {
+                probs[a] = uniform;
+            }
+            return probs;
+        }
+        // Epsilon-greedy probabilities
+        const greedyAction = this.argmax(entry.qValues);
+        const exploreProb = this.epsilon / this.numActions;
+        for (let a = 0; a < this.numActions; a++) {
+            probs[a] = exploreProb;
+        }
+        probs[greedyAction] += 1 - this.epsilon;
+        return probs;
+    }
+    /**
+     * Get Q-values for a state
+     */
+    getQValues(state) {
+        const stateKey = this.hashState(state);
+        const entry = this.qTable.get(stateKey);
+        if (!entry) {
+            return new Float32Array(this.numActions);
+        }
+        return new Float32Array(entry.qValues);
+    }
+    /**
+     * Get statistics
+     */
+    getStats() {
+        return {
+            updateCount: this.updateCount,
+            qTableSize: this.qTable.size,
+            epsilon: this.epsilon,
+            avgTDError: this.avgTDError,
+            stepCount: this.stepCount,
+        };
+    }
+    /**
+     * Reset algorithm state
+     */
+    reset() {
+        this.qTable.clear();
+        this.traces.clear();
+        this.epsilon = this.config.explorationInitial;
+        this.stepCount = 0;
+        this.updateCount = 0;
+        this.avgTDError = 0;
+    }
+    // ==========================================================================
+    // Private Methods
+    // ==========================================================================
+    hashState(state) {
+        const bins = 10;
+        const parts = [];
+        for (let i = 0; i < Math.min(8, state.length); i++) {
+            const normalized = (state[i] + 1) / 2;
+            const bin = Math.floor(Math.max(0, Math.min(bins - 1, normalized * bins)));
+            parts.push(bin);
+        }
+        return parts.join(',');
+    }
+    hashAction(action) {
+        let hash = 0;
+        for (let i = 0; i < action.length; i++) {
+            hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+        }
+        return hash;
+    }
+    getOrCreateEntry(stateKey) {
+        let entry = this.qTable.get(stateKey);
+        if (!entry) {
+            entry = {
+                qValues: new Float32Array(this.numActions),
+                visits: 0,
+                lastUpdate: Date.now(),
+            };
+            this.qTable.set(stateKey, entry);
+        }
+        return entry;
+    }
+    expectedValue(qValues) {
+        // Expected value under epsilon-greedy policy
+        const greedyAction = this.argmax(qValues);
+        const exploreProb = this.epsilon / this.numActions;
+        let expected = 0;
+        for (let a = 0; a < this.numActions; a++) {
+            const prob = exploreProb + (a === greedyAction ? 1 - this.epsilon : 0);
+            expected += prob * qValues[a];
+        }
+        return expected;
+    }
+    updateTrace(stateKey, action) {
+        // Decay all traces
+        for (const [key, trace] of this.traces) {
+            for (let a = 0; a < this.numActions; a++) {
+                trace[a] *= this.config.gamma * this.config.traceDecay;
+            }
+            const maxTrace = Math.max(...trace);
+            if (maxTrace < 0.001) {
+                this.traces.delete(key);
+            }
+        }
+        // Set current trace (replacing traces for same state-action)
+        let trace = this.traces.get(stateKey);
+        if (!trace) {
+            trace = new Float32Array(this.numActions);
+            this.traces.set(stateKey, trace);
+        }
+        trace[action] = 1.0;
+    }
+    updateWithTraces(tdError) {
+        const lr = this.config.learningRate;
+        for (const [stateKey, trace] of this.traces) {
+            const entry = this.qTable.get(stateKey);
+            if (entry) {
+                for (let a = 0; a < this.numActions; a++) {
+                    entry.qValues[a] += lr * tdError * trace[a];
+                }
+                entry.visits++;
+                entry.lastUpdate = Date.now();
+            }
+        }
+    }
+    pruneQTable() {
+        const entries = Array.from(this.qTable.entries())
+            .sort((a, b) => a[1].lastUpdate - b[1].lastUpdate);
+        const toRemove = entries.length - Math.floor(this.config.maxStates * 0.8);
+        for (let i = 0; i < toRemove; i++) {
+            this.qTable.delete(entries[i][0]);
+        }
+    }
+    argmax(values) {
+        let maxIdx = 0;
+        let maxVal = values[0];
+        for (let i = 1; i < values.length; i++) {
+            if (values[i] > maxVal) {
+                maxVal = values[i];
+                maxIdx = i;
+            }
+        }
+        return maxIdx;
+    }
+}
+/**
+ * Factory function
+ */
+export function createSARSA(config) {
+    return new SARSAAlgorithm(config);
+}
+//# sourceMappingURL=sarsa.js.map

package/src/algorithms/sarsa.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarsa.js","sourceRoot":"","sources":["sarsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAkBH;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAC/C,SAAS,EAAE,OAAO;IAClB,YAAY,EAAE,GAAG;IACjB,KAAK,EAAE,IAAI;IACX,WAAW,EAAE,CAAC;IACd,aAAa,EAAE,CAAC;IAChB,WAAW,EAAE,CAAC;IACd,MAAM,EAAE,CAAC;IACT,aAAa,EAAE,CAAC;IAChB,kBAAkB,EAAE,GAAG;IACvB,gBAAgB,EAAE,IAAI;IACtB,gBAAgB,EAAE,KAAK;IACvB,SAAS,EAAE,KAAK;IAChB,gBAAgB,EAAE,KAAK;IACvB,oBAAoB,EAAE,KAAK;IAC3B,UAAU,EAAE,GAAG;CAChB,CAAC;AAWF;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAc;IAE5B,UAAU;IACF,MAAM,GAA4B,IAAI,GAAG,EAAE,CAAC;IAEpD,cAAc;IACN,OAAO,CAAS;IAChB,SAAS,GAAG,CAAC,CAAC;IAEtB,oBAAoB;IACZ,UAAU,GAAG,CAAC,CAAC;IAEvB,qBAAqB;IACb,MAAM,GAA8B,IAAI,GAAG,EAAE,CAAC;IAEtD,aAAa;IACL,WAAW,GAAG,CAAC,CAAC;IAChB,UAAU,GAAG,CAAC,CAAC;IAEvB,YAAY,SAA+B,EAAE;QAC3C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,MAAM,EAAE,CAAC;QACrD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,UAAsB;QAC3B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACxB,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,2BAA2B;QAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;YACrC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAEzC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAEpD,wBAAwB;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;YAEtD,kBAAkB;YAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAExC,iDAAiD;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBACjC,0DAA0D;gBAC1D,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACpF,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC5E,CAAC;YAED,WAAW;YACX,MAAM,OAAO,GAAG,OAAO,GAAG,QAAQ,CAAC;YACnC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;gBACrC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACnC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,OAAO,CAAC;gBAC7D,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACjC,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;QAEtD,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACxE,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,eAAe,CAAC;QAC5E,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAE1C,oBAAoB;QACpB,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CACrB,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAC5B,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAC/E,CAAC;QAEF,kBAAkB;QAClB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;QAEzD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,iCAAiC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAmB,EAAE,UAAmB,IAAI;QACpD,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,KAAmB;QACxC,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,uBAAuB;YACvB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;YACrB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,+BAA+B;QAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,KAAK,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;QACzB,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QAExC,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAmB;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,SAAS,CAAC,KAAmB;QACnC,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3E,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAEO,UAAU,CAAC,MAAc;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,gBAAgB,CAAC,QAAgB;QACvC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG;gBACN,OAAO,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC1C,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;aACvB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,aAAa,CAAC,OAAqB;QACzC,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,WAAW,GAAG,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvE,QAAQ,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,MAAc;QAClD,mBAAmB;QACnB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YACzD,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YACpC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,6DAA6D;QAC7D,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;IACtB,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAEpC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACzC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9C,CAAC;gBACD,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAErD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,MAAoB;QACjC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC;gBACvB,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,MAA6B;IACvD,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC"}

package/src/algorithms/sarsa.ts

@@ -0,0 +1,383 @@
+/**
+ * SARSA (State-Action-Reward-State-Action)
+ *
+ * On-policy TD learning algorithm with:
+ * - Epsilon-greedy exploration
+ * - State hashing for continuous states
+ * - Expected SARSA variant (optional)
+ * - Eligibility traces (SARSA-lambda)
+ *
+ * Performance Target: <1ms per update
+ */
+
+import type { Trajectory, RLConfig } from '../types.js';
+
+/**
+ * SARSA configuration
+ */
+export interface SARSAConfig extends RLConfig {
+  algorithm: 'sarsa';
+  explorationInitial: number;
+  explorationFinal: number;
+  explorationDecay: number;
+  maxStates: number;
+  useExpectedSARSA: boolean;
+  useEligibilityTraces: boolean;
+  traceDecay: number;
+}
+
+/**
+ * Default SARSA configuration
+ */
+export const DEFAULT_SARSA_CONFIG: SARSAConfig = {
+  algorithm: 'sarsa',
+  learningRate: 0.1,
+  gamma: 0.99,
+  entropyCoef: 0,
+  valueLossCoef: 1,
+  maxGradNorm: 1,
+  epochs: 1,
+  miniBatchSize: 1,
+  explorationInitial: 1.0,
+  explorationFinal: 0.01,
+  explorationDecay: 10000,
+  maxStates: 10000,
+  useExpectedSARSA: false,
+  useEligibilityTraces: false,
+  traceDecay: 0.9,
+};
+
+/**
+ * Q-table entry
+ */
+interface SARSAEntry {
+  qValues: Float32Array;
+  visits: number;
+  lastUpdate: number;
+}
+
+/**
+ * SARSA Algorithm Implementation
+ */
+export class SARSAAlgorithm {
+  private config: SARSAConfig;
+
+  // Q-table
+  private qTable: Map<string, SARSAEntry> = new Map();
+
+  // Exploration
+  private epsilon: number;
+  private stepCount = 0;
+
+  // Number of actions
+  private numActions = 4;
+
+  // Eligibility traces
+  private traces: Map<string, Float32Array> = new Map();
+
+  // Statistics
+  private updateCount = 0;
+  private avgTDError = 0;
+
+  constructor(config: Partial<SARSAConfig> = {}) {
+    this.config = { ...DEFAULT_SARSA_CONFIG, ...config };
+    this.epsilon = this.config.explorationInitial;
+  }
+
+  /**
+   * Update Q-values from trajectory using SARSA
+   */
+  update(trajectory: Trajectory): { tdError: number } {
+    const startTime = performance.now();
+
+    if (trajectory.steps.length < 2) {
+      return { tdError: 0 };
+    }
+
+    let totalTDError = 0;
+
+    // Reset eligibility traces
+    if (this.config.useEligibilityTraces) {
+      this.traces.clear();
+    }
+
+    for (let i = 0; i < trajectory.steps.length - 1; i++) {
+      const step = trajectory.steps[i];
+      const nextStep = trajectory.steps[i + 1];
+
+      const stateKey = this.hashState(step.stateBefore);
+      const action = this.hashAction(step.action);
+      const nextStateKey = this.hashState(step.stateAfter);
+      const nextAction = this.hashAction(nextStep.action);
+
+      // Get or create entries
+      const qEntry = this.getOrCreateEntry(stateKey);
+      const nextEntry = this.getOrCreateEntry(nextStateKey);
+
+      // Current Q-value
+      const currentQ = qEntry.qValues[action];
+
+      // Compute target Q-value using SARSA update rule
+      let targetQ: number;
+
+      if (this.config.useExpectedSARSA) {
+        // Expected SARSA: use expected value under current policy
+        targetQ = step.reward + this.config.gamma * this.expectedValue(nextEntry.qValues);
+      } else {
+        // Standard SARSA: use actual next action
+        targetQ = step.reward + this.config.gamma * nextEntry.qValues[nextAction];
+      }
+
+      // TD error
+      const tdError = targetQ - currentQ;
+      totalTDError += Math.abs(tdError);
+
+      if (this.config.useEligibilityTraces) {
+        this.updateTrace(stateKey, action);
+        this.updateWithTraces(tdError);
+      } else {
+        qEntry.qValues[action] += this.config.learningRate * tdError;
+        qEntry.visits++;
+        qEntry.lastUpdate = Date.now();
+      }
+    }
+
+    // Handle terminal state
+    const lastStep = trajectory.steps[trajectory.steps.length - 1];
+    const lastStateKey = this.hashState(lastStep.stateBefore);
+    const lastAction = this.hashAction(lastStep.action);
+    const lastEntry = this.getOrCreateEntry(lastStateKey);
+
+    const terminalTDError = lastStep.reward - lastEntry.qValues[lastAction];
+    lastEntry.qValues[lastAction] += this.config.learningRate * terminalTDError;
+    totalTDError += Math.abs(terminalTDError);
+
+    // Decay exploration
+    this.stepCount += trajectory.steps.length;
+    this.epsilon = Math.max(
+      this.config.explorationFinal,
+      this.config.explorationInitial - this.stepCount / this.config.explorationDecay
+    );
+
+    // Prune if needed
+    if (this.qTable.size > this.config.maxStates) {
+      this.pruneQTable();
+    }
+
+    this.updateCount++;
+    this.avgTDError = totalTDError / trajectory.steps.length;
+
+    const elapsed = performance.now() - startTime;
+    if (elapsed > 1) {
+      console.warn(`SARSA update exceeded target: ${elapsed.toFixed(2)}ms > 1ms`);
+    }
+
+    return { tdError: this.avgTDError };
+  }
+
+  /**
+   * Get action using epsilon-greedy policy
+   */
+  getAction(state: Float32Array, explore: boolean = true): number {
+    if (explore && Math.random() < this.epsilon) {
+      return Math.floor(Math.random() * this.numActions);
+    }
+
+    const stateKey = this.hashState(state);
+    const entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      return Math.floor(Math.random() * this.numActions);
+    }
+
+    return this.argmax(entry.qValues);
+  }
+
+  /**
+   * Get action probabilities for a state
+   */
+  getActionProbabilities(state: Float32Array): Float32Array {
+    const probs = new Float32Array(this.numActions);
+    const stateKey = this.hashState(state);
+    const entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      // Uniform distribution
+      const uniform = 1 / this.numActions;
+      for (let a = 0; a < this.numActions; a++) {
+        probs[a] = uniform;
+      }
+      return probs;
+    }
+
+    // Epsilon-greedy probabilities
+    const greedyAction = this.argmax(entry.qValues);
+    const exploreProb = this.epsilon / this.numActions;
+
+    for (let a = 0; a < this.numActions; a++) {
+      probs[a] = exploreProb;
+    }
+    probs[greedyAction] += 1 - this.epsilon;
+
+    return probs;
+  }
+
+  /**
+   * Get Q-values for a state
+   */
+  getQValues(state: Float32Array): Float32Array {
+    const stateKey = this.hashState(state);
+    const entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      return new Float32Array(this.numActions);
+    }
+
+    return new Float32Array(entry.qValues);
+  }
+
+  /**
+   * Get statistics
+   */
+  getStats(): Record<string, number> {
+    return {
+      updateCount: this.updateCount,
+      qTableSize: this.qTable.size,
+      epsilon: this.epsilon,
+      avgTDError: this.avgTDError,
+      stepCount: this.stepCount,
+    };
+  }
+
+  /**
+   * Reset algorithm state
+   */
+  reset(): void {
+    this.qTable.clear();
+    this.traces.clear();
+    this.epsilon = this.config.explorationInitial;
+    this.stepCount = 0;
+    this.updateCount = 0;
+    this.avgTDError = 0;
+  }
+
+  // ==========================================================================
+  // Private Methods
+  // ==========================================================================
+
+  private hashState(state: Float32Array): string {
+    const bins = 10;
+    const parts: number[] = [];
+
+    for (let i = 0; i < Math.min(8, state.length); i++) {
+      const normalized = (state[i] + 1) / 2;
+      const bin = Math.floor(Math.max(0, Math.min(bins - 1, normalized * bins)));
+      parts.push(bin);
+    }
+
+    return parts.join(',');
+  }
+
+  private hashAction(action: string): number {
+    let hash = 0;
+    for (let i = 0; i < action.length; i++) {
+      hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+    }
+    return hash;
+  }
+
+  private getOrCreateEntry(stateKey: string): SARSAEntry {
+    let entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      entry = {
+        qValues: new Float32Array(this.numActions),
+        visits: 0,
+        lastUpdate: Date.now(),
+      };
+      this.qTable.set(stateKey, entry);
+    }
+
+    return entry;
+  }
+
+  private expectedValue(qValues: Float32Array): number {
+    // Expected value under epsilon-greedy policy
+    const greedyAction = this.argmax(qValues);
+    const exploreProb = this.epsilon / this.numActions;
+
+    let expected = 0;
+    for (let a = 0; a < this.numActions; a++) {
+      const prob = exploreProb + (a === greedyAction ? 1 - this.epsilon : 0);
+      expected += prob * qValues[a];
+    }
+
+    return expected;
+  }
+
+  private updateTrace(stateKey: string, action: number): void {
+    // Decay all traces
+    for (const [key, trace] of this.traces) {
+      for (let a = 0; a < this.numActions; a++) {
+        trace[a] *= this.config.gamma * this.config.traceDecay;
+      }
+
+      const maxTrace = Math.max(...trace);
+      if (maxTrace < 0.001) {
+        this.traces.delete(key);
+      }
+    }
+
+    // Set current trace (replacing traces for same state-action)
+    let trace = this.traces.get(stateKey);
+    if (!trace) {
+      trace = new Float32Array(this.numActions);
+      this.traces.set(stateKey, trace);
+    }
+    trace[action] = 1.0;
+  }
+
+  private updateWithTraces(tdError: number): void {
+    const lr = this.config.learningRate;
+
+    for (const [stateKey, trace] of this.traces) {
+      const entry = this.qTable.get(stateKey);
+      if (entry) {
+        for (let a = 0; a < this.numActions; a++) {
+          entry.qValues[a] += lr * tdError * trace[a];
+        }
+        entry.visits++;
+        entry.lastUpdate = Date.now();
+      }
+    }
+  }
+
+  private pruneQTable(): void {
+    const entries = Array.from(this.qTable.entries())
+      .sort((a, b) => a[1].lastUpdate - b[1].lastUpdate);
+
+    const toRemove = entries.length - Math.floor(this.config.maxStates * 0.8);
+    for (let i = 0; i < toRemove; i++) {
+      this.qTable.delete(entries[i][0]);
+    }
+  }
+
+  private argmax(values: Float32Array): number {
+    let maxIdx = 0;
+    let maxVal = values[0];
+    for (let i = 1; i < values.length; i++) {
+      if (values[i] > maxVal) {
+        maxVal = values[i];
+        maxIdx = i;
+      }
+    }
+    return maxIdx;
+  }
+}
+
+/**
+ * Factory function
+ */
+export function createSARSA(config?: Partial<SARSAConfig>): SARSAAlgorithm {
+  return new SARSAAlgorithm(config);
+}