@sparkleideas/neural 3.5.2-patch.1

package/src/algorithms/q-learning.js

@@ -0,0 +1,259 @@
+/**
+ * Tabular Q-Learning
+ *
+ * Classic Q-learning algorithm with:
+ * - Epsilon-greedy exploration
+ * - State hashing for continuous states
+ * - Eligibility traces (optional)
+ * - Experience replay
+ *
+ * Suitable for smaller state spaces or discretized environments.
+ * Performance Target: <1ms per update
+ */
+/**
+ * Default Q-Learning configuration
+ */
+export const DEFAULT_QLEARNING_CONFIG = {
+    algorithm: 'q-learning',
+    learningRate: 0.1,
+    gamma: 0.99,
+    entropyCoef: 0,
+    valueLossCoef: 1,
+    maxGradNorm: 1,
+    epochs: 1,
+    miniBatchSize: 1,
+    explorationInitial: 1.0,
+    explorationFinal: 0.01,
+    explorationDecay: 10000,
+    maxStates: 10000,
+    useEligibilityTraces: false,
+    traceDecay: 0.9,
+};
+/**
+ * Q-Learning Algorithm Implementation
+ */
+export class QLearning {
+    config;
+    // Q-table
+    qTable = new Map();
+    // Exploration
+    epsilon;
+    stepCount = 0;
+    // Number of actions
+    numActions = 4;
+    // Eligibility traces
+    traces = new Map();
+    // Statistics
+    updateCount = 0;
+    avgTDError = 0;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_QLEARNING_CONFIG, ...config };
+        this.epsilon = this.config.explorationInitial;
+    }
+    /**
+     * Update Q-values from trajectory
+     */
+    update(trajectory) {
+        const startTime = performance.now();
+        if (trajectory.steps.length === 0) {
+            return { tdError: 0 };
+        }
+        let totalTDError = 0;
+        // Reset eligibility traces for new trajectory
+        if (this.config.useEligibilityTraces) {
+            this.traces.clear();
+        }
+        for (let i = 0; i < trajectory.steps.length; i++) {
+            const step = trajectory.steps[i];
+            const stateKey = this.hashState(step.stateBefore);
+            const action = this.hashAction(step.action);
+            // Get or create Q-entry
+            const qEntry = this.getOrCreateEntry(stateKey);
+            // Current Q-value
+            const currentQ = qEntry.qValues[action];
+            // Compute target Q-value
+            let targetQ;
+            if (i === trajectory.steps.length - 1) {
+                // Terminal state
+                targetQ = step.reward;
+            }
+            else {
+                const nextStateKey = this.hashState(step.stateAfter);
+                const nextEntry = this.getOrCreateEntry(nextStateKey);
+                const maxNextQ = Math.max(...nextEntry.qValues);
+                targetQ = step.reward + this.config.gamma * maxNextQ;
+            }
+            // TD error
+            const tdError = targetQ - currentQ;
+            totalTDError += Math.abs(tdError);
+            if (this.config.useEligibilityTraces) {
+                // Update eligibility trace
+                this.updateTrace(stateKey, action);
+                // Update all states with traces
+                this.updateWithTraces(tdError);
+            }
+            else {
+                // Simple Q-learning update
+                qEntry.qValues[action] += this.config.learningRate * tdError;
+                qEntry.visits++;
+                qEntry.lastUpdate = Date.now();
+            }
+        }
+        // Decay exploration
+        this.stepCount += trajectory.steps.length;
+        this.epsilon = Math.max(this.config.explorationFinal, this.config.explorationInitial - this.stepCount / this.config.explorationDecay);
+        // Prune Q-table if too large
+        if (this.qTable.size > this.config.maxStates) {
+            this.pruneQTable();
+        }
+        this.updateCount++;
+        this.avgTDError = totalTDError / trajectory.steps.length;
+        const elapsed = performance.now() - startTime;
+        if (elapsed > 1) {
+            console.warn(`Q-learning update exceeded target: ${elapsed.toFixed(2)}ms > 1ms`);
+        }
+        return { tdError: this.avgTDError };
+    }
+    /**
+     * Get action using epsilon-greedy policy
+     */
+    getAction(state, explore = true) {
+        if (explore && Math.random() < this.epsilon) {
+            return Math.floor(Math.random() * this.numActions);
+        }
+        const stateKey = this.hashState(state);
+        const entry = this.qTable.get(stateKey);
+        if (!entry) {
+            return Math.floor(Math.random() * this.numActions);
+        }
+        return this.argmax(entry.qValues);
+    }
+    /**
+     * Get Q-values for a state
+     */
+    getQValues(state) {
+        const stateKey = this.hashState(state);
+        const entry = this.qTable.get(stateKey);
+        if (!entry) {
+            return new Float32Array(this.numActions);
+        }
+        return new Float32Array(entry.qValues);
+    }
+    /**
+     * Get statistics
+     */
+    getStats() {
+        return {
+            updateCount: this.updateCount,
+            qTableSize: this.qTable.size,
+            epsilon: this.epsilon,
+            avgTDError: this.avgTDError,
+            stepCount: this.stepCount,
+        };
+    }
+    /**
+     * Reset Q-table
+     */
+    reset() {
+        this.qTable.clear();
+        this.traces.clear();
+        this.epsilon = this.config.explorationInitial;
+        this.stepCount = 0;
+        this.updateCount = 0;
+        this.avgTDError = 0;
+    }
+    // ==========================================================================
+    // Private Methods
+    // ==========================================================================
+    hashState(state) {
+        // Discretize state by binning values
+        const bins = 10;
+        const parts = [];
+        // Use first 8 dimensions for hashing
+        for (let i = 0; i < Math.min(8, state.length); i++) {
+            const normalized = (state[i] + 1) / 2; // Assume [-1, 1] range
+            const bin = Math.floor(Math.max(0, Math.min(bins - 1, normalized * bins)));
+            parts.push(bin);
+        }
+        return parts.join(',');
+    }
+    hashAction(action) {
+        let hash = 0;
+        for (let i = 0; i < action.length; i++) {
+            hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+        }
+        return hash;
+    }
+    getOrCreateEntry(stateKey) {
+        let entry = this.qTable.get(stateKey);
+        if (!entry) {
+            entry = {
+                qValues: new Float32Array(this.numActions),
+                visits: 0,
+                lastUpdate: Date.now(),
+            };
+            this.qTable.set(stateKey, entry);
+        }
+        return entry;
+    }
+    updateTrace(stateKey, action) {
+        // Decay all existing traces
+        for (const [key, trace] of this.traces) {
+            for (let a = 0; a < this.numActions; a++) {
+                trace[a] *= this.config.gamma * this.config.traceDecay;
+            }
+            // Remove near-zero traces
+            const maxTrace = Math.max(...trace);
+            if (maxTrace < 0.001) {
+                this.traces.delete(key);
+            }
+        }
+        // Set trace for current state-action
+        let trace = this.traces.get(stateKey);
+        if (!trace) {
+            trace = new Float32Array(this.numActions);
+            this.traces.set(stateKey, trace);
+        }
+        trace[action] = 1.0;
+    }
+    updateWithTraces(tdError) {
+        const lr = this.config.learningRate;
+        for (const [stateKey, trace] of this.traces) {
+            const entry = this.qTable.get(stateKey);
+            if (entry) {
+                for (let a = 0; a < this.numActions; a++) {
+                    entry.qValues[a] += lr * tdError * trace[a];
+                }
+                entry.visits++;
+                entry.lastUpdate = Date.now();
+            }
+        }
+    }
+    pruneQTable() {
+        // Remove least recently used states
+        const entries = Array.from(this.qTable.entries())
+            .sort((a, b) => a[1].lastUpdate - b[1].lastUpdate);
+        const toRemove = entries.length - Math.floor(this.config.maxStates * 0.8);
+        for (let i = 0; i < toRemove; i++) {
+            this.qTable.delete(entries[i][0]);
+        }
+    }
+    argmax(values) {
+        let maxIdx = 0;
+        let maxVal = values[0];
+        for (let i = 1; i < values.length; i++) {
+            if (values[i] > maxVal) {
+                maxVal = values[i];
+                maxIdx = i;
+            }
+        }
+        return maxIdx;
+    }
+}
+/**
+ * Factory function
+ */
+export function createQLearning(config) {
+    return new QLearning(config);
+}
+//# sourceMappingURL=q-learning.js.map

package/src/algorithms/q-learning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"q-learning.js","sourceRoot":"","sources":["q-learning.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAiBH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,SAAS,EAAE,YAAY;IACvB,YAAY,EAAE,GAAG;IACjB,KAAK,EAAE,IAAI;IACX,WAAW,EAAE,CAAC;IACd,aAAa,EAAE,CAAC;IAChB,WAAW,EAAE,CAAC;IACd,MAAM,EAAE,CAAC;IACT,aAAa,EAAE,CAAC;IAChB,kBAAkB,EAAE,GAAG;IACvB,gBAAgB,EAAE,IAAI;IACtB,gBAAgB,EAAE,KAAK;IACvB,SAAS,EAAE,KAAK;IAChB,oBAAoB,EAAE,KAAK;IAC3B,UAAU,EAAE,GAAG;CAChB,CAAC;AAWF;;GAEG;AACH,MAAM,OAAO,SAAS;IACZ,MAAM,CAAkB;IAEhC,UAAU;IACF,MAAM,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEhD,cAAc;IACN,OAAO,CAAS;IAChB,SAAS,GAAG,CAAC,CAAC;IAEtB,oBAAoB;IACZ,UAAU,GAAG,CAAC,CAAC;IAEvB,qBAAqB;IACb,MAAM,GAA8B,IAAI,GAAG,EAAE,CAAC;IAEtD,aAAa;IACL,WAAW,GAAG,CAAC,CAAC;IAChB,UAAU,GAAG,CAAC,CAAC;IAEvB,YAAY,SAAmC,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QACzD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,UAAsB;QAC3B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACxB,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,8CAA8C;QAC9C,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;YACrC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE5C,wBAAwB;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAE/C,kBAAkB;YAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAExC,yBAAyB;YACzB,IAAI,OAAe,CAAC;YACpB,IAAI,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,iBAAiB;gBACjB,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACrD,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;gBACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBAChD,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC;YACvD,CAAC;YAED,WAAW;YACX,MAAM,OAAO,GAAG,OAAO,GAAG,QAAQ,CAAC;YACnC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;gBACrC,2BAA2B;gBAC3B,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAEnC,gCAAgC;gBAChC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,2BAA2B;gBAC3B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,OAAO,CAAC;gBAC7D,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACjC,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CACrB,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAC5B,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAC/E,CAAC;QAEF,6BAA6B;QAC7B,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;QAEzD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,sCAAsC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACnF,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAmB,EAAE,UAAmB,IAAI;QACpD,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAmB;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,SAAS,CAAC,KAAmB;QACnC,qCAAqC;QACrC,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,qCAAqC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,uBAAuB;YAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3E,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAEO,UAAU,CAAC,MAAc;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,gBAAgB,CAAC,QAAgB;QACvC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG;gBACN,OAAO,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC1C,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;aACvB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,MAAc;QAClD,4BAA4B;QAC5B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YACzD,CAAC;YAED,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YACpC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;IACtB,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAEpC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACzC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9C,CAAC;gBACD,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,oCAAoC;QACpC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAErD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,MAAoB;QACjC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC;gBACvB,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiC;IAC/D,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC"}

package/src/algorithms/q-learning.ts

@@ -0,0 +1,333 @@
+/**
+ * Tabular Q-Learning
+ *
+ * Classic Q-learning algorithm with:
+ * - Epsilon-greedy exploration
+ * - State hashing for continuous states
+ * - Eligibility traces (optional)
+ * - Experience replay
+ *
+ * Suitable for smaller state spaces or discretized environments.
+ * Performance Target: <1ms per update
+ */
+
+import type { Trajectory, RLConfig } from '../types.js';
+
+/**
+ * Q-Learning configuration
+ */
+export interface QLearningConfig extends RLConfig {
+  algorithm: 'q-learning';
+  explorationInitial: number;
+  explorationFinal: number;
+  explorationDecay: number;
+  maxStates: number;
+  useEligibilityTraces: boolean;
+  traceDecay: number;
+}
+
+/**
+ * Default Q-Learning configuration
+ */
+export const DEFAULT_QLEARNING_CONFIG: QLearningConfig = {
+  algorithm: 'q-learning',
+  learningRate: 0.1,
+  gamma: 0.99,
+  entropyCoef: 0,
+  valueLossCoef: 1,
+  maxGradNorm: 1,
+  epochs: 1,
+  miniBatchSize: 1,
+  explorationInitial: 1.0,
+  explorationFinal: 0.01,
+  explorationDecay: 10000,
+  maxStates: 10000,
+  useEligibilityTraces: false,
+  traceDecay: 0.9,
+};
+
+/**
+ * Q-table entry
+ */
+interface QEntry {
+  qValues: Float32Array;
+  visits: number;
+  lastUpdate: number;
+}
+
+/**
+ * Q-Learning Algorithm Implementation
+ */
+export class QLearning {
+  private config: QLearningConfig;
+
+  // Q-table
+  private qTable: Map<string, QEntry> = new Map();
+
+  // Exploration
+  private epsilon: number;
+  private stepCount = 0;
+
+  // Number of actions
+  private numActions = 4;
+
+  // Eligibility traces
+  private traces: Map<string, Float32Array> = new Map();
+
+  // Statistics
+  private updateCount = 0;
+  private avgTDError = 0;
+
+  constructor(config: Partial<QLearningConfig> = {}) {
+    this.config = { ...DEFAULT_QLEARNING_CONFIG, ...config };
+    this.epsilon = this.config.explorationInitial;
+  }
+
+  /**
+   * Update Q-values from trajectory
+   */
+  update(trajectory: Trajectory): { tdError: number } {
+    const startTime = performance.now();
+
+    if (trajectory.steps.length === 0) {
+      return { tdError: 0 };
+    }
+
+    let totalTDError = 0;
+
+    // Reset eligibility traces for new trajectory
+    if (this.config.useEligibilityTraces) {
+      this.traces.clear();
+    }
+
+    for (let i = 0; i < trajectory.steps.length; i++) {
+      const step = trajectory.steps[i];
+      const stateKey = this.hashState(step.stateBefore);
+      const action = this.hashAction(step.action);
+
+      // Get or create Q-entry
+      const qEntry = this.getOrCreateEntry(stateKey);
+
+      // Current Q-value
+      const currentQ = qEntry.qValues[action];
+
+      // Compute target Q-value
+      let targetQ: number;
+      if (i === trajectory.steps.length - 1) {
+        // Terminal state
+        targetQ = step.reward;
+      } else {
+        const nextStateKey = this.hashState(step.stateAfter);
+        const nextEntry = this.getOrCreateEntry(nextStateKey);
+        const maxNextQ = Math.max(...nextEntry.qValues);
+        targetQ = step.reward + this.config.gamma * maxNextQ;
+      }
+
+      // TD error
+      const tdError = targetQ - currentQ;
+      totalTDError += Math.abs(tdError);
+
+      if (this.config.useEligibilityTraces) {
+        // Update eligibility trace
+        this.updateTrace(stateKey, action);
+
+        // Update all states with traces
+        this.updateWithTraces(tdError);
+      } else {
+        // Simple Q-learning update
+        qEntry.qValues[action] += this.config.learningRate * tdError;
+        qEntry.visits++;
+        qEntry.lastUpdate = Date.now();
+      }
+    }
+
+    // Decay exploration
+    this.stepCount += trajectory.steps.length;
+    this.epsilon = Math.max(
+      this.config.explorationFinal,
+      this.config.explorationInitial - this.stepCount / this.config.explorationDecay
+    );
+
+    // Prune Q-table if too large
+    if (this.qTable.size > this.config.maxStates) {
+      this.pruneQTable();
+    }
+
+    this.updateCount++;
+    this.avgTDError = totalTDError / trajectory.steps.length;
+
+    const elapsed = performance.now() - startTime;
+    if (elapsed > 1) {
+      console.warn(`Q-learning update exceeded target: ${elapsed.toFixed(2)}ms > 1ms`);
+    }
+
+    return { tdError: this.avgTDError };
+  }
+
+  /**
+   * Get action using epsilon-greedy policy
+   */
+  getAction(state: Float32Array, explore: boolean = true): number {
+    if (explore && Math.random() < this.epsilon) {
+      return Math.floor(Math.random() * this.numActions);
+    }
+
+    const stateKey = this.hashState(state);
+    const entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      return Math.floor(Math.random() * this.numActions);
+    }
+
+    return this.argmax(entry.qValues);
+  }
+
+  /**
+   * Get Q-values for a state
+   */
+  getQValues(state: Float32Array): Float32Array {
+    const stateKey = this.hashState(state);
+    const entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      return new Float32Array(this.numActions);
+    }
+
+    return new Float32Array(entry.qValues);
+  }
+
+  /**
+   * Get statistics
+   */
+  getStats(): Record<string, number> {
+    return {
+      updateCount: this.updateCount,
+      qTableSize: this.qTable.size,
+      epsilon: this.epsilon,
+      avgTDError: this.avgTDError,
+      stepCount: this.stepCount,
+    };
+  }
+
+  /**
+   * Reset Q-table
+   */
+  reset(): void {
+    this.qTable.clear();
+    this.traces.clear();
+    this.epsilon = this.config.explorationInitial;
+    this.stepCount = 0;
+    this.updateCount = 0;
+    this.avgTDError = 0;
+  }
+
+  // ==========================================================================
+  // Private Methods
+  // ==========================================================================
+
+  private hashState(state: Float32Array): string {
+    // Discretize state by binning values
+    const bins = 10;
+    const parts: number[] = [];
+
+    // Use first 8 dimensions for hashing
+    for (let i = 0; i < Math.min(8, state.length); i++) {
+      const normalized = (state[i] + 1) / 2; // Assume [-1, 1] range
+      const bin = Math.floor(Math.max(0, Math.min(bins - 1, normalized * bins)));
+      parts.push(bin);
+    }
+
+    return parts.join(',');
+  }
+
+  private hashAction(action: string): number {
+    let hash = 0;
+    for (let i = 0; i < action.length; i++) {
+      hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+    }
+    return hash;
+  }
+
+  private getOrCreateEntry(stateKey: string): QEntry {
+    let entry = this.qTable.get(stateKey);
+
+    if (!entry) {
+      entry = {
+        qValues: new Float32Array(this.numActions),
+        visits: 0,
+        lastUpdate: Date.now(),
+      };
+      this.qTable.set(stateKey, entry);
+    }
+
+    return entry;
+  }
+
+  private updateTrace(stateKey: string, action: number): void {
+    // Decay all existing traces
+    for (const [key, trace] of this.traces) {
+      for (let a = 0; a < this.numActions; a++) {
+        trace[a] *= this.config.gamma * this.config.traceDecay;
+      }
+
+      // Remove near-zero traces
+      const maxTrace = Math.max(...trace);
+      if (maxTrace < 0.001) {
+        this.traces.delete(key);
+      }
+    }
+
+    // Set trace for current state-action
+    let trace = this.traces.get(stateKey);
+    if (!trace) {
+      trace = new Float32Array(this.numActions);
+      this.traces.set(stateKey, trace);
+    }
+    trace[action] = 1.0;
+  }
+
+  private updateWithTraces(tdError: number): void {
+    const lr = this.config.learningRate;
+
+    for (const [stateKey, trace] of this.traces) {
+      const entry = this.qTable.get(stateKey);
+      if (entry) {
+        for (let a = 0; a < this.numActions; a++) {
+          entry.qValues[a] += lr * tdError * trace[a];
+        }
+        entry.visits++;
+        entry.lastUpdate = Date.now();
+      }
+    }
+  }
+
+  private pruneQTable(): void {
+    // Remove least recently used states
+    const entries = Array.from(this.qTable.entries())
+      .sort((a, b) => a[1].lastUpdate - b[1].lastUpdate);
+
+    const toRemove = entries.length - Math.floor(this.config.maxStates * 0.8);
+    for (let i = 0; i < toRemove; i++) {
+      this.qTable.delete(entries[i][0]);
+    }
+  }
+
+  private argmax(values: Float32Array): number {
+    let maxIdx = 0;
+    let maxVal = values[0];
+    for (let i = 1; i < values.length; i++) {
+      if (values[i] > maxVal) {
+        maxVal = values[i];
+        maxIdx = i;
+      }
+    }
+    return maxIdx;
+  }
+}
+
+/**
+ * Factory function
+ */
+export function createQLearning(config?: Partial<QLearningConfig>): QLearning {
+  return new QLearning(config);
+}

package/src/algorithms/sarsa.d.ts

@@ -0,0 +1,82 @@
+/**
+ * SARSA (State-Action-Reward-State-Action)
+ *
+ * On-policy TD learning algorithm with:
+ * - Epsilon-greedy exploration
+ * - State hashing for continuous states
+ * - Expected SARSA variant (optional)
+ * - Eligibility traces (SARSA-lambda)
+ *
+ * Performance Target: <1ms per update
+ */
+import type { Trajectory, RLConfig } from '../types.js';
+/**
+ * SARSA configuration
+ */
+export interface SARSAConfig extends RLConfig {
+    algorithm: 'sarsa';
+    explorationInitial: number;
+    explorationFinal: number;
+    explorationDecay: number;
+    maxStates: number;
+    useExpectedSARSA: boolean;
+    useEligibilityTraces: boolean;
+    traceDecay: number;
+}
+/**
+ * Default SARSA configuration
+ */
+export declare const DEFAULT_SARSA_CONFIG: SARSAConfig;
+/**
+ * SARSA Algorithm Implementation
+ */
+export declare class SARSAAlgorithm {
+    private config;
+    private qTable;
+    private epsilon;
+    private stepCount;
+    private numActions;
+    private traces;
+    private updateCount;
+    private avgTDError;
+    constructor(config?: Partial<SARSAConfig>);
+    /**
+     * Update Q-values from trajectory using SARSA
+     */
+    update(trajectory: Trajectory): {
+        tdError: number;
+    };
+    /**
+     * Get action using epsilon-greedy policy
+     */
+    getAction(state: Float32Array, explore?: boolean): number;
+    /**
+     * Get action probabilities for a state
+     */
+    getActionProbabilities(state: Float32Array): Float32Array;
+    /**
+     * Get Q-values for a state
+     */
+    getQValues(state: Float32Array): Float32Array;
+    /**
+     * Get statistics
+     */
+    getStats(): Record<string, number>;
+    /**
+     * Reset algorithm state
+     */
+    reset(): void;
+    private hashState;
+    private hashAction;
+    private getOrCreateEntry;
+    private expectedValue;
+    private updateTrace;
+    private updateWithTraces;
+    private pruneQTable;
+    private argmax;
+}
+/**
+ * Factory function
+ */
+export declare function createSARSA(config?: Partial<SARSAConfig>): SARSAAlgorithm;
+//# sourceMappingURL=sarsa.d.ts.map

package/src/algorithms/sarsa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarsa.d.ts","sourceRoot":"","sources":["sarsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,SAAS,EAAE,OAAO,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAgBlC,CAAC;AAWF;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAc;IAG5B,OAAO,CAAC,MAAM,CAAsC;IAGpD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAAK;IAGtB,OAAO,CAAC,UAAU,CAAK;IAGvB,OAAO,CAAC,MAAM,CAAwC;IAGtD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAK;gBAEX,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM;IAK7C;;OAEG;IACH,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE;IAwFnD;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,GAAE,OAAc,GAAG,MAAM;IAe/D;;OAEG;IACH,sBAAsB,CAAC,KAAK,EAAE,YAAY,GAAG,YAAY;IA0BzD;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,YAAY;IAW7C;;OAEG;IACH,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAUlC;;OAEG;IACH,KAAK,IAAI,IAAI;IAab,OAAO,CAAC,SAAS;IAajB,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,gBAAgB;IAexB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,WAAW;IAsBnB,OAAO,CAAC,gBAAgB;IAexB,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,MAAM;CAWf;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,cAAc,CAEzE"}