@game_ryo/lsji 0.1.0

package/src/core/qlearning.js

@@ -0,0 +1,197 @@
+/**
+ * Q-Learning Engine (Temporal Difference Learning)
+ * 
+ * Migrated from the original Cloudflare Workers implementation.
+ * Implements tabular Q-learning with epsilon-greedy exploration.
+ */
+
+import { StateEncoder } from './env.js';
+
+/**
+ * Q-Learning configuration
+ * @typedef {Object} QLearningConfig
+ * @property {number} [alpha=0.1] - Learning rate (0.0 to 1.0)
+ * @property {number} [gamma=0.9] - Discount factor (0.0 to 1.0)
+ * @property {number} [epsilon=0.1] - Exploration rate (0.0 to 1.0)
+ * @property {Storage} storage - Storage backend for Q-table persistence
+ */
+
+/**
+ * Q-Learning Engine for Tabular Reinforcement Learning
+ */
+export class QLearning {
+  /**
+   * @param {QLearningConfig} config
+   */
+  constructor({ alpha = 0.1, gamma = 0.9, epsilon = 0.1, storage } = {}) {
+    if (!storage) {
+      throw new Error('Storage is required for QLearning');
+    }
+    
+    this.alpha = Math.max(0, Math.min(1, alpha));
+    this.gamma = Math.max(0, Math.min(1, gamma));
+    this.epsilon = Math.max(0, Math.min(1, epsilon));
+    this.storage = storage;
+    
+    // In-memory Q-table cache for performance
+    this.qTable = new Map();
+    this.initialized = false;
+  }
+
+  /**
+   * Initialize Q-table from storage
+   * @returns {Promise<void>}
+   */
+  async initialize() {
+    if (this.initialized) return;
+    
+    const records = await this.storage.getQTable();
+    for (const record of records) {
+      const key = `${record.state}:${record.action}`;
+      this.qTable.set(key, record.q_value);
+    }
+    this.initialized = true;
+  }
+
+  /**
+   * Get Q-value for state-action pair
+   * @param {string} state - State key
+   * @param {number} action - Action
+   * @returns {number} Q-value (0 if unseen)
+   */
+  getQValue(state, action) {
+    return this.qTable.get(`${state}:${action}`) ?? 0;
+  }
+
+  /**
+   * Set Q-value for state-action pair
+   * @param {string} state - State key
+   * @param {number} action - Action
+   * @param {number} value - Q-value
+   * @returns {Promise<void>}
+   */
+  async setQValue(state, action, value) {
+    const key = `${state}:${action}`;
+    this.qTable.set(key, value);
+    await this.storage.updateQ(state, action, value);
+  }
+
+  /**
+   * Select action using epsilon-greedy policy
+   * @param {string} state - Current state
+   * @param {number} actionSize - Number of possible actions
+   * @returns {Promise<number>} Selected action
+   */
+  async act(state, actionSize) {
+    await this.initialize();
+    
+    // Exploration: random action
+    if (Math.random() < this.epsilon) {
+      return Math.floor(Math.random() * actionSize);
+    }
+    
+    // Exploitation: best known action
+    let bestAction = 0;
+    let bestValue = -Infinity;
+    
+    for (let action = 0; action < actionSize; action++) {
+      const value = this.getQValue(state, action);
+      if (value > bestValue || (value === bestValue && Math.random() < 0.5)) {
+        bestValue = value;
+        bestAction = action;
+      }
+    }
+    
+    return bestAction;
+  }
+
+  /**
+   * Update Q-value using TD learning rule
+   * Q(s,a) <- Q(s,a) + alpha * (reward + gamma * max_a' Q(s',a') - Q(s,a))
+   * 
+   * @param {string} state - Current state
+   * @param {number} action - Action taken
+   * @param {number} reward - Reward received
+   * @param {string} nextState - Next state
+   * @param {number} nextActionSize - Number of actions in next state
+   * @returns {Promise<number>} Updated Q-value
+   */
+  async learn(state, action, reward, nextState, nextActionSize) {
+    await this.initialize();
+    
+    const currentQ = this.getQValue(state, action);
+    
+    // Find max Q-value for next state
+    let maxNextQ = 0;
+    for (let a = 0; a < nextActionSize; a++) {
+      const q = this.getQValue(nextState, a);
+      if (q > maxNextQ) maxNextQ = q;
+    }
+    
+    // TD update: Q(s,a) = Q(s,a) + alpha * (reward + gamma * max Q(s',a') - Q(s,a))
+    const target = reward + this.gamma * maxNextQ;
+    const newQ = currentQ + this.alpha * (target - currentQ);
+    
+    await this.setQValue(state, action, newQ);
+    return newQ;
+  }
+
+  /**
+   * Simple Q-value update (original worker.js style)
+   * Q(s,a) <- Q(s,a) + alpha * (reward - Q(s,a))
+   * Used for terminal states or simplified updates
+   * 
+   * @param {string} state - State
+   * @param {number} action - Action
+   * @param {number} reward - Reward
+   * @returns {Promise<number>} Updated Q-value
+   */
+  async learnSimple(state, action, reward) {
+    await this.initialize();
+    
+    const currentQ = this.getQValue(state, action);
+    // Original formula: oldQ + 0.1 * (reward - oldQ)
+    const newQ = currentQ + this.alpha * (reward - currentQ);
+    
+    await this.setQValue(state, action, newQ);
+    return newQ;
+  }
+
+  /**
+   * Reset Q-table (clear all learned values)
+   * @returns {Promise<void>}
+   */
+  async reset() {
+    this.qTable.clear();
+    // Note: storage reset would require a new method
+    this.initialized = false;
+  }
+
+  /**
+   * Get all Q-values for a state
+   * @param {string} state - State key
+   * @param {number} actionSize - Number of actions
+   * @returns {Object<string, number>} Action -> Q-value mapping
+   */
+  getStateValues(state, actionSize) {
+    const values = {};
+    for (let action = 0; action < actionSize; action++) {
+      values[action] = this.getQValue(state, action);
+    }
+    return values;
+  }
+
+  /**
+   * Get entire Q-table (for inspection/debugging)
+   * @returns {Promise<Array<{state: string, action: number, q_value: number}>>}
+   */
+  async getFullQTable() {
+    await this.initialize();
+    const results = [];
+    for (const [key, value] of this.qTable) {
+      const [state, action] = key.split(':');
+      results.push({ state, action: parseInt(action, 10), q_value: value });
+    }
+    return results.sort((a, b) => a.state.localeCompare(b.state) || a.action - b.action);
+  }
+}

package/src/envs/rps.js

@@ -0,0 +1,168 @@
+/**
+ * Rock-Paper-Scissors Environment
+ * 
+ * Implements the Env interface for the classic RPS game.
+ * Migrated from the original Cloudflare Workers implementation.
+ * 
+ * State: The opponent's last action (0=Rock, 1=Scissors, 2=Paper)
+ * Actions: 0=Rock, 1=Scissors, 2=Paper
+ * Rewards: Win=1, Lose=-1, Draw=0
+ */
+
+import { Env } from '../core/env.js';
+
+const HAND_NAMES = ['Rock', 'Scissors', 'Paper'];
+
+/**
+ * Opponent strategy type
+ * @typedef {'random'|'always_rock'|'counter'|'sequential'} OpponentStrategy
+ */
+
+/**
+ * RockPaperScissorsEnv - Classic RPS environment
+ */
+export class RockPaperScissorsEnv extends Env {
+  /**
+   * @param {Object} options
+   * @param {OpponentStrategy} [options.opponent='random'] - Opponent strategy
+   */
+  constructor({ opponent = 'random' } = {}) {
+    super();
+    this.opponent = opponent;
+    this.lastOpponentAction = 0;
+    this.episodeCount = 0;
+  }
+
+  /**
+   * Get current state (opponent's last action)
+   * @returns {string} State key
+   */
+  getState() {
+    return String(this.lastOpponentAction);
+  }
+
+  /**
+   * Execute action against opponent
+   * @param {number} action - Agent's action (0, 1, 2)
+   * @returns {Promise<StepResult>} Step result
+   */
+  async step(action) {
+    // Determine opponent action based on strategy
+    let opponentAction;
+    switch (this.opponent) {
+      case 'always_rock':
+        opponentAction = 0;
+        break;
+      case 'counter':
+        // Counter agent's previous action
+        opponentAction = this.episodeCount === 0 ? 0 : (action + 2) % 3;
+        break;
+      case 'sequential':
+        opponentAction = this.episodeCount % 3;
+        break;
+      case 'random':
+      default:
+        opponentAction = Math.floor(Math.random() * 3);
+    }
+
+    // Calculate reward from agent's perspective
+    // judge = (agent - opponent + 3) % 3
+    // 2 = agent wins, 1 = agent loses, 0 = draw
+    const judge = (action - opponentAction + 3) % 3;
+    const reward = judge === 2 ? 1 : judge === 1 ? -1 : 0;
+
+    // Update state for next step
+    this.lastOpponentAction = opponentAction;
+    this.episodeCount++;
+
+    return {
+      state: String(opponentAction),
+      reward,
+      done: false,
+      info: { opponentAction, judge }
+    };
+  }
+
+  /**
+   * Number of possible actions (Rock, Paper, Scissors)
+   * @returns {number} 3
+   */
+  actionSize() {
+    return 3;
+  }
+
+  /**
+   * Reset environment to initial state
+   * @returns {Promise<string>} Initial state
+   */
+  async reset() {
+    this.lastOpponentAction = 0;
+    this.episodeCount = 0;
+    return '0';
+  }
+
+  /**
+   * Render current state
+   * @returns {string} Human-readable representation
+   */
+  render() {
+    return `RPS Env | Last opponent: ${HAND_NAMES[this.lastOpponentAction]} | Episodes: ${this.episodeCount}`;
+  }
+
+  /**
+   * Get human-readable hand name
+   * @param {number} hand - Hand index (0, 1, 2)
+   * @returns {string} Hand name
+   */
+  static getHandName(hand) {
+    return HAND_NAMES[hand] || 'Unknown';
+  }
+
+  /**
+   * Calculate outcome between two hands
+   * @param {number} agentHand - Agent's hand
+   * @param {number} opponentHand - Opponent's hand
+   * @returns {{judge: number, reward: number, outcome: string}} Result
+   */
+  static calculateOutcome(agentHand, opponentHand) {
+    const judge = (agentHand - opponentHand + 3) % 3;
+    const reward = judge === 2 ? 1 : judge === 1 ? -1 : 0;
+    const outcome = judge === 2 ? 'WIN' : judge === 1 ? 'LOSE' : 'DRAW';
+    return { judge, reward, outcome };
+  }
+}
+
+/**
+ * Training pattern definitions (matching original worker.js)
+ * @readonly
+ * @enum {number}
+ */
+export const TrainingPattern = {
+  RANDOM: 0,        // Random actions
+  ALWAYS_ROCK: 1,   // Always play Rock
+  COUNTER: 2,       // Counter previous action
+  SEQUENTIAL: 3     // Sequential 0,1,2,0,1,2...
+};
+
+/**
+ * Get action for training pattern
+ * @param {number} pattern - Training pattern (0-3)
+ * @param {number} episode - Current episode number
+ * @param {number} lastAction - Last action taken
+ * @returns {number} Action to take
+ */
+export function getTrainingAction(pattern, episode, lastAction = 0) {
+  switch (pattern) {
+    case TrainingPattern.ALWAYS_ROCK:
+      return 0;
+    case TrainingPattern.COUNTER:
+      return episode === 0 ? 0 : (lastAction + 2) % 3;
+    case TrainingPattern.SEQUENTIAL:
+      return episode % 3;
+    case TrainingPattern.RANDOM:
+    default:
+      return Math.floor(Math.random() * 3);
+  }
+}
+
+export default RockPaperScissorsEnv;

package/src/index.js

@@ -0,0 +1,22 @@
+/**
+ * LSJI - Main Entry Point
+ * 
+ * Public API exports for the reinforcement learning framework.
+ */
+
+// Core
+export { Env, StateEncoder } from './core/env.js';
+export { QLearning } from './core/qlearning.js';
+export { Agent } from './core/agent.js';
+
+// Storage
+export { Storage, createStorage } from './storage/index.js';
+export { SqliteStorage } from './storage/sqlite.js';
+export { BetterSqliteStorage } from './storage/better-sqlite.js';
+export { MemoryStorage } from './storage/memory.js';
+
+// Environments
+export { RockPaperScissorsEnv, TrainingPattern, getTrainingAction } from './envs/rps.js';
+
+// Version
+export const VERSION = '0.1.0';

package/src/storage/better-sqlite.js

@@ -0,0 +1,133 @@
+/**
+ * Better-SQLite3 Storage Implementation
+ * 
+ * High-performance synchronous SQLite using better-sqlite3 package.
+ * Install with: npm install better-sqlite3
+ */
+
+import { Storage } from './index.js';
+
+/**
+ * BetterSqliteStorage - Uses better-sqlite3
+ * Falls back gracefully if package not installed
+ */
+export class BetterSqliteStorage extends Storage {
+  /**
+   * @param {string} path - Database file path
+   */
+  constructor(path = './lsji.db') {
+    super();
+    this.path = path;
+    this.db = null;
+    this.Database = null;
+  }
+
+  async initialize() {
+    try {
+      // Dynamic import to avoid hard dependency
+      const module = await import('better-sqlite3');
+      this.Database = module.default;
+    } catch (e) {
+      throw new Error('better-sqlite3 not installed. Run: npm install better-sqlite3');
+    }
+    
+    this.db = new this.Database(this.path);
+    
+    // Create tables (same schema as sqlite.js)
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS settings (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      
+      CREATE TABLE IF NOT EXISTS battle_history (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        mode TEXT NOT NULL CHECK (mode IN ('train', 'test')),
+        hand_a INTEGER NOT NULL,
+        hand_b INTEGER NOT NULL,
+        reward INTEGER NOT NULL,
+        created_at TEXT NOT NULL
+      );
+      
+      CREATE TABLE IF NOT EXISTS q_table (
+        state TEXT NOT NULL,
+        action INTEGER NOT NULL,
+        q_value REAL NOT NULL DEFAULT 0,
+        PRIMARY KEY (state, action)
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_battle_history_date 
+        ON battle_history(created_at);
+      CREATE INDEX IF NOT EXISTS idx_battle_history_mode 
+        ON battle_history(mode);
+    `);
+    
+    // Initialize default settings
+    const stmt = this.db.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)');
+    stmt.run('is_active', '1');
+  }
+
+  async close() {
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  async getSetting(key) {
+    const stmt = this.db.prepare('SELECT key, value FROM settings WHERE key = ?');
+    const row = stmt.get(key);
+    return row ? { key: row.key, value: row.value } : null;
+  }
+
+  async setSetting(key, value) {
+    const stmt = this.db.prepare(`
+      INSERT INTO settings (key, value) VALUES (?, ?) 
+      ON CONFLICT(key) DO UPDATE SET value = ?
+    `);
+    stmt.run(key, String(value), String(value));
+  }
+
+  async getQTable() {
+    const stmt = this.db.prepare('SELECT state, action, q_value FROM q_table ORDER BY state, action');
+    return stmt.all();
+  }
+
+  async updateQ(state, action, qValue) {
+    const stmt = this.db.prepare(`
+      INSERT INTO q_table (state, action, q_value) VALUES (?, ?, ?)
+      ON CONFLICT(state, action) DO UPDATE SET q_value = ?
+    `);
+    stmt.run(state, action, qValue, qValue);
+  }
+
+  async addBattle(record) {
+    const stmt = this.db.prepare(`
+      INSERT INTO battle_history (mode, hand_a, hand_b, reward, created_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    stmt.run(record.mode, record.handA, record.handB, record.reward, record.createdAt);
+  }
+
+  async getTodayBattleCount() {
+    const today = new Date().toISOString().split('T')[0];
+    const stmt = this.db.prepare(`
+      SELECT COUNT(*) as count FROM battle_history 
+      WHERE date(created_at) = date(?)
+    `);
+    const row = stmt.get(today);
+    return row ? row.count : 0;
+  }
+
+  async getPerformanceStats() {
+    const stmt = this.db.prepare(`
+      SELECT 
+        mode,
+        COUNT(*) as total,
+        ROUND(AVG(CASE WHEN reward > 0 THEN 1.0 ELSE 0.0 END) * 100, 1) as win_rate
+      FROM battle_history 
+      GROUP BY mode
+    `);
+    return stmt.all();
+  }
+}

package/src/storage/index.js

@@ -0,0 +1,146 @@
+/**
+ * Storage Abstraction Interface
+ * 
+ * Defines the contract for all storage backends.
+ * Implementations: SqliteStorage, BetterSqliteStorage, MemoryStorage
+ */
+
+/**
+ * Battle record
+ * @typedef {Object} BattleRecord
+ * @property {'train'|'test'} mode - Battle mode
+ * @property {number} handA - AI's action
+ * @property {number} handB - Opponent's action
+ * @property {number} reward - Reward (-1, 0, 1)
+ * @property {string} createdAt - ISO timestamp
+ */
+
+/**
+ * Q-table record
+ * @typedef {Object} QTableRecord
+ * @property {string} state - State key
+ * @property {number} action - Action
+ * @property {number} qValue - Q-value
+ */
+
+/**
+ * Performance stat by mode
+ * @typedef {Object} PerformanceStat
+ * @property {string} mode - 'train' or 'test'
+ * @property {number} total - Total battles
+ * @property {number} winRate - Win rate percentage
+ */
+
+/**
+ * Abstract Storage Interface
+ * All storage backends must implement these methods.
+ */
+export class Storage {
+  /**
+   * Initialize the storage (create tables, connections, etc.)
+   * @returns {Promise<void>}
+   */
+  async initialize() {
+    throw new Error('initialize() must be implemented');
+  }
+
+  /**
+   * Close the storage connection
+   * @returns {Promise<void>}
+   */
+  async close() {
+    throw new Error('close() must be implemented');
+  }
+
+  /**
+   * Get a setting value
+   * @param {string} key - Setting key
+   * @returns {Promise<{key: string, value: string}|null>}
+   */
+  async getSetting(key) {
+    throw new Error('getSetting() must be implemented');
+  }
+
+  /**
+   * Set a setting value
+   * @param {string} key - Setting key
+   * @param {string|number} value - Setting value
+   * @returns {Promise<void>}
+   */
+  async setSetting(key, value) {
+    throw new Error('setSetting() must be implemented');
+  }
+
+  /**
+   * Get all Q-table records
+   * @returns {Promise<Array<QTableRecord>>}
+   */
+  async getQTable() {
+    throw new Error('getQTable() must be implemented');
+  }
+
+  /**
+   * Update Q-value for state-action pair
+   * @param {string} state - State key
+   * @param {number} action - Action
+   * @param {number} qValue - Q-value
+   * @returns {Promise<void>}
+   */
+  async updateQ(state, action, qValue) {
+    throw new Error('updateQ() must be implemented');
+  }
+
+  /**
+   * Add a battle record
+   * @param {BattleRecord} record - Battle record
+   * @returns {Promise<void>}
+   */
+  async addBattle(record) {
+    throw new Error('addBattle() must be implemented');
+  }
+
+  /**
+   * Get today's battle count
+   * @returns {Promise<number>}
+   */
+  async getTodayBattleCount() {
+    throw new Error('getTodayBattleCount() must be implemented');
+  }
+
+  /**
+   * Get performance statistics by mode
+   * @returns {Promise<Array<PerformanceStat>>}
+   */
+  async getPerformanceStats() {
+    throw new Error('getPerformanceStats() must be implemented');
+  }
+}
+
+/**
+ * Create storage instance by type
+ * @param {'sqlite'|'better-sqlite'|'memory'} type - Storage type
+ * @param {Object} options - Storage options
+ * @returns {Promise<Storage>} Initialized storage instance
+ */
+export async function createStorage(type, options = {}) {
+  let storage;
+  
+  switch (type) {
+    case 'sqlite':
+      const { SqliteStorage } = await import('./sqlite.js');
+      storage = new SqliteStorage(options.path || './lsji.db');
+      break;
+    case 'better-sqlite':
+      const { BetterSqliteStorage } = await import('./better-sqlite.js');
+      storage = new BetterSqliteStorage(options.path || './lsji.db');
+      break;
+    case 'memory':
+    default:
+      const { MemoryStorage } = await import('./memory.js');
+      storage = new MemoryStorage();
+      break;
+  }
+  
+  await storage.initialize();
+  return storage;
+}