@game_ryo/lsji 0.1.0

package/src/cli.js

@@ -0,0 +1,232 @@
+/**
+ * LSJI CLI - Command Line Interface
+ * 
+ * Provides train/play/status/start/stop commands.
+ * Migrated from the original Cloudflare Workers HTTP endpoints.
+ */
+
+import { Agent } from './core/agent.js';
+import { QLearning } from './core/qlearning.js';
+import { createStorage } from './storage/index.js';
+import { RockPaperScissorsEnv, TrainingPattern, getTrainingAction } from './envs/rps.js';
+
+// Hand names for display
+const HAND_NAMES = ['Rock', 'Scissors', 'Paper'];
+
+/**
+ * Parse command line arguments
+ * @returns {Object} Parsed arguments
+ */
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const command = args[0];
+  const options = {};
+  
+  for (let i = 1; i < args.length; i++) {
+    const arg = args[i];
+    if (arg.startsWith('--')) {
+      const key = arg.slice(2);
+      const nextArg = args[i + 1];
+      if (nextArg && !nextArg.startsWith('--')) {
+        options[key] = nextArg;
+        i++;
+      } else {
+        options[key] = true;
+      }
+    }
+  }
+  
+  return { command, options };
+}
+
+/**
+ * Format output as JSON or table
+ * @param {Object} data - Data to output
+ * @param {boolean} json - Whether to output JSON
+ */
+function output(data, json = false) {
+  if (json) {
+    console.log(JSON.stringify(data, null, 2));
+  } else if (Array.isArray(data)) {
+    console.table(data);
+  } else {
+    console.log(data);
+  }
+}
+
+/**
+ * Main CLI entry point
+ */
+export async function main() {
+  const { command, options } = parseArgs();
+  const jsonOutput = options.json === true || options.json === '';
+  
+  // Determine storage type
+  const storageType = options.storage || process.env.LSJI_STORAGE || 'sqlite';
+  const dbPath = options.dbPath || process.env.LSJI_DB_PATH || './lsji.db';
+  
+  let storage;
+  try {
+    storage = await createStorage(storageType, { path: dbPath });
+  } catch (e) {
+    console.error(`Failed to initialize storage (${storageType}):`, e.message);
+    if (storageType !== 'memory') {
+      console.error('Falling back to memory storage...');
+      storage = await createStorage('memory');
+    } else {
+      process.exit(1);
+    }
+  }
+  
+  // Create Q-learning engine
+  const qlearning = new QLearning({
+    alpha: parseFloat(options.alpha) || 0.1,
+    gamma: parseFloat(options.gamma) || 0.9,
+    epsilon: parseFloat(options.epsilon) || 0.1,
+    storage
+  });
+  
+  // Create environment (Rock-Paper-Scissors)
+  const opponent = options.opponent || 'random';
+  const env = new RockPaperScissorsEnv({ opponent });
+  
+  // Create agent
+  const agent = new Agent({ qlearning, storage, env });
+  
+  try {
+    switch (command) {
+      case 'train': {
+        const episodes = parseInt(options.episodes) || 200;
+        const pattern = parseInt(options.pattern) || 0;
+        const batchSize = parseInt(options.batchSize) || 200;
+        
+        // Create action selector based on pattern
+        let actionSelector = null;
+        if (pattern > 0) {
+          actionSelector = (episode, lastAction) => getTrainingAction(pattern, episode, lastAction);
+        }
+        
+        console.log(`Training: ${episodes} episodes, pattern=${pattern}, opponent=${opponent}...`);
+        const result = await agent.train({ episodes, actionSelector, batchSize });
+        console.log('Training complete:', result);
+        break;
+      }
+      
+      case 'play': {
+        const hand = parseInt(options.hand);
+        if (isNaN(hand) || hand < 0 || hand > 2) {
+          console.error('Error: --hand must be 0 (Rock), 1 (Scissors), or 2 (Paper)');
+          process.exit(1);
+        }
+        
+        // For play, we need to pass the user's hand to the environment
+        // The RPS env uses its own opponent strategy, so we'll use a custom approach
+        const state = await env.getState() || '0';
+        const actionSize = env.actionSize();
+        const aiHand = await qlearning.act(state, actionSize);
+        
+        // Calculate outcome manually for display
+        const { judge, reward, outcome } = RockPaperScissorsEnv.calculateOutcome(aiHand, hand);
+        
+        // Step the environment with AI's action (to update state and Q-table)
+        const result = await env.step(aiHand);
+        
+        // Update Q-table with actual result
+        await qlearning.learnSimple(state, aiHand, result.reward);
+        
+        // Record battle
+        await storage.addBattle({
+          mode: 'test',
+          handA: aiHand,
+          handB: hand,
+          reward: result.reward,
+          createdAt: new Date().toISOString()
+        });
+        
+        const playResult = {
+          aiHand,
+          userHand: hand,
+          outcome,
+          aiHandName: RockPaperScissorsEnv.getHandName(aiHand),
+          userHandName: RockPaperScissorsEnv.getHandName(hand)
+        };
+        
+        if (jsonOutput) {
+          output(playResult, true);
+        } else {
+          console.log(`You: ${playResult.userHandName} | AI: ${playResult.aiHandName} => ${playResult.outcome}`);
+        }
+        break;
+      }
+      
+      case 'status': {
+        const result = await agent.status();
+        output(result, jsonOutput);
+        break;
+      }
+      
+      case 'start': {
+        const result = await agent.start();
+        output(result, jsonOutput);
+        break;
+      }
+      
+      case 'stop': {
+        const result = await agent.stop();
+        output(result, jsonOutput);
+        break;
+      }
+      
+      case 'help':
+      case '--help':
+      case '-h':
+      default: {
+        console.log(`
+LSJI CLI - Reinforcement Learning Agent Framework
+
+Usage: lsji <command> [options]
+
+Commands:
+  train     Train the agent
+  play      Play against the agent
+  status    Show system status and statistics
+  start     Enable training/play
+  stop      Disable training/play
+  help      Show this help
+
+Options:
+  --storage <type>     Storage backend: sqlite, better-sqlite, memory (default: sqlite)
+  --db-path <path>     Database file path (default: ./lsji.db)
+  --alpha <number>     Learning rate (default: 0.1)
+  --gamma <number>     Discount factor (default: 0.9)
+  --epsilon <number>   Exploration rate (default: 0.1)
+  --opponent <type>    Opponent strategy: random, always_rock, counter, sequential (default: random)
+  --json               Output as JSON
+
+Train options:
+  --episodes <number>  Number of episodes (default: 200)
+  --pattern <number>   Training pattern 0-3 (default: 0)
+                       0=random, 1=always_rock, 2=counter, 3=sequential
+  --batch-size <number> Batch size for DB (default: 200)
+
+Play options:
+  --hand <number>      Your hand: 0=Rock, 1=Scissors, 2=Paper
+
+Examples:
+  lsji train --episodes 500
+  lsji train --episodes 100 --pattern 1 --opponent always_rock
+  lsji play --hand 0
+  lsji status --json
+  lsji start
+  lsji stop
+`);
+        break;
+      }
+    }
+  } catch (error) {
+    console.error('Error:', error.message);
+    process.exit(1);
+  } finally {
+    await storage.close();
+  }
+}

package/src/core/agent.js

@@ -0,0 +1,239 @@
+/**
+ * High-Level Agent Orchestration
+ * 
+ * Combines Q-Learning engine with environment and storage.
+ * Provides train/play/status/start/stop operations.
+ * Migrated from the original Cloudflare Workers routes.
+ */
+
+import { QLearning } from './qlearning.js';
+
+/**
+ * Agent configuration
+ * @typedef {Object} AgentConfig
+ * @property {QLearning} qlearning - Q-Learning engine
+ * @property {Storage} storage - Storage backend
+ * @property {Env} [env] - Environment (optional, for play/train modes)
+ */
+
+/**
+ * Training result
+ * @typedef {Object} TrainResult
+ * @property {number} episodes - Number of episodes completed
+ * @property {number} wins - Number of wins (reward > 0)
+ * @property {number} losses - Number of losses (reward < 0)
+ * @property {number} draws - Number of draws (reward = 0)
+ */
+
+/**
+ * Play result
+ * @typedef {Object} PlayResult
+ * @property {number} action - Agent's chosen action
+ * @property {number} reward - Reward received
+ * @property {boolean} done - Whether episode ended
+ * @property {Object} info - Additional info from environment
+ */
+
+/**
+ * Status information
+ * @typedef {Object} StatusInfo
+ * @property {'running'|'stopped'} status - System status
+ * @property {number} todayTotal - Battles today
+ * @property {number} limit - Daily limit
+ * @property {Array} performance - Performance by mode
+ * @property {Array} aiBrain - Full Q-table
+ */
+
+/**
+ * Agent - Main orchestration class for RL agent
+ */
+export class Agent {
+  /**
+   * @param {AgentConfig} config
+   */
+  constructor({ qlearning, storage, env = null } = {}) {
+    if (!qlearning || !storage) {
+      throw new Error('QLearning and Storage are required');
+    }
+    this.qlearning = qlearning;
+    this.storage = storage;
+    this.env = env;
+    this.isActive = true;
+  }
+
+  /**
+   * Check if system is active
+   * @returns {Promise<boolean>}
+   */
+  async checkActive() {
+    const config = await this.storage.getSetting('is_active');
+    this.isActive = config ? config.value === '1' : true;
+    return this.isActive;
+  }
+
+  /**
+   * Start the system (enable training/play)
+   * @returns {Promise<{status: string, message: string}>}
+   */
+  async start() {
+    await this.storage.setSetting('is_active', 1);
+    this.isActive = true;
+    return { status: 'success', message: 'System STARTED' };
+  }
+
+  /**
+   * Stop the system (disable training/play)
+   * @returns {Promise<{status: string, message: string}>}
+   */
+  async stop() {
+    await this.storage.setSetting('is_active', 0);
+    this.isActive = false;
+    return { status: 'success', message: 'System STOPPED' };
+  }
+
+  /**
+   * Get current status and statistics
+   * @returns {Promise<StatusInfo>}
+   */
+  async status() {
+    await this.checkActive();
+    
+    const todayCount = await this.storage.getTodayBattleCount();
+    const stats = await this.storage.getPerformanceStats();
+    const brain = await this.qlearning.getFullQTable();
+    
+    return {
+      status: this.isActive ? 'running' : 'stopped',
+      todayTotal: todayCount,
+      limit: 90000,
+      performance: stats,
+      aiBrain: brain
+    };
+  }
+
+  /**
+   * Train the agent with specified episodes
+   * 
+   * @param {Object} options
+   * @param {number} [options.episodes=200] - Number of training episodes
+   * @param {Function} [options.actionSelector] - Function(episode, lastAction) -> action
+   * @param {number} [options.batchSize=200] - Batch size for DB operations
+   * @returns {Promise<TrainResult>}
+   */
+  async train({ episodes = 200, actionSelector = null, batchSize = 200 } = {}) {
+    await this.checkActive();
+    if (!this.isActive) {
+      throw new Error('System is paused. Use start() first.');
+    }
+    if (!this.env) {
+      throw new Error('Environment is required for training');
+    }
+
+    let wins = 0, losses = 0, draws = 0;
+    const batchOperations = [];
+    let lastAction = 0;
+
+    for (let i = 0; i < episodes; i++) {
+      // Determine action - use custom selector or default to random
+      let action;
+      if (actionSelector) {
+        action = actionSelector(i, lastAction);
+      } else {
+        action = Math.floor(Math.random() * this.env.actionSize());
+      }
+      lastAction = action;
+
+      // Get current state
+      const state = await this.env.getState() || '0';
+      
+      // Execute action in environment
+      const result = await this.env.step(action);
+      
+      // Track reward
+      const reward = result.reward;
+      if (reward > 0) wins++;
+      else if (reward < 0) losses++;
+      else draws++;
+
+      // Q-learning update
+      await this.qlearning.learnSimple(state, action, reward);
+
+      // Batch database operations
+      batchOperations.push(this.storage.addBattle({
+        mode: 'train',
+        handA: action,
+        handB: result.info?.opponentAction ?? 0,
+        reward,
+        createdAt: new Date().toISOString()
+      }));
+
+      // Flush batch
+      if (batchOperations.length >= batchSize) {
+        await Promise.all(batchOperations);
+        batchOperations.length = 0;
+      }
+    }
+
+    // Flush remaining
+    if (batchOperations.length > 0) {
+      await Promise.all(batchOperations);
+    }
+
+    return { episodes, wins, losses, draws };
+  }
+
+  /**
+   * Play a single step against the agent
+   * Uses epsilon-greedy policy for action selection.
+   * 
+   * @param {Object} [options] - Play options
+   * @param {number} [options.userAction] - Optional user action (for envs that need it)
+   * @returns {Promise<PlayResult>}
+   */
+  async play(options = {}) {
+    await this.checkActive();
+    if (!this.isActive) {
+      throw new Error('System is paused. Use start() first.');
+    }
+    if (!this.env) {
+      throw new Error('Environment is required for play mode');
+    }
+
+    // Get current state
+    const state = await this.env.getState() || '0';
+    const actionSize = this.env.actionSize();
+
+    // Epsilon-greedy action selection
+    const action = await this.qlearning.act(state, actionSize);
+
+    // Execute in environment
+    const result = await this.env.step(action);
+
+    // Update Q-table
+    await this.qlearning.learnSimple(state, action, result.reward);
+
+    // Record battle
+    await this.storage.addBattle({
+      mode: 'test',
+      handA: action,
+      handB: result.info?.opponentAction ?? options.userAction ?? 0,
+      reward: result.reward,
+      createdAt: new Date().toISOString()
+    });
+
+    return {
+      action,
+      reward: result.reward,
+      done: result.done,
+      info: result.info
+    };
+  }
+
+  /**
+   * Set the environment (for dependency injection)
+   * @param {Env} env
+   */
+  setEnvironment(env) {
+    this.env = env;
+  }
+}

package/src/core/env.js

@@ -0,0 +1,86 @@
+/**
+ * Environment Interface for Reinforcement Learning
+ * 
+ * All RL environments must implement this interface.
+ * The agent interacts with the environment through this contract.
+ */
+
+/**
+ * Result of a single environment step
+ * @typedef {Object} StepResult
+ * @property {string} state - The new state after taking the action
+ * @property {number} reward - The reward received for the action
+ * @property {boolean} done - Whether the episode has ended
+ * @property {Object} [info] - Additional diagnostic information
+ */
+
+/**
+ * Base Environment Class
+ * 
+ * @abstract
+ */
+export class Env {
+  /**
+   * Get the current state representation
+   * @returns {string} Current state identifier
+   */
+  getState() {
+    throw new Error('getState() must be implemented by subclass');
+  }
+
+  /**
+   * Execute an action in the environment
+   * @param {number} action - Action to execute
+   * @returns {Promise<StepResult>} Result of the step
+   */
+  async step(action) {
+    throw new Error('step() must be implemented by subclass');
+  }
+
+  /**
+   * Get the number of possible actions
+   * @returns {number} Action space size
+   */
+  actionSize() {
+    throw new Error('actionSize() must be implemented by subclass');
+  }
+
+  /**
+   * Reset the environment to initial state
+   * @returns {Promise<string>} Initial state
+   */
+  async reset() {
+    throw new Error('reset() must be implemented by subclass');
+  }
+
+  /**
+   * Render the environment (optional, for debugging/visualization)
+   * @returns {string} Human-readable representation
+   */
+  render() {
+    return '';
+  }
+}
+
+/**
+ * State encoder/decoder utilities for tabular Q-learning
+ */
+export const StateEncoder = {
+  /**
+   * Encode a state object to string key
+   * @param {Object} state - State object
+   * @returns {string} Encoded state key
+   */
+  encode(state) {
+    return JSON.stringify(state);
+  },
+
+  /**
+   * Decode a state key to object
+   * @param {string} key - Encoded state key
+   * @returns {Object} Decoded state
+   */
+  decode(key) {
+    return JSON.parse(key);
+  }
+};