@game_ryo/lsji 0.1.0 → 0.3.0

package/docs/docs/examples/advanced-training.md

@@ -1,244 +0,0 @@
----
-title: Advanced Training
-description: Custom training patterns and techniques
----
-
-# Advanced Training
-
-Learn advanced training techniques for better agent performance.
-
-## Custom Action Selectors
-
-The `train()` method accepts an `actionSelector` function for custom training patterns.
-
-```typescript
-const result = await agent.train({
-  episodes: 1000,
-  actionSelector: (episode, lastAction) => {
-    // Your custom logic here
-    return action;
-  }
-});
-```
-
-### Epsilon-Greedy with Decay
-
-```typescript
-let epsilon = 1.0;
-const minEpsilon = 0.01;
-const decayRate = 0.9995;
-
-const result = await agent.train({
-  episodes: 10000,
-  actionSelector: (episode, lastAction) => {
-    epsilon = Math.max(minEpsilon, epsilon * decayRate);
-    
-    if (Math.random() < epsilon) {
-      return Math.floor(Math.random() * 3); // Explore
-    }
-    
-    // Exploit: use agent's Q-learning
-    const state = await agent.env.getState();
-    return agent.qlearning.act(state, 3);
-  }
-});
-```
-
-### Curriculum Learning
-
-Start with easy opponents, progress to harder ones.
-
-```typescript
-const opponents = ['always_rock', 'sequential', 'counter', 'random'];
-const episodesPerStage = 250;
-
-for (const opponent of opponents) {
-  const env = new RockPaperScissorsEnv({ opponent });
-  agent.setEnvironment(env);
-  
-  console.log(`Training against ${opponent}...`);
-  await agent.train({ episodes: episodesPerStage });
-  
-  const status = await agent.status();
-  console.log(`Win rate: ${status.performance.find(p => p.mode === 'train')?.win_rate}%`);
-}
-```
-
-### Self-Play Training
-
-Train agent against itself.
-
-```typescript
-// Create two agents sharing the same Q-table
-const storage = await createStorage('sqlite', { path: './selfplay.db' });
-const qlearning = new QLearning({ alpha: 0.1, gamma: 0.9, epsilon: 0.1, storage });
-
-const env1 = new RockPaperScissorsEnv({ opponent: 'random' });
-const env2 = new RockPaperScissorsEnv({ opponent: 'random' });
-
-const agent1 = new Agent({ qlearning, storage, env: env1 });
-const agent2 = new Agent({ qlearning, storage, env: env2 });
-
-// Alternate training
-for (let i = 0; i < 100; i++) {
-  await agent1.train({ episodes: 50 });
-  await agent2.train({ episodes: 50 });
-  
-  if (i % 10 === 0) {
-    const status = await agent1.status();
-    console.log(`Iteration ${i}: ${status.performance[0].win_rate}% win rate`);
-  }
-}
-```
-
-## Hyperparameter Tuning
-
-### Grid Search
-
-```typescript
-const configs = [
-  { alpha: 0.05, gamma: 0.9, epsilon: 0.1 },
-  { alpha: 0.1, gamma: 0.9, epsilon: 0.1 },
-  { alpha: 0.2, gamma: 0.9, epsilon: 0.1 },
-  { alpha: 0.1, gamma: 0.95, epsilon: 0.1 },
-  { alpha: 0.1, gamma: 0.9, epsilon: 0.2 },
-];
-
-for (const config of configs) {
-  const storage = await createStorage('memory');
-  const qlearning = new QLearning({ ...config, storage });
-  const env = new RockPaperScissorsEnv({ opponent: 'random' });
-  const agent = new Agent({ qlearning, storage, env });
-  
-  await agent.train({ episodes: 2000 });
-  const status = await agent.status();
-  const winRate = status.performance.find(p => p.mode === 'train')?.win_rate || 0;
-  
-  console.log(`${JSON.stringify(config)} => ${winRate}%`);
-  await storage.close();
-}
-```
-
-### Bayesian Optimization
-
-Use libraries like `bayes-opt` for efficient hyperparameter search.
-
-## Evaluation Techniques
-
-### Fixed Opponent Evaluation
-
-```typescript
-async function evaluate(agent, opponent, games = 100) {
-  const env = new RockPaperScissorsEnv({ opponent });
-  agent.setEnvironment(env);
-  
-  let wins = 0, losses = 0, draws = 0;
-  
-  for (let i = 0; i < games; i++) {
-    const result = await agent.play(Math.floor(Math.random() * 3));
-    if (result.reward > 0) wins++;
-    else if (result.reward < 0) losses++;
-    else draws++;
-  }
-  
-  return { wins, losses, draws, winRate: wins / games };
-}
-
-const agents = {
-  random: await evaluate(agent, 'random'),
-  alwaysRock: await evaluate(agent, 'always_rock'),
-  counter: await evaluate(agent, 'counter'),
-  sequential: await evaluate(agent, 'sequential'),
-};
-
-console.table(agents);
-```
-
-### Cross-Validation
-
-```typescript
-async function crossValidate(config, folds = 5, episodesPerFold = 1000) {
-  const results = [];
-  
-  for (let fold = 0; fold < folds; fold++) {
-    const storage = await createStorage('memory');
-    const qlearning = new QLearning({ ...config, storage });
-    const env = new RockPaperScissorsEnv({ opponent: 'random' });
-    const agent = new Agent({ qlearning, storage, env });
-    
-    await agent.train({ episodes: episodesPerFold });
-    const evalResult = await evaluate(agent, 'random', 200);
-    results.push(evalResult.winRate);
-    
-    await storage.close();
-  }
-  
-  const mean = results.reduce((a, b) => a + b, 0) / results.length;
-  const std = Math.sqrt(results.reduce((a, b) => a + (b - mean) ** 2, 0) / results.length);
-  
-  return { mean, std, results };
-}
-```
-
-## Checkpointing and Resuming
-
-```typescript
-// Save Q-table periodically
-async function trainWithCheckpoints(agent, episodes, checkpointEvery = 100) {
-  for (let i = 0; i < episodes; i += checkpointEvery) {
-    const batch = Math.min(checkpointEvery, episodes - i);
-    await agent.train({ episodes: batch });
-    
-    // Q-table automatically persisted to storage
-    const status = await agent.status();
-    console.log(`Checkpoint ${i + batch}: ${status.aiBrain.length} Q-values`);
-  }
-}
-
-// Resume from existing Q-table
-const storage = await createStorage('sqlite', { path: './existing.db' });
-const qlearning = new QLearning({ alpha: 0.1, gamma: 0.9, epsilon: 0.1, storage });
-// Q-table loads automatically on first use
-```
-
-## Distributed Training
-
-Run multiple training processes with shared storage.
-
-```bash
-# Terminal 1
-lsji train --episodes 500 --db-path ./shared.db --storage sqlite
-
-# Terminal 2 (same database)
-lsji train --episodes 500 --db-path ./shared.db --storage sqlite
-
-# Terminal 3
-lsji train --episodes 500 --db-path ./shared.db --storage sqlite
-```
-
-All processes read/write to the same SQLite database, enabling parallel training.
-
-## Monitoring Training Progress
-
-```typescript
-async function trainWithLogging(agent, episodes) {
-  const history = [];
-  
-  for (let i = 0; i < episodes; i += 100) {
-    await agent.train({ episodes: 100 });
-    
-    const status = await agent.status();
-    const trainStat = status.performance.find(p => p.mode === 'train');
-    
-    history.push({
-      episode: i + 100,
-      winRate: trainStat?.win_rate || 0,
-      qTableSize: status.aiBrain.length
-    });
-    
-    console.log(`Episode ${i + 100}: ${history[history.length - 1].winRate}% win rate`);
-  }
-  
-  return history;
-}
-```

package/docs/docs/examples/custom-environment.md

@@ -1,198 +0,0 @@
----
-title: Custom Environment
-description: Build your own RL environment
----
-
-# Custom Environment Example
-
-This guide shows how to create a custom environment by extending the `Env` base class.
-
-## Grid World Environment
-
-A simple 1D grid where the agent learns to move right to reach the goal.
-
-```typescript
-import { Env } from 'lsji';
-
-class GridWorldEnv extends Env {
-  constructor(gridSize = 10) {
-    super();
-    this.gridSize = gridSize;
-    this.position = 0;
-  }
-
-  getState(): string {
-    return String(this.position);
-  }
-
-  async step(action: number): Promise<StepResult> {
-    // Actions: 0 = left, 1 = right
-    if (action === 0) {
-      this.position = Math.max(0, this.position - 1);
-    } else if (action === 1) {
-      this.position = Math.min(this.gridSize - 1, this.position + 1);
-    }
-
-    const done = this.position === this.gridSize - 1;
-    const reward = done ? 1 : -0.01; // Small penalty for each step
-
-    return {
-      state: String(this.position),
-      reward,
-      done,
-      info: { position: this.position }
-    };
-  }
-
-  actionSize(): number {
-    return 2; // Left, Right
-  }
-
-  async reset(): Promise<string> {
-    this.position = 0;
-    return '0';
-  }
-
-  render(): string {
-    const bar = ' '.repeat(this.gridSize);
-    const chars = bar.split('');
-    chars[this.position] = 'A';
-    chars[this.gridSize - 1] = 'G';
-    return `[${chars.join('')}]`;
-  }
-}
-```
-
-## Using the Custom Environment
-
-```typescript
-import { Agent, QLearning, createStorage } from 'lsji';
-import { GridWorldEnv } from './grid-world';
-
-async function main() {
-  const storage = await createStorage('sqlite', { path: './gridworld.db' });
-  
-  const qlearning = new QLearning({
-    alpha: 0.1,
-    gamma: 0.9,
-    epsilon: 0.1,
-    storage
-  });
-
-  const env = new GridWorldEnv(10);
-  const agent = new Agent({ qlearning, storage, env });
-
-  console.log('Training...');
-  await agent.train({ episodes: 5000 });
-
-  console.log('Testing...');
-  await agent.play(); // Single step
-  
-  const status = await agent.status();
-  console.log('Q-table:', status.aiBrain);
-
-  await storage.close();
-}
-
-main().catch(console.error);
-```
-
-## Multi-State Environment
-
-For environments with multiple state variables, use `StateEncoder`:
-
-```typescript
-import { Env, StateEncoder } from 'lsji';
-
-interface GameState {
-  playerHP: number;
-  enemyHP: number;
-  hasPotion: boolean;
-}
-
-class BattleEnv extends Env {
-  state: GameState = { playerHP: 100, enemyHP: 50, hasPotion: true };
-
-  getState(): string {
-    return StateEncoder.encode(this.state);
-  }
-
-  async step(action: number): Promise<StepResult> {
-    // 0 = attack, 1 = heal, 2 = defend
-    let reward = 0;
-    let done = false;
-
-    if (action === 0) { // Attack
-      this.state.enemyHP -= 10;
-      reward = this.state.enemyHP <= 0 ? 10 : -1;
-      done = this.state.enemyHP <= 0;
-    } else if (action === 1) { // Heal
-      if (this.state.hasPotion) {
-        this.state.playerHP = Math.min(100, this.state.playerHP + 30);
-        this.state.hasPotion = false;
-        reward = -1;
-      } else {
-        reward = -5; // No potion penalty
-      }
-    } else if (action === 2) { // Defend
-      reward = -0.5;
-    }
-
-    // Enemy counter-attack
-    if (!done) {
-      this.state.playerHP -= 5;
-      if (this.state.playerHP <= 0) {
-        reward = -10;
-        done = true;
-      }
-    }
-
-    return {
-      state: StateEncoder.encode(this.state),
-      reward,
-      done,
-      info: { ...this.state }
-    };
-  }
-
-  actionSize(): number {
-    return 3;
-  }
-
-  async reset(): Promise<string> {
-    this.state = { playerHP: 100, enemyHP: 50, hasPotion: true };
-    return StateEncoder.encode(this.state);
-  }
-}
-```
-
-## Key Points
-
-1. **State as string** — Use `StateEncoder.encode()` for complex states
-2. **Reward design** — Shape rewards to guide learning (dense vs sparse)
-3. **Action space** — Keep small for tabular Q-learning
-4. **Episode termination** — Always implement `done` condition
-5. **Reset** — Must restore initial state completely
-
-## Testing Your Environment
-
-```typescript
-async function testEnv() {
-  const env = new GridWorldEnv(5);
-  
-  console.log('Initial:', env.getState());
-  console.log(env.render());
-  
-  for (let i = 0; i < 10; i++) {
-    const result = await env.step(1); // Always move right
-    console.log(`Step ${i}: state=${result.state}, reward=${result.reward}, done=${result.done}`);
-    console.log(env.render());
-    if (result.done) break;
-  }
-  
-  await env.reset();
-  console.log('After reset:', env.getState());
-}
-
-testEnv();
-```

package/docs/docs/examples/custom-storage.md

@@ -1,251 +0,0 @@
----
-title: Custom Storage Backend
-description: Implement your own storage backend
----
-
-# Custom Storage Backend
-
-Create a custom storage backend by extending the abstract `Storage` class.
-
-## Interface to Implement
-
-```typescript
-import { Storage } from 'lsji';
-
-abstract class Storage {
-  abstract initialize(): Promise<void>;
-  abstract close(): Promise<void>;
-  abstract getSetting(key: string): Promise<{key: string, value: string} | null>;
-  abstract setSetting(key: string, value: string | number): Promise<void>;
-  abstract getQTable(): Promise<Array<{state: string, action: number, q_value: number}>>;
-  abstract updateQ(state: string, action: number, qValue: number): Promise<void>;
-  abstract addBattle(record: BattleRecord): Promise<void>;
-  abstract getTodayBattleCount(): Promise<number>;
-  abstract getPerformanceStats(): Promise<Array<{mode: string, total: number, win_rate: number}>>;
-}
-
-interface BattleRecord {
-  mode: 'train' | 'test';
-  handA: number;
-  handB: number;
-  reward: number;
-  createdAt: string;
-}
-```
-
-## Example: Redis Storage
-
-```typescript
-import { Storage } from 'lsji';
-import Redis from 'ioredis';
-
-class RedisStorage extends Storage {
-  constructor(redisUrl = 'redis://localhost:6379') {
-    super();
-    this.redis = new Redis(redisUrl);
-    this.keyPrefix = 'lsji:';
-  }
-
-  async initialize() {
-    // Set default settings
-    await this.setSetting('is_active', 1);
-  }
-
-  async close() {
-    await this.redis.quit();
-  }
-
-  async getSetting(key) {
-    const value = await this.redis.get(this.keyPrefix + 'setting:' + key);
-    return value ? { key, value } : null;
-  }
-
-  async setSetting(key, value) {
-    await this.redis.set(this.keyPrefix + 'setting:' + key, String(value));
-  }
-
-  async getQTable() {
-    const keys = await this.redis.keys(this.keyPrefix + 'q:*');
-    const results = [];
-    
-    for (const key of keys) {
-      const value = await this.redis.get(key);
-      const parts = key.replace(this.keyPrefix + 'q:', '').split(':');
-      results.push({
-        state: parts[0],
-        action: parseInt(parts[1], 10),
-        q_value: parseFloat(value)
-      });
-    }
-    
-    return results.sort((a, b) => a.state.localeCompare(b.state) || a.action - b.action);
-  }
-
-  async updateQ(state, action, qValue) {
-    await this.redis.set(
-      this.keyPrefix + `q:${state}:${action}`, 
-      qValue.toString()
-    );
-  }
-
-  async addBattle(record) {
-    const battleKey = this.keyPrefix + `battle:${Date.now()}:${Math.random()}`;
-    await this.redis.hset(battleKey, {
-      mode: record.mode,
-      hand_a: record.handA.toString(),
-      hand_b: record.handB.toString(),
-      reward: record.reward.toString(),
-      created_at: record.createdAt
-    });
-    
-    // Add to sorted set for date queries
-    await this.redis.zadd(
-      this.keyPrefix + 'battles:by_date',
-      new Date(record.createdAt).getTime(),
-      battleKey
-    );
-  }
-
-  async getTodayBattleCount() {
-    const today = new Date();
-    today.setHours(0, 0, 0, 0);
-    const tomorrow = new Date(today);
-    tomorrow.setDate(tomorrow.getDate() + 1);
-    
-    return this.redis.zcount(
-      this.keyPrefix + 'battles:by_date',
-      today.getTime(),
-      tomorrow.getTime()
-    );
-  }
-
-  async getPerformanceStats() {
-    const keys = await this.redis.zrange(this.keyPrefix + 'battles:by_date', 0, -1);
-    const stats = new Map();
-    
-    for (const key of keys) {
-      const battle = await this.redis.hgetall(key);
-      const mode = battle.mode;
-      
-      if (!stats.has(mode)) {
-        stats.set(mode, { total: 0, wins: 0 });
-      }
-      
-      const stat = stats.get(mode);
-      stat.total++;
-      if (parseInt(battle.reward) > 0) stat.wins++;
-    }
-    
-    return Array.from(stats.entries()).map(([mode, stat]) => ({
-      mode,
-      total: stat.total,
-      win_rate: stat.total > 0 ? Math.round((stat.wins / stat.total) * 1000) / 10 : 0
-    }));
-  }
-}
-```
-
-## Register Custom Storage
-
-Add to `createStorage` factory (or use directly):
-
-```typescript
-import { createStorage } from 'lsji';
-
-// Option 1: Use directly
-const storage = new RedisStorage('redis://localhost:6379');
-await storage.initialize();
-
-// Option 2: Extend createStorage (modify src/storage/index.js)
-import { RedisStorage } from './redis-storage';
-
-// In your code:
-const storage = new RedisStorage();
-await storage.initialize();
-```
-
-## Example: PostgreSQL Storage
-
-```typescript
-import { Pool } from 'pg';
-
-class PostgresStorage extends Storage {
-  constructor(connectionString) {
-    super();
-    this.pool = new Pool({ connectionString });
-  }
-
-  async initialize() {
-    await this.pool.query(`
-      CREATE TABLE IF NOT EXISTS settings (
-        key TEXT PRIMARY KEY,
-        value TEXT NOT NULL
-      );
-      CREATE TABLE IF NOT EXISTS battle_history (
-        id SERIAL PRIMARY KEY,
-        mode TEXT NOT NULL,
-        hand_a INTEGER NOT NULL,
-        hand_b INTEGER NOT NULL,
-        reward INTEGER NOT NULL,
-        created_at TIMESTAMP NOT NULL
-      );
-      CREATE TABLE IF NOT EXISTS q_table (
-        state TEXT NOT NULL,
-        action INTEGER NOT NULL,
-        q_value REAL NOT NULL DEFAULT 0,
-        PRIMARY KEY (state, action)
-      );
-    `);
-    await this.setSetting('is_active', 1);
-  }
-
-  async close() {
-    await this.pool.end();
-  }
-
-  async getSetting(key) {
-    const res = await this.pool.query('SELECT key, value FROM settings WHERE key = $1', [key]);
-    return res.rows[0] || null;
-  }
-
-  async setSetting(key, value) {
-    await this.pool.query(
-      `INSERT INTO settings (key, value) VALUES ($1, $2) 
-       ON CONFLICT (key) DO UPDATE SET value = $2`,
-      [key, String(value)]
-    );
-  }
-
-  // ... implement other methods similarly
-}
-```
-
-## Testing Custom Storage
-
-```typescript
-import { MemoryStorage } from 'lsji';
-
-// Use MemoryStorage as reference implementation for testing
-async function testStorageImplementation(StorageClass) {
-  const storage = new StorageClass();
-  await storage.initialize();
-  
-  // Test settings
-  await storage.setSetting('test', 'value');
-  const setting = await storage.getSetting('test');
-  assert(setting.value === 'value');
-  
-  // Test Q-table
-  await storage.updateQ('state1', 0, 0.5);
-  const qTable = await storage.getQTable();
-  assert(qTable[0].q_value === 0.5);
-  
-  // Test battles
-  await storage.addBattle({ mode: 'train', handA: 0, handB: 1, reward: 1, createdAt: new Date().toISOString() });
-  const count = await storage.getTodayBattleCount();
-  assert(count === 1);
-  
-  await storage.close();
-  console.log('All tests passed!');
-}
-```