@icyfenix-dmla/cli 2026.5.3-821 → 2026.5.4-1216

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icyfenix-dmla/cli",
-  "version": "2026.5.3-821",
+  "version": "2026.5.4-1216",
   "description": "DMLA 沙箱服务命令行工具",
   "type": "module",
   "main": "src/index.js",

package/scripts/build.js

@@ -49,11 +49,11 @@ function copyDir(src, dest, filter = null) {
   return true
 }
 
-// 复制服务器代码（只复制 .js 文件）
+// 复制服务器代码（复制 .js 和 .py 文件）
 console.log('\n📋 复制服务器代码...')
 console.log(`   源目录: ${localServerSrc}`)
 console.log(`   目标目录: ${cliServerDest}`)
-copyDir(localServerSrc, cliServerDest, (name) => name.endsWith('.js'))
+copyDir(localServerSrc, cliServerDest, (name) => name.endsWith('.js') || name.endsWith('.py'))
 
 // 复制共享模块（复制所有 .py 文件和 __init__.py）
 console.log('\n📋 复制共享模块...')

package/shared_modules/__init__.py

@@ -1,10 +1,10 @@
 # shared 模块包初始化
 # 包含统计学习系列文档中可复用的类定义
 
-from .linear import *
-from .cnn import *
 from .bayesian import *
-from .tree import *
+from .cnn import *
+from .linear import *
+from .neural import *
 from .svm import *
+from .tree import *
 from .unsupervised import *
-from .neural import *

package/shared_modules/bayesian/__init__.py

@@ -1,6 +1,6 @@
-# Bayesian 模块
-from .simple_bayesian_network import SimpleBayesianNetwork
+# BAYESIAN 模块
+from .bayesian_network import SimpleBayesianNetwork
 from .gaussian_mixture_model import GaussianMixtureModel
 from .multinomial_naive_bayes import MultinomialNaiveBayes
 
-__all__ = ['SimpleBayesianNetwork', 'GaussianMixtureModel', 'MultinomialNaiveBayes']
+__all__ = ['SimpleBayesianNetwork', 'GaussianMixtureModel', 'MultinomialNaiveBayes']

package/shared_modules/cnn/__init__.py

@@ -1,5 +1,8 @@
 # CNN 模块
-from .alex_net import AlexNet
+from .alexnet import AlexNet
+from .minimal_preprocess_cache import MinimalPreprocessCache
+from .realtime_dataset import RealtimeAugmentDataset
+from .realtime_dataset import RealtimeValDataset
 from .tiny_imagenet_dataset import TinyImageNetDataset
 
-__all__ = ['AlexNet', 'TinyImageNetDataset']
+__all__ = ['AlexNet', 'MinimalPreprocessCache', 'RealtimeAugmentDataset', 'RealtimeValDataset', 'TinyImageNetDataset']

package/shared_modules/cnn/minimal_preprocess_cache.py

@@ -0,0 +1,153 @@
+# MinimalPreprocessCache 类定义
+# 从文档自动提取生成
+
+import os
+from PIL import Image
+
+class MinimalPreprocessCache:
+    """
+    最小缓存策略：执行 64*64 -> 224*224 Resize，保存为 JPEG 格式
+    缓存大小：约 600MB，训练时约 4GB 内存
+    """
+    def __init__(self, data_dir, cache_dir):
+        self.data_dir = data_dir
+        self.cache_dir = cache_dir
+        self.train_cache = os.path.join(cache_dir, 'train')
+        self.val_cache = os.path.join(cache_dir, 'val')
+        self.manifest_path = os.path.join(cache_dir, 'manifest.json')
+        
+    def preprocess_image(self, img_path, save_path):
+        """单张图片预处理：Resize(224) → JPEG"""
+        img = Image.open(img_path).convert('RGB')
+        img = img.resize((224, 224), Image.BILINEAR)
+        img.save(save_path, 'JPEG', quality=95)
+    
+    def check_cache_exists(self):
+        """检查缓存是否已完整存在"""
+        return os.path.exists(self.manifest_path)
+    
+    def get_cache_stats(self):
+        """获取缓存统计信息"""
+        if os.path.exists(self.manifest_path):
+            with open(self.manifest_path, 'r') as f:
+                manifest = json.load(f)
+            return manifest.get('train_count', 0), manifest.get('val_count', 0)
+        return 0, 0
+    
+    def _preprocess_train_set(self, progress):
+        """预处理训练集"""
+        train_dir = os.path.join(self.data_dir, 'train')
+        classes = sorted(os.listdir(train_dir))
+        
+        os.makedirs(self.train_cache, exist_ok=True)
+        total_count = 0
+        
+        for cls_idx, cls in enumerate(classes):
+            cls_cache_dir = os.path.join(self.train_cache, cls)
+            
+            # 中断恢复：检查已存在的类别目录
+            if os.path.exists(cls_cache_dir):
+                existing_files = [f for f in os.listdir(cls_cache_dir) if f.endswith('.JPEG')]
+                if len(existing_files) >= 500:
+                    total_count += len(existing_files)
+                    progress.update(cls_idx + 1, message=f"跳过已缓存类别 {cls_idx+1}/200: {cls}")
+                    continue
+            
+            os.makedirs(cls_cache_dir, exist_ok=True)
+            
+            images_dir = os.path.join(train_dir, cls, 'images')
+            if not os.path.exists(images_dir):
+                continue
+            
+            count = 0
+            for img_name in os.listdir(images_dir):
+                if img_name.endswith('.JPEG'):
+                    img_path = os.path.join(images_dir, img_name)
+                    save_path = os.path.join(cls_cache_dir, img_name)
+                    
+                    try:
+                        self.preprocess_image(img_path, save_path)
+                        count += 1
+                        total_count += 1
+                    except Exception as e:
+                        print(f"Warning: Failed to process {img_path}: {e}")
+            
+            progress.update(cls_idx + 1, message=f"预处理类别 {cls_idx+1}/200: {cls} ({count} 张)")
+        
+        return total_count
+    
+    def _preprocess_val_set(self, progress):
+        """预处理验证集"""
+        val_dir = os.path.join(self.data_dir, 'val')
+        val_images_dir = os.path.join(val_dir, 'images')
+        val_annotations = os.path.join(val_dir, 'val_annotations.txt')
+        
+        # 读取类别映射
+        wnids_path = os.path.join(self.data_dir, 'wnids.txt')
+        with open(wnids_path, 'r') as f:
+            wnids = [line.strip() for line in f.readlines()]
+        class_to_idx = {wnid: idx for idx, wnid in enumerate(wnids)}
+        
+        # 读取标注文件
+        with open(val_annotations, 'r') as f:
+            val_lines = f.readlines()
+        total_val = len(val_lines)
+        
+        os.makedirs(self.val_cache, exist_ok=True)
+        
+        # 中断恢复
+        existing_files = [f for f in os.listdir(self.val_cache) if f.endswith('.JPEG')]
+        start_idx = len(existing_files)
+        if start_idx >= total_val:
+            progress.update(total_val, message=f"验证集已缓存: {total_val} 张")
+            return total_val, []
+        
+        labels = []
+        progress.reset(total_steps=total_val, description="预处理验证集")
+        
+        for line_idx in range(start_idx, total_val):
+            parts = val_lines[line_idx].strip().split('\t')
+            if len(parts) >= 2:
+                img_name = parts[0]
+                img_path = os.path.join(val_images_dir, img_name)
+                save_path = os.path.join(self.val_cache, f'val_{line_idx}.JPEG')
+                if os.path.exists(img_path):
+                    try:
+                        self.preprocess_image(img_path, save_path)
+                        labels.append(class_to_idx.get(parts[1], 0))
+                    except Exception as e:
+                        print(f"处理图片出现异常 {img_path}: {e}")
+                
+                if (line_idx + 1) % 100 == 0 or line_idx == total_val - 1:
+                    progress.update(line_idx + 1, message=f"预处理验证集 {line_idx+1}/{total_val}")
+        
+        return total_val, labels
+    
+    def run(self, progress):
+        """执行预处理（支持断点续传）"""
+        start_time = time.time()
+        os.makedirs(self.cache_dir, exist_ok=True)
+        train_count = self._preprocess_train_set(progress)
+        val_count, val_labels = self._preprocess_val_set(progress)
+        
+        # 保存清单文件
+        manifest = {
+            'train_count': train_count,
+            'val_count': val_count,
+            'val_labels': val_labels if val_labels else self._load_existing_val_labels()
+        }
+        with open(self.manifest_path, 'w') as f:
+            json.dump(manifest, f)
+        
+        elapsed = time.time() - start_time
+        progress.complete(message=f"预处理完成: 训练集 {train_count} 张, 验证集 {val_count} 张, 耗时 {elapsed:.1f}s")
+        
+        return train_count, val_count
+    
+    def _load_existing_val_labels(self):
+        """加载已有的验证集标签"""
+        if os.path.exists(self.manifest_path):
+            with open(self.manifest_path, 'r') as f:
+                manifest = json.load(f)
+                return manifest.get('val_labels', [])
+        return []

package/shared_modules/cnn/realtime_dataset.py

@@ -0,0 +1,178 @@
+# 实时数据增强 Dataset 类
+# 从缓存读取 JPEG，实时执行数据增强
+
+import os
+import json
+from PIL import Image
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms
+
+
+class RealtimeAugmentDataset(Dataset):
+    """
+    实时执行数据增强的 Dataset
+
+    流程：
+    1. 从缓存读取 JPEG（224×224）
+    2. CPU 执行 ToTensor + RandomFlip + RandomCrop + ColorJitter
+    3. Normalize 参数提供，可在 GPU 执行
+
+    特点：
+    - 内存占用低：只缓存当前 batch
+    - 数据增强随机性：每次 epoch 看到不同版本
+    - 多线程友好：配合 DataLoader num_workers
+    """
+
+    # ImageNet Normalize 参数
+    MEAN = [0.485, 0.456, 0.406]
+    STD = [0.229, 0.224, 0.225]
+
+    def __init__(self, cache_dir, augment=True, normalize_on_gpu=False):
+        """
+        Args:
+            cache_dir: 缓存目录路径
+            augment: 是否执行数据增强（训练集 True，验证集 False）
+            normalize_on_gpu: 是否将 Normalize 移到 GPU 执行
+        """
+        self.cache_dir = cache_dir
+        self.augment = augment
+        self.normalize_on_gpu = normalize_on_gpu
+
+        # 加载清单文件
+        manifest_path = os.path.join(cache_dir, 'manifest.json')
+        if os.path.exists(manifest_path):
+            with open(manifest_path, 'r') as f:
+                self.manifest = json.load(f)
+        else:
+            self.manifest = {'val_labels': []}
+
+        # 加载图片路径和标签
+        self.image_paths = []
+        self.labels = []
+
+        train_cache = os.path.join(cache_dir, 'train')
+        if os.path.exists(train_cache):
+            # 加载类别映射
+            wnids_path = '/data/datasets/tiny-imagenet-200/wnids.txt'
+            if os.path.exists(wnids_path):
+                with open(wnids_path, 'r') as f:
+                    wnids = [line.strip() for line in f.readlines()]
+                class_to_idx = {wnid: idx for idx, wnid in enumerate(wnids)}
+            else:
+                class_to_idx = {}
+
+            classes = sorted(os.listdir(train_cache))
+            for cls_idx, cls in enumerate(classes):
+                cls_dir = os.path.join(train_cache, cls)
+                if os.path.isdir(cls_dir):
+                    for img_name in os.listdir(cls_dir):
+                        if img_name.endswith('.JPEG'):
+                            self.image_paths.append(os.path.join(cls_dir, img_name))
+                            self.labels.append(class_to_idx.get(cls, cls_idx))
+
+        # CPU 数据增强（训练集）
+        if augment:
+            self.transform = transforms.Compose([
+                transforms.ToTensor(),
+                transforms.RandomHorizontalFlip(p=0.5),
+                transforms.RandomCrop(224, padding=4),
+                transforms.ColorJitter(brightness=0.2, contrast=0.2),
+            ])
+        else:
+            # 验证集预处理（无增强）
+            self.transform = transforms.Compose([
+                transforms.ToTensor(),
+            ])
+
+        # Normalize（可在 CPU 或 GPU 执行）
+        if not normalize_on_gpu:
+            self.transform = transforms.Compose([
+                self.transform,
+                transforms.Normalize(mean=self.MEAN, std=self.STD)
+            ])
+
+    def __len__(self):
+        return len(self.image_paths)
+
+    def __getitem__(self, idx):
+        """
+        获取单张图片
+
+        Returns:
+            image: tensor [3, 224, 224]
+            label: int
+        """
+        img_path = self.image_paths[idx]
+        label = self.labels[idx]
+
+        # 从缓存读取 JPEG
+        image = Image.open(img_path).convert('RGB')
+
+        # 执行变换
+        image = self.transform(image)
+
+        return image, label
+
+    def get_normalize_params(self):
+        """获取 Normalize 参数（供 GPU 执行时使用）"""
+        return torch.tensor(self.MEAN).view(3, 1, 1), torch.tensor(self.STD).view(3, 1, 1)
+
+
+class RealtimeValDataset(Dataset):
+    """
+    验证集 Dataset（从缓存读取）
+
+    扁平化结构：val/val_<idx>.JPEG
+    """
+
+    MEAN = [0.485, 0.456, 0.406]
+    STD = [0.229, 0.224, 0.225]
+
+    def __init__(self, cache_dir, normalize_on_gpu=False):
+        self.cache_dir = cache_dir
+        self.val_cache = os.path.join(cache_dir, 'val')
+        self.normalize_on_gpu = normalize_on_gpu
+
+        # 加载清单文件获取标签
+        manifest_path = os.path.join(cache_dir, 'manifest.json')
+        if os.path.exists(manifest_path):
+            with open(manifest_path, 'r') as f:
+                manifest = json.load(f)
+                self.labels = manifest.get('val_labels', [])
+        else:
+            self.labels = []
+
+        # 构建图片路径列表
+        self.image_paths = []
+        if os.path.exists(self.val_cache):
+            for i in range(len(self.labels)):
+                img_path = os.path.join(self.val_cache, f'val_{i}.JPEG')
+                if os.path.exists(img_path):
+                    self.image_paths.append(img_path)
+
+        # 验证集变换（无增强）
+        self.transform = transforms.Compose([
+            transforms.ToTensor(),
+        ])
+
+        if not normalize_on_gpu:
+            self.transform = transforms.Compose([
+                self.transform,
+                transforms.Normalize(mean=self.MEAN, std=self.STD)
+            ])
+
+    def __len__(self):
+        return len(self.image_paths)
+
+    def __getitem__(self, idx):
+        img_path = self.image_paths[idx]
+        label = self.labels[idx]
+
+        image = Image.open(img_path).convert('RGB')
+        image = self.transform(image)
+
+        return image, label
+
+    def get_normalize_params(self):
+        return torch.tensor(self.MEAN).view(3, 1, 1), torch.tensor(self.STD).view(3, 1, 1)

package/shared_modules/linear/__init__.py

@@ -1,6 +1,8 @@
-# Linear 模块
+# LINEAR 模块
+from .lasso_regression import LassoRegression
 from .logistic_regression import LogisticRegression
+from .naive_bayes import MultinomialNaiveBayes
+from .naive_bayes import GaussianNaiveBayes
 from .ridge_regression import RidgeRegression
-from .lasso_regression import LassoRegression
 
-__all__ = ['LogisticRegression', 'RidgeRegression', 'LassoRegression']
+__all__ = ['LassoRegression', 'LogisticRegression', 'MultinomialNaiveBayes', 'GaussianNaiveBayes', 'RidgeRegression']

package/shared_modules/neural/__init__.py

@@ -1,4 +1,4 @@
-# Neural 模块
+# NEURAL 模块
 from .perceptron import Perceptron
 
-__all__ = ['Perceptron']
+__all__ = ['Perceptron']

package/shared_modules/svm/__init__.py

@@ -2,4 +2,4 @@
 from .kernel_svm import KernelSVM
 from .simple_svm import SimpleSVM
 
-__all__ = ['KernelSVM', 'SimpleSVM']
+__all__ = ['KernelSVM', 'SimpleSVM']

package/shared_modules/tree/__init__.py

@@ -1,6 +1,6 @@
-# Tree 模块
+# TREE 模块
+from .ada_boost import AdaBoost
 from .decision_tree_classifier import DecisionTreeClassifier
 from .random_forest_classifier import RandomForestClassifier
-from .ada_boost import AdaBoost
 
-__all__ = ['DecisionTreeClassifier', 'RandomForestClassifier', 'AdaBoost']
+__all__ = ['AdaBoost', 'DecisionTreeClassifier', 'RandomForestClassifier']

package/shared_modules/unsupervised/__init__.py

@@ -1,5 +1,5 @@
-# Unsupervised 模块
+# UNSUPERVISED 模块
 from .kmeans import KMeans
 from .pca import PCA
 
-__all__ = ['KMeans', 'PCA']
+__all__ = ['KMeans', 'PCA']

package/src/commands/server.js

@@ -326,6 +326,44 @@ function findServerPath() {
   return null
 }
 
+/**
+ * 查找 kernel_runner.py 路径
+ * --dev 模式下需要挂载此文件
+ */
+function findKernelRunnerPath() {
+  // 开发环境路径：packages/cli/src/commands -> ../../../local-server/src/kernel_runner.py
+  const devPath = path.resolve(__dirname, '../../../local-server/src/kernel_runner.py')
+  // npm 包路径：packages/cli/src/commands -> ../server/kernel_runner.py（构建后）
+  const npmPath = path.resolve(__dirname, '../server/kernel_runner.py')
+
+  if (fs.existsSync(devPath)) {
+    return devPath
+  }
+  if (fs.existsSync(npmPath)) {
+    return npmPath
+  }
+  return null
+}
+
+/**
+ * 查找 dmla_progress.py 路径
+ * --dev 模式下需要挂载此文件
+ */
+function findProgressReporterPath() {
+  // 开发环境路径：packages/cli/src/commands -> ../../../local-server/src/dmla_progress.py
+  const devPath = path.resolve(__dirname, '../../../local-server/src/dmla_progress.py')
+  // npm 包路径：packages/cli/src/commands -> ../server/dmla_progress.py（构建后）
+  const npmPath = path.resolve(__dirname, '../server/dmla_progress.py')
+
+  if (fs.existsSync(devPath)) {
+    return devPath
+  }
+  if (fs.existsSync(npmPath)) {
+    return npmPath
+  }
+  return null
+}
+
 /**
  * 查找共享模块目录
  * --dev 模式下需要挂载此目录
@@ -427,10 +465,23 @@ export async function startServerSync(port, useGpu = false, dev = false) {
 
   // 查找共享模块路径（--dev 模式需要）
   const sharedModulesPath = dev ? findSharedModulesPath() : null
+  // 查找 kernel_runner.py 路径（--dev 模式需要）
+  const kernelRunnerPath = dev ? findKernelRunnerPath() : null
+  // 查找 dmla_progress.py 路径（--dev 模式需要）
+  const progressReporterPath = dev ? findProgressReporterPath() : null
+
   if (dev && !sharedModulesPath) {
     console.log(chalk.yellow('⚠️ --dev 模式需要共享模块目录'))
     console.log(chalk.gray('   未找到 shared_modules，将仅使用镜像内置模块'))
   }
+  if (dev && !kernelRunnerPath) {
+    console.log(chalk.yellow('⚠️ --dev 模式需要 kernel_runner.py'))
+    console.log(chalk.gray('   未找到 kernel_runner.py，将仅使用镜像内置版本'))
+  }
+  if (dev && !progressReporterPath) {
+    console.log(chalk.yellow('⚠️ --dev 模式需要 dmla_progress.py'))
+    console.log(chalk.gray('   未找到 dmla_progress.py，将仅使用镜像内置版本'))
+  }
 
   console.log(chalk.gray(`   镜像类型: ${imageResolution.message}`))
   console.log(chalk.gray('   同步模式启动...'))
@@ -438,6 +489,12 @@ export async function startServerSync(port, useGpu = false, dev = false) {
   if (dev && sharedModulesPath) {
     console.log(chalk.gray(`   共享模块: ${sharedModulesPath}`))
   }
+  if (dev && kernelRunnerPath) {
+    console.log(chalk.gray(`   执行器: ${kernelRunnerPath}`))
+  }
+  if (dev && progressReporterPath) {
+    console.log(chalk.gray(`   进度报告: ${progressReporterPath}`))
+  }
   console.log()
 
   // 设置环境变量
@@ -452,6 +509,12 @@ export async function startServerSync(port, useGpu = false, dev = false) {
     if (sharedModulesPath) {
       process.env.SHARED_MODULES_PATH = sharedModulesPath
     }
+    if (kernelRunnerPath) {
+      process.env.KERNEL_RUNNER_PATH = kernelRunnerPath
+    }
+    if (progressReporterPath) {
+      process.env.PROGRESS_REPORTER_PATH = progressReporterPath
+    }
   }
 
   // 动态 import 服务器模块并直接运行
@@ -542,9 +605,20 @@ export async function startServer(port, useGpu = false, dev = false) {
 
     // 查找共享模块路径（--dev 模式需要）
     const sharedModulesPath = dev ? findSharedModulesPath() : null
+    // 查找 kernel_runner.py 路径（--dev 模式需要）
+    const kernelRunnerPath = dev ? findKernelRunnerPath() : null
+    // 查找 dmla_progress.py 路径（--dev 模式需要）
+    const progressReporterPath = dev ? findProgressReporterPath() : null
+
     if (dev && sharedModulesPath) {
       console.log(chalk.gray(`   共享模块: ${sharedModulesPath}`))
     }
+    if (dev && kernelRunnerPath) {
+      console.log(chalk.gray(`   执行器: ${kernelRunnerPath}`))
+    }
+    if (dev && progressReporterPath) {
+      console.log(chalk.gray(`   进度报告: ${progressReporterPath}`))
+    }
 
     // 日志文件路径
     const logDir = path.resolve(__dirname, '../../logs')
@@ -574,6 +648,12 @@ export async function startServer(port, useGpu = false, dev = false) {
       if (sharedModulesPath) {
         env.SHARED_MODULES_PATH = sharedModulesPath
       }
+      if (kernelRunnerPath) {
+        env.KERNEL_RUNNER_PATH = kernelRunnerPath
+      }
+      if (progressReporterPath) {
+        env.PROGRESS_REPORTER_PATH = progressReporterPath
+      }
     }
 
     // 写入启动日志