@holoscript/framework 6.0.3

package/src/training/SpatialTrainingDataTypes.ts

@@ -0,0 +1,216 @@
+/**
+ * Spatial Training Data Types
+ *
+ * Type definitions for the spatial reasoning training data pipeline.
+ * Used to generate instruction-response pairs from HoloScript compositions
+ * with spatial constraints (spatial_adjacent, spatial_contains, spatial_reachable)
+ * for fine-tuning LLMs on spatial reasoning tasks.
+ *
+ * @module training/SpatialTrainingDataTypes
+ */
+
+// =============================================================================
+// DIFFICULTY LEVELS
+// =============================================================================
+
+/**
+ * Difficulty levels for generated spatial reasoning examples.
+ *
+ * - basic: 2 objects, single spatial relationship
+ * - intermediate: 3-5 objects, multiple relationships, mixed constraint types
+ * - advanced: 6+ objects, occlusion, nested containment, chained reachability
+ */
+export type SpatialDifficulty = 'basic' | 'intermediate' | 'advanced';
+
+// =============================================================================
+// SPATIAL RELATIONSHIP TYPES
+// =============================================================================
+
+/**
+ * The three core spatial relationship types from HoloScript's constraint system.
+ */
+export type SpatialRelationshipType = 'spatial_adjacent' | 'spatial_contains' | 'spatial_reachable';
+
+// =============================================================================
+// SCENE OBJECT DEFINITION
+// =============================================================================
+
+/**
+ * A scene object with spatial properties for training data generation.
+ */
+export interface SceneObject {
+  /** Unique identifier for the object */
+  id: string;
+  /** Object type (e.g., 'cube', 'sphere', 'zone', 'npc') */
+  type: string;
+  /** 3D position */
+  position: { x: number; y: number; z: number };
+  /** 3D scale */
+  scale: { x: number; y: number; z: number };
+  /** Bounding box (for containment checks) */
+  bounds?: {
+    min: { x: number; y: number; z: number };
+    max: { x: number; y: number; z: number };
+  };
+  /** Whether this object acts as an obstacle */
+  isObstacle?: boolean;
+  /** Optional color for visual description */
+  color?: string;
+  /** Optional geometry type */
+  geometry?: string;
+}
+
+// =============================================================================
+// SPATIAL RELATIONSHIP DEFINITION
+// =============================================================================
+
+/**
+ * A spatial relationship between two scene objects.
+ */
+export interface SpatialRelationship {
+  /** The relationship type */
+  type: SpatialRelationshipType;
+  /** Source object ID */
+  sourceId: string;
+  /** Target object ID */
+  targetId: string;
+  /** Whether the relationship is satisfied (true=positive, false=negative) */
+  satisfied: boolean;
+  /** Constraint parameters */
+  params: SpatialRelationshipParams;
+}
+
+/**
+ * Parameters for spatial relationship constraints.
+ */
+export interface SpatialRelationshipParams {
+  /** For adjacent: maximum distance */
+  maxDistance?: number;
+  /** For adjacent: minimum distance */
+  minDistance?: number;
+  /** For adjacent: axis restriction */
+  axis?: string;
+  /** For contains: margin */
+  margin?: number;
+  /** For contains: strict mode */
+  strict?: boolean;
+  /** For reachable: max path length */
+  maxPathLength?: number;
+  /** For reachable: obstacle types */
+  obstacleTypes?: string[];
+  /** For reachable: algorithm */
+  algorithm?: string;
+}
+
+// =============================================================================
+// SCENE DEFINITION
+// =============================================================================
+
+/**
+ * A complete scene with objects and spatial relationships.
+ */
+export interface SpatialScene {
+  /** Scene name/identifier */
+  name: string;
+  /** Objects in the scene */
+  objects: SceneObject[];
+  /** Spatial relationships between objects */
+  relationships: SpatialRelationship[];
+  /** Difficulty level */
+  difficulty: SpatialDifficulty;
+  /** The HoloScript composition source */
+  holoScriptSource: string;
+}
+
+// =============================================================================
+// TRAINING EXAMPLE (INSTRUCTION-RESPONSE PAIR)
+// =============================================================================
+
+/**
+ * A single training example with instruction and response.
+ * Suitable for fine-tuning LLMs on spatial reasoning tasks.
+ */
+export interface SpatialTrainingExample {
+  /** Unique example ID */
+  id: string;
+  /** The instruction/question for the LLM */
+  instruction: string;
+  /** The expected response/answer */
+  response: string;
+  /** The HoloScript source that defines the scene */
+  context: string;
+  /** Spatial relationship type being tested */
+  relationshipType: SpatialRelationshipType;
+  /** Whether this is a positive or negative example */
+  isPositive: boolean;
+  /** Difficulty level */
+  difficulty: SpatialDifficulty;
+  /** Tags for categorization */
+  tags: string[];
+}
+
+// =============================================================================
+// GENERATOR CONFIGURATION
+// =============================================================================
+
+/**
+ * Configuration options for the SpatialTrainingDataGenerator.
+ */
+export interface SpatialGeneratorConfig {
+  /** Number of examples to generate per relationship type per difficulty level */
+  examplesPerCategory?: number;
+  /** Which relationship types to include */
+  relationshipTypes?: SpatialRelationshipType[];
+  /** Which difficulty levels to include */
+  difficultyLevels?: SpatialDifficulty[];
+  /** Ratio of positive to negative examples (default 0.5 = equal) */
+  positiveRatio?: number;
+  /** Random seed for reproducibility (if provided) */
+  seed?: number;
+  /** Whether to include HoloScript context in output */
+  includeContext?: boolean;
+}
+
+// =============================================================================
+// GENERATOR STATISTICS
+// =============================================================================
+
+/**
+ * Statistics about generated training data.
+ */
+export interface SpatialGeneratorStats {
+  /** Total number of examples generated */
+  totalExamples: number;
+  /** Breakdown by relationship type */
+  byRelationship: Record<SpatialRelationshipType, number>;
+  /** Breakdown by difficulty */
+  byDifficulty: Record<SpatialDifficulty, number>;
+  /** Breakdown by positive/negative */
+  positiveCount: number;
+  negativeCount: number;
+  /** Number of unique templates used */
+  uniqueTemplatesUsed: number;
+}
+
+// =============================================================================
+// JSONL OUTPUT FORMAT
+// =============================================================================
+
+/**
+ * JSONL line format for fine-tuning output.
+ * Each line in the JSONL file is one of these objects.
+ */
+export interface SpatialTrainingJSONLEntry {
+  /** The instruction/question */
+  instruction: string;
+  /** The expected response */
+  response: string;
+  /** Metadata for filtering/analysis */
+  metadata: {
+    id: string;
+    relationship_type: SpatialRelationshipType;
+    is_positive: boolean;
+    difficulty: SpatialDifficulty;
+    tags: string[];
+  };
+}

package/src/training/TrainingPipelineConfig.ts

@@ -0,0 +1,215 @@
+/**
+ * TrainingPipelineConfig - Unified Training Pipeline Configuration
+ *
+ * Integrates all training pipeline components:
+ * - Quality Scoring (W.010): Multi-dimensional quality filtering
+ * - Hard Dedup (W.004): Exact/near-duplicate removal (external)
+ * - SoftDedup (W.008): N-gram commonness-based reweighting
+ * - LR Schedule (W.009): Warmup + cosine decay
+ * - Hyperparameters (W.006, W.007): Learning rate, batch size, epochs
+ *
+ * Pipeline order: Quality Filter -> Hard Dedup -> SoftDedup -> Training
+ *
+ * @module training/TrainingPipelineConfig
+ */
+
+import type { SoftDedupConfig } from './SoftDedup';
+import type { LRSchedulerConfig } from './LRScheduler';
+import type { QualityScoringConfig } from './QualityScoringPipeline';
+import { DEFAULT_SOFTDEDUP_CONFIG } from './SoftDedup';
+import { DEFAULT_LR_SCHEDULER_CONFIG } from './LRScheduler';
+import { DEFAULT_SCORING_CONFIG } from './QualityScoringPipeline';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+/**
+ * Complete training pipeline configuration.
+ *
+ * Encompasses all stages from data preparation through training execution.
+ */
+export interface TrainingPipelineConfig {
+  /** Data quality filtering configuration (W.010) */
+  qualityScoring: QualityScoringConfig;
+
+  /** Soft deduplication configuration (W.008) */
+  softDedup: SoftDedupConfig;
+
+  /** Learning rate schedule configuration (W.009) */
+  lrSchedule: LRSchedulerConfig;
+
+  /** Core training hyperparameters (W.006, W.007) */
+  hyperparameters: TrainingHyperparameters;
+
+  /** Pipeline-level settings */
+  pipeline: PipelineSettings;
+}
+
+/**
+ * Core training hyperparameters per W.006 and W.007.
+ */
+export interface TrainingHyperparameters {
+  /**
+   * Base learning rate.
+   * Per W.006: 2e-4 for SFT (NOT 2e-5).
+   */
+  learningRate: number;
+
+  /**
+   * Number of training epochs.
+   * Per W.006: 2 epochs (NOT 3). "Loss converges in 1-2 epochs."
+   */
+  epochs: number;
+
+  /**
+   * Optimizer.
+   * Per W.006: paged_adamw_8bit (NOT adamw_torch).
+   */
+  optimizer: 'paged_adamw_8bit' | 'adamw_torch' | 'adafactor';
+
+  /**
+   * Micro-batch size per device.
+   * Per W.007: 8-16 for 7B models.
+   */
+  microBatchSize: number;
+
+  /**
+   * Gradient accumulation steps.
+   * Per W.007: 2-4 steps for effective batch 32-512.
+   */
+  gradientAccumulationSteps: number;
+
+  /**
+   * Maximum gradient norm for clipping.
+   */
+  maxGradNorm: number;
+
+  /**
+   * Weight decay coefficient.
+   */
+  weightDecay: number;
+}
+
+/**
+ * Pipeline-level settings controlling the data preparation flow.
+ */
+export interface PipelineSettings {
+  /**
+   * Whether to apply quality scoring filter before training.
+   * Per W.010: Apply BEFORE deduplication to avoid wasting compute on junk.
+   */
+  enableQualityFilter: boolean;
+
+  /**
+   * Whether to apply SoftDedup after hard dedup.
+   * Per W.008: Apply AFTER hard dedup (W.004), not instead of it.
+   */
+  enableSoftDedup: boolean;
+
+  /**
+   * Whether to use the LR scheduler (warmup + cosine decay).
+   * Per W.009: Always use.
+   */
+  enableLRSchedule: boolean;
+
+  /**
+   * Seed for reproducibility.
+   */
+  seed?: number;
+}
+
+// =============================================================================
+// DEFAULT CONFIGURATION
+// =============================================================================
+
+/**
+ * Default training pipeline configuration.
+ *
+ * Implements the full optimization pipeline:
+ * - Quality Filter (W.010) -> Hard Dedup (W.004) -> SoftDedup (W.008)
+ * - LR Schedule: warmup 10% + cosine decay (W.009)
+ * - Hyperparameters: LR=2e-4, epochs=2, paged_adamw_8bit (W.006)
+ * - Batch: micro=8, accumulation=4, effective=32 (W.007)
+ */
+export const DEFAULT_TRAINING_PIPELINE_CONFIG: TrainingPipelineConfig = {
+  qualityScoring: DEFAULT_SCORING_CONFIG,
+  softDedup: DEFAULT_SOFTDEDUP_CONFIG,
+  lrSchedule: DEFAULT_LR_SCHEDULER_CONFIG,
+  hyperparameters: {
+    learningRate: 2e-4,
+    epochs: 2,
+    optimizer: 'paged_adamw_8bit',
+    microBatchSize: 8,
+    gradientAccumulationSteps: 4,
+    maxGradNorm: 1.0,
+    weightDecay: 0.01,
+  },
+  pipeline: {
+    enableQualityFilter: true,
+    enableSoftDedup: true,
+    enableLRSchedule: true,
+  },
+};
+
+// =============================================================================
+// CONFIG BUILDER
+// =============================================================================
+
+/** Deep partial utility type */
+type DeepPartial<T> = {
+  [P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
+};
+
+/**
+ * Build a TrainingPipelineConfig with custom overrides.
+ *
+ * @example
+ * ```ts
+ * const config = buildTrainingPipelineConfig({
+ *   hyperparameters: { learningRate: 1e-4, epochs: 3 },
+ *   softDedup: { temperature: 0.5 },
+ *   lrSchedule: { totalSteps: 5000 },
+ * });
+ * ```
+ */
+export function buildTrainingPipelineConfig(
+  overrides: DeepPartial<TrainingPipelineConfig> = {}
+): TrainingPipelineConfig {
+  return {
+    qualityScoring: {
+      ...DEFAULT_TRAINING_PIPELINE_CONFIG.qualityScoring,
+      ...(overrides.qualityScoring as Partial<QualityScoringConfig> | undefined),
+    },
+    softDedup: {
+      ...DEFAULT_TRAINING_PIPELINE_CONFIG.softDedup,
+      ...(overrides.softDedup as Partial<SoftDedupConfig> | undefined),
+    },
+    lrSchedule: {
+      ...DEFAULT_TRAINING_PIPELINE_CONFIG.lrSchedule,
+      ...(overrides.lrSchedule as Partial<LRSchedulerConfig> | undefined),
+    },
+    hyperparameters: {
+      ...DEFAULT_TRAINING_PIPELINE_CONFIG.hyperparameters,
+      ...(overrides.hyperparameters as Partial<TrainingHyperparameters> | undefined),
+    },
+    pipeline: {
+      ...DEFAULT_TRAINING_PIPELINE_CONFIG.pipeline,
+      ...(overrides.pipeline as Partial<PipelineSettings> | undefined),
+    },
+  };
+}
+
+/**
+ * Compute the total training steps from dataset size and hyperparameters.
+ *
+ * @param datasetSize - Number of training examples (after dedup)
+ * @param config - Training pipeline configuration
+ * @returns Total number of training steps
+ */
+export function computeTotalSteps(datasetSize: number, config: TrainingPipelineConfig): number {
+  const { microBatchSize, gradientAccumulationSteps, epochs } = config.hyperparameters;
+  const effectiveBatchSize = microBatchSize * gradientAccumulationSteps;
+  const stepsPerEpoch = Math.ceil(datasetSize / effectiveBatchSize);
+  return stepsPerEpoch * epochs;
+}

package/src/training/constants.ts

@@ -0,0 +1,94 @@
+/**
+ * Training Constants
+ *
+ * Canonical constants shared between @holoscript/core and TrainingMonkey.
+ * This module has ZERO imports from other workspace packages (G.GAP.01 prevention).
+ *
+ * @version 1.0.0
+ */
+
+/**
+ * 9 training categories covering all HoloScript domains
+ */
+export const TRAINING_CATEGORIES = [
+  'vr-interaction',
+  'multiplayer',
+  'physics',
+  'ui-spatial',
+  'ai-agents',
+  'procedural',
+  'audio-spatial',
+  'visual-effects',
+  'game-mechanics',
+] as const;
+
+export type TrainingCategory = (typeof TRAINING_CATEGORIES)[number];
+
+/**
+ * 4 difficulty levels for training data
+ */
+export const DIFFICULTY_LEVELS = ['beginner', 'intermediate', 'advanced', 'production'] as const;
+
+export type DifficultyLevel = (typeof DIFFICULTY_LEVELS)[number];
+
+/**
+ * Quality score thresholds for training data evaluation
+ */
+export const QUALITY_THRESHOLDS = {
+  Excellent: { min: 90, max: 100 },
+  VeryGood: { min: 80, max: 89 },
+  Acceptable: { min: 70, max: 79 },
+} as const;
+
+export type QualityTier = keyof typeof QUALITY_THRESHOLDS;
+
+/**
+ * Default quality constraints for training data generators
+ */
+export const DEFAULT_GENERATOR_THRESHOLDS = {
+  min_compression_ratio: 5,
+  max_compression_ratio: 15,
+  max_duplication_rate: 0.05,
+  min_templates_per_difficulty: 10,
+  min_quality_score: 0.7,
+} as const;
+
+/**
+ * RuleForge domain categories used by the rule generator
+ */
+export const RULEFORGE_DOMAINS = [
+  'ai_agents',
+  'physics',
+  'robotics',
+  'audio',
+  'rendering',
+  'interaction',
+  'multiplayer',
+  'vr_ar',
+] as const;
+
+export type RuleForgeDomain = (typeof RULEFORGE_DOMAINS)[number];
+
+/**
+ * Get quality tier for a given score
+ */
+export function getQualityTier(score: number): QualityTier | 'BelowAcceptable' {
+  if (score >= QUALITY_THRESHOLDS.Excellent.min) return 'Excellent';
+  if (score >= QUALITY_THRESHOLDS.VeryGood.min) return 'VeryGood';
+  if (score >= QUALITY_THRESHOLDS.Acceptable.min) return 'Acceptable';
+  return 'BelowAcceptable';
+}
+
+/**
+ * Validate that a category string is a valid TrainingCategory
+ */
+export function isValidCategory(category: string): category is TrainingCategory {
+  return (TRAINING_CATEGORIES as readonly string[]).includes(category);
+}
+
+/**
+ * Validate that a difficulty string is a valid DifficultyLevel
+ */
+export function isValidDifficulty(difficulty: string): difficulty is DifficultyLevel {
+  return (DIFFICULTY_LEVELS as readonly string[]).includes(difficulty);
+}

package/src/training/index.ts

@@ -0,0 +1,138 @@
+/**
+ * Training Data Pipeline
+ *
+ * Generates labeled spatial reasoning training examples from
+ * HoloScript compositions with spatial constraints, monitors
+ * SNN sparsity for energy-efficient neural computation, and provides
+ * training data optimization via SoftDedup and LR scheduling.
+ *
+ * Pipeline order (per training rules):
+ *   Quality Filter (W.010) -> Hard Dedup (W.004) -> SoftDedup (W.008) -> Training
+ *
+ * @module training
+ */
+
+export {
+  SpatialTrainingDataGenerator,
+  createSpatialTrainingDataGenerator,
+} from './SpatialTrainingDataGenerator';
+
+export type {
+  SpatialDifficulty,
+  SpatialRelationshipType,
+  SceneObject,
+  SpatialRelationship,
+  SpatialScene,
+  SpatialTrainingExample,
+  SpatialGeneratorConfig,
+  SpatialGeneratorStats,
+  SpatialTrainingJSONLEntry,
+  SpatialRelationshipParams,
+} from './SpatialTrainingDataTypes';
+
+export { SparsityMonitor, createSparsityMonitor } from './SparsityMonitor';
+
+export type { LayerActivityInput } from './SparsityMonitor';
+
+export type {
+  SNNLayerMetrics,
+  SparsitySnapshot,
+  EnergyEfficiencyMetrics,
+  SparsityViolation,
+  SparsityMonitorConfig,
+  SparsityMonitorStats,
+  SparsityQualityHistoryEntry,
+} from './SparsityMonitorTypes';
+
+// SoftDedup (W.008): N-gram commonness-based reweighting
+export { SoftDedup, createSoftDedup, DEFAULT_SOFTDEDUP_CONFIG } from './SoftDedup';
+
+export type { SoftDedupConfig, SoftDedupResult, NgramStats, SoftDedupStats } from './SoftDedup';
+
+// LR Scheduler (W.009): Warmup + cosine decay
+export {
+  LRScheduler,
+  createSFTScheduler,
+  createGRPOScheduler,
+  DEFAULT_LR_SCHEDULER_CONFIG,
+  GRPO_LR_SCHEDULER_CONFIG,
+} from './LRScheduler';
+
+export type { LRSchedulerConfig, LRSchedulerSnapshot, LRScheduleStats } from './LRScheduler';
+
+// Unified Training Pipeline Configuration
+export {
+  DEFAULT_TRAINING_PIPELINE_CONFIG,
+  buildTrainingPipelineConfig,
+  computeTotalSteps,
+} from './TrainingPipelineConfig';
+
+export type {
+  TrainingPipelineConfig,
+  TrainingHyperparameters,
+  PipelineSettings,
+} from './TrainingPipelineConfig';
+
+// Quality Scoring Pipeline
+export { QualityScoringPipeline, DEFAULT_SCORING_CONFIG } from './QualityScoringPipeline';
+
+export type { QualityScore, QualityDetail, QualityScoringConfig } from './QualityScoringPipeline';
+
+// TrainingMonkey Integration: Spatial reasoning data -> Alpaca format pipeline
+export {
+  TrainingMonkeyIntegration,
+  createTrainingMonkeyIntegration,
+  DEFAULT_INTEGRATION_CONFIG,
+} from './trainingmonkey';
+
+export type {
+  AlpacaEntry,
+  WeightedAlpacaEntry,
+  DatasetSplit,
+  SplitStats,
+  TrainingMonkeyConfig,
+  TrainingMonkeyIntegrationConfig,
+  IntegrationResult,
+} from './trainingmonkey';
+
+// ============================================================================
+// Canonical Training Constants (Gap 1: Extracted from TrainingMonkey)
+// ============================================================================
+
+// Constants (zero-dependency module)
+export {
+  TRAINING_CATEGORIES,
+  DIFFICULTY_LEVELS,
+  QUALITY_THRESHOLDS,
+  DEFAULT_GENERATOR_THRESHOLDS,
+  RULEFORGE_DOMAINS,
+  getQualityTier,
+  isValidCategory,
+  isValidDifficulty,
+  type TrainingCategory,
+  type DifficultyLevel,
+  type QualityTier,
+  type RuleForgeDomain,
+} from './constants';
+
+// Training Data Schema (JSONL-compatible)
+export {
+  validateTrainingExample,
+  type TrainingExample,
+  type TrainingExampleMetadata,
+  type TrainingQualityScore,
+  type TrainingValidationResult,
+  type TrainingValidationError,
+  type CompressionResult as TrainingCompressionResult,
+  type GeneratorMetrics,
+} from './schema';
+
+// Trait Mappings (TM <-> HS)
+export {
+  TM_REGISTERED_TRAITS,
+  validateTraitName,
+  generateValidationReport,
+  type TrainingMetadata,
+  type TraitMapping,
+  type TraitValidationReport,
+} from './trait-mappings';