synthra 0.1.0

data/tech_docs/integration/ci_cd.md

@@ -0,0 +1,707 @@
+# CI/CD Integration
+
+Setting up FakeDataDSL in continuous integration pipelines.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [GitHub Actions](#github-actions)
+- [GitLab CI](#gitlab-ci)
+- [CircleCI](#circleci)
+- [Jenkins](#jenkins)
+- [Schema Validation](#schema-validation)
+- [Test Data Generation](#test-data-generation)
+- [Best Practices](#best-practices)
+
+---
+
+## Overview
+
+FakeDataDSL can be integrated into CI/CD pipelines for:
+
+1. **Schema Validation** - Validate DSL files before deployment
+2. **Test Data Generation** - Generate consistent test data
+3. **Database Seeding** - Populate staging environments
+4. **Load Testing** - Generate data for performance tests
+
+---
+
+## GitHub Actions
+
+### Basic Workflow
+
+Create `.github/workflows/test.yml`:
+
+```yaml
+name: Test
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+      
+      - name: Validate schemas
+        run: |
+          bundle exec fake_data_dsl validate schemas/*.dsl
+      
+      - name: Run tests
+        run: bundle exec rspec
+        env:
+          FAKE_DATA_SEED: ${{ github.run_id }}
+```
+
+### Schema Validation Workflow
+
+Create `.github/workflows/validate-schemas.yml`:
+
+```yaml
+name: Validate Schemas
+
+on:
+  push:
+    paths:
+      - 'schemas/**'
+      - 'db/schemas/**'
+      - 'spec/schemas/**'
+  pull_request:
+    paths:
+      - 'schemas/**'
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+      
+      - name: Validate all schemas
+        run: |
+          echo "Validating schemas..."
+          for file in schemas/*.dsl db/schemas/*.dsl spec/schemas/*.dsl; do
+            if [ -f "$file" ]; then
+              echo "Checking $file..."
+              bundle exec fake_data_dsl validate "$file"
+            fi
+          done
+      
+      - name: Test schema generation
+        run: |
+          echo "Testing generation..."
+          for file in schemas/*.dsl; do
+            if [ -f "$file" ]; then
+              schema_name=$(basename "$file" .dsl | sed 's/.*/\u&/')
+              echo "Generating $schema_name..."
+              bundle exec fake_data_dsl generate "$schema_name" -d schemas -c 5
+            fi
+          done
+```
+
+### Generate Test Fixtures
+
+Create `.github/workflows/generate-fixtures.yml`:
+
+```yaml
+name: Generate Test Fixtures
+
+on:
+  workflow_dispatch:
+    inputs:
+      count:
+        description: 'Number of records per schema'
+        required: false
+        default: '100'
+      seed:
+        description: 'Random seed for reproducibility'
+        required: false
+        default: '12345'
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+      
+      - name: Generate fixtures
+        run: |
+          mkdir -p fixtures
+          
+          bundle exec fake_data_dsl generate User \
+            -d schemas \
+            -c ${{ inputs.count }} \
+            -s ${{ inputs.seed }} \
+            --pretty > fixtures/users.json
+          
+          bundle exec fake_data_dsl generate Order \
+            -d schemas \
+            -c ${{ inputs.count }} \
+            -s ${{ inputs.seed }} \
+            --pretty > fixtures/orders.json
+      
+      - name: Upload fixtures
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-fixtures
+          path: fixtures/
+```
+
+---
+
+## GitLab CI
+
+### Basic Pipeline
+
+Create `.gitlab-ci.yml`:
+
+```yaml
+stages:
+  - validate
+  - test
+  - generate
+
+variables:
+  FAKE_DATA_SEED: $CI_PIPELINE_ID
+
+validate-schemas:
+  stage: validate
+  image: ruby:3.3
+  before_script:
+    - bundle install
+  script:
+    - bundle exec fake_data_dsl validate schemas/*.dsl
+  only:
+    changes:
+      - schemas/**/*
+
+test:
+  stage: test
+  image: ruby:3.3
+  before_script:
+    - bundle install
+  script:
+    - bundle exec rspec
+  artifacts:
+    reports:
+      junit: rspec.xml
+
+generate-fixtures:
+  stage: generate
+  image: ruby:3.3
+  before_script:
+    - bundle install
+  script:
+    - mkdir -p fixtures
+    - |
+      for schema in User Order Product; do
+        bundle exec fake_data_dsl generate $schema \
+          -d schemas -c 100 -s $FAKE_DATA_SEED \
+          --ndjson > fixtures/${schema,,}.ndjson
+      done
+  artifacts:
+    paths:
+      - fixtures/
+    expire_in: 1 week
+  only:
+    - tags
+    - schedules
+```
+
+### Staging Environment Seeding
+
+```yaml
+seed-staging:
+  stage: deploy
+  image: ruby:3.3
+  environment:
+    name: staging
+  before_script:
+    - bundle install
+  script:
+    - |
+      # Generate seed data
+      bundle exec fake_data_dsl generate User \
+        -d schemas -c 1000 -s 12345 \
+        --ndjson > /tmp/users.ndjson
+      
+      # Import to staging database
+      psql $STAGING_DATABASE_URL -c "
+        COPY users (id, name, email, created_at)
+        FROM '/tmp/users.ndjson'
+        WITH (FORMAT csv)
+      "
+  only:
+    - develop
+  when: manual
+```
+
+---
+
+## CircleCI
+
+### Configuration
+
+Create `.circleci/config.yml`:
+
+```yaml
+version: 2.1
+
+orbs:
+  ruby: circleci/ruby@2.1
+
+jobs:
+  validate-schemas:
+    docker:
+      - image: cimg/ruby:3.3
+    steps:
+      - checkout
+      - ruby/install-deps
+      - run:
+          name: Validate schemas
+          command: |
+            bundle exec fake_data_dsl validate schemas/*.dsl
+            bundle exec fake_data_dsl validate spec/schemas/*.dsl
+
+  test:
+    docker:
+      - image: cimg/ruby:3.3
+      - image: cimg/postgres:15.0
+        environment:
+          POSTGRES_USER: test
+          POSTGRES_DB: test_db
+    environment:
+      FAKE_DATA_SEED: << pipeline.number >>
+    steps:
+      - checkout
+      - ruby/install-deps
+      - run:
+          name: Setup database
+          command: bundle exec rails db:setup
+      - run:
+          name: Run tests
+          command: bundle exec rspec
+
+  generate-fixtures:
+    docker:
+      - image: cimg/ruby:3.3
+    steps:
+      - checkout
+      - ruby/install-deps
+      - run:
+          name: Generate fixtures
+          command: |
+            mkdir -p fixtures
+            bundle exec fake_data_dsl generate User -d schemas -c 100 --pretty > fixtures/users.json
+      - store_artifacts:
+          path: fixtures
+          destination: test-fixtures
+
+workflows:
+  test-and-validate:
+    jobs:
+      - validate-schemas
+      - test:
+          requires:
+            - validate-schemas
+      - generate-fixtures:
+          requires:
+            - test
+          filters:
+            branches:
+              only: main
+```
+
+---
+
+## Jenkins
+
+### Jenkinsfile
+
+```groovy
+pipeline {
+    agent any
+    
+    environment {
+        FAKE_DATA_SEED = "${BUILD_NUMBER}"
+    }
+    
+    stages {
+        stage('Setup') {
+            steps {
+                sh 'bundle install'
+            }
+        }
+        
+        stage('Validate Schemas') {
+            steps {
+                sh '''
+                    for file in schemas/*.dsl; do
+                        echo "Validating $file..."
+                        bundle exec fake_data_dsl validate "$file"
+                    done
+                '''
+            }
+        }
+        
+        stage('Test') {
+            steps {
+                sh 'bundle exec rspec --format RspecJunitFormatter --out rspec.xml'
+            }
+            post {
+                always {
+                    junit 'rspec.xml'
+                }
+            }
+        }
+        
+        stage('Generate Fixtures') {
+            when {
+                branch 'main'
+            }
+            steps {
+                sh '''
+                    mkdir -p fixtures
+                    bundle exec fake_data_dsl generate User \
+                        -d schemas -c 100 -s ${FAKE_DATA_SEED} \
+                        --pretty > fixtures/users.json
+                '''
+            }
+            post {
+                success {
+                    archiveArtifacts artifacts: 'fixtures/*.json'
+                }
+            }
+        }
+        
+        stage('Seed Staging') {
+            when {
+                branch 'develop'
+            }
+            steps {
+                withCredentials([string(credentialsId: 'staging-db-url', variable: 'DATABASE_URL')]) {
+                    sh '''
+                        bundle exec fake_data_dsl generate User \
+                            -d schemas -c 1000 -s 12345 \
+                            --ndjson | \
+                        psql ${DATABASE_URL} -c "COPY users FROM STDIN"
+                    '''
+                }
+            }
+        }
+    }
+}
+```
+
+---
+
+## Schema Validation
+
+### Pre-commit Hook
+
+Create `.git/hooks/pre-commit`:
+
+```bash
+#!/bin/bash
+
+# Validate changed schema files
+changed_schemas=$(git diff --cached --name-only --diff-filter=ACM | grep '\.dsl$')
+
+if [ -n "$changed_schemas" ]; then
+    echo "Validating schema files..."
+    
+    for file in $changed_schemas; do
+        echo "  Checking $file..."
+        if ! bundle exec fake_data_dsl validate "$file" 2>&1; then
+            echo "❌ Schema validation failed for $file"
+            exit 1
+        fi
+    done
+    
+    echo "✅ All schemas valid"
+fi
+```
+
+Make it executable:
+
+```bash
+chmod +x .git/hooks/pre-commit
+```
+
+### Validation Script
+
+Create `scripts/validate_schemas.sh`:
+
+```bash
+#!/bin/bash
+set -e
+
+echo "========================================"
+echo "FakeDataDSL Schema Validation"
+echo "========================================"
+
+SCHEMA_DIRS=("schemas" "db/schemas" "spec/schemas")
+ERRORS=0
+
+for dir in "${SCHEMA_DIRS[@]}"; do
+    if [ -d "$dir" ]; then
+        echo ""
+        echo "Checking $dir/..."
+        
+        for file in "$dir"/*.dsl; do
+            if [ -f "$file" ]; then
+                echo -n "  $(basename "$file")... "
+                
+                if bundle exec fake_data_dsl validate "$file" > /dev/null 2>&1; then
+                    echo "✓"
+                else
+                    echo "✗"
+                    bundle exec fake_data_dsl validate "$file" 2>&1 | sed 's/^/    /'
+                    ERRORS=$((ERRORS + 1))
+                fi
+            fi
+        done
+    fi
+done
+
+echo ""
+echo "========================================"
+
+if [ $ERRORS -gt 0 ]; then
+    echo "❌ $ERRORS schema(s) failed validation"
+    exit 1
+else
+    echo "✅ All schemas valid"
+    exit 0
+fi
+```
+
+---
+
+## Test Data Generation
+
+### Fixture Generation Script
+
+Create `scripts/generate_fixtures.rb`:
+
+```ruby
+#!/usr/bin/env ruby
+require 'fake_data_dsl'
+require 'json'
+require 'fileutils'
+
+# Configuration
+SEED = ENV.fetch('SEED', 12345).to_i
+OUTPUT_DIR = ENV.fetch('OUTPUT_DIR', 'fixtures')
+SCHEMAS_DIR = ENV.fetch('SCHEMAS_DIR', 'schemas')
+
+# Records per schema
+COUNTS = {
+  'User' => 100,
+  'Order' => 500,
+  'Product' => 200
+}
+
+# Setup
+FileUtils.mkdir_p(OUTPUT_DIR)
+registry = FakeDataDSL::Registry.new
+registry.load_dir(SCHEMAS_DIR)
+
+puts "Generating fixtures (seed: #{SEED})..."
+
+COUNTS.each do |schema_name, count|
+  output_file = File.join(OUTPUT_DIR, "#{schema_name.downcase}.json")
+  
+  print "  #{schema_name} (#{count} records)... "
+  
+  schema = registry.schema(schema_name)
+  data = schema.generate_many(count, seed: SEED)
+  
+  File.write(output_file, JSON.pretty_generate(data))
+  
+  puts "✓ #{output_file}"
+end
+
+puts "Done!"
+```
+
+### Load Testing Data Generation
+
+Create `scripts/generate_load_test_data.sh`:
+
+```bash
+#!/bin/bash
+set -e
+
+COUNT=${1:-100000}
+SEED=${2:-12345}
+OUTPUT_DIR="load_test_data"
+
+echo "Generating load test data..."
+echo "  Count: $COUNT"
+echo "  Seed: $SEED"
+echo ""
+
+mkdir -p "$OUTPUT_DIR"
+
+# Generate large datasets using streaming
+for schema in User Order Product; do
+    echo "Generating $schema..."
+    bundle exec fake_data_dsl generate "$schema" \
+        -d schemas \
+        -c "$COUNT" \
+        -s "$SEED" \
+        --ndjson > "$OUTPUT_DIR/${schema,,}.ndjson"
+    
+    lines=$(wc -l < "$OUTPUT_DIR/${schema,,}.ndjson")
+    size=$(du -h "$OUTPUT_DIR/${schema,,}.ndjson" | cut -f1)
+    echo "  → $lines records, $size"
+done
+
+echo ""
+echo "Load test data ready in $OUTPUT_DIR/"
+```
+
+---
+
+## Best Practices
+
+### 1. Use Deterministic Seeds
+
+```yaml
+# Use CI variables for reproducible builds
+env:
+  FAKE_DATA_SEED: ${{ github.run_id }}  # GitHub Actions
+  FAKE_DATA_SEED: $CI_PIPELINE_ID       # GitLab
+  FAKE_DATA_SEED: ${BUILD_NUMBER}       # Jenkins
+```
+
+### 2. Cache Dependencies
+
+```yaml
+# GitHub Actions
+- uses: ruby/setup-ruby@v1
+  with:
+    bundler-cache: true
+
+# GitLab CI
+cache:
+  paths:
+    - vendor/bundle
+```
+
+### 3. Fail Fast on Schema Errors
+
+```yaml
+- name: Validate schemas
+  run: |
+    set -e  # Exit on first error
+    bundle exec fake_data_dsl validate schemas/*.dsl
+```
+
+### 4. Generate Artifacts
+
+```yaml
+# Store generated data as artifacts
+- uses: actions/upload-artifact@v4
+  with:
+    name: test-data
+    path: fixtures/
+    retention-days: 7
+```
+
+### 5. Separate Validation and Generation
+
+```yaml
+jobs:
+  validate:
+    # Run on all PRs
+    
+  generate:
+    needs: validate
+    # Only run on main/tags
+```
+
+### 6. Environment-Specific Seeds
+
+```ruby
+# config/initializers/fake_data_dsl.rb
+SEED = case Rails.env
+       when 'test'     then ENV.fetch('FAKE_DATA_SEED', 12345).to_i
+       when 'staging'  then Date.today.to_time.to_i  # Daily variation
+       when 'development' then rand(1..10000)
+       end
+```
+
+### 7. Monitor Generation Time
+
+```yaml
+- name: Generate with timing
+  run: |
+    time bundle exec fake_data_dsl generate User \
+      -d schemas -c 10000 --ndjson > /dev/null
+```
+
+---
+
+## Troubleshooting
+
+### Common Issues
+
+**1. Schema not found**
+```yaml
+# Ensure schemas directory is correct
+- run: ls -la schemas/
+- run: bundle exec fake_data_dsl validate schemas/user.dsl
+```
+
+**2. Memory issues with large datasets**
+```yaml
+# Use streaming for large counts
+- run: |
+    bundle exec fake_data_dsl generate User \
+      -d schemas -c 1000000 \
+      --ndjson > data.ndjson  # Streams to file
+```
+
+**3. Reproducibility issues**
+```yaml
+# Always use explicit seeds
+- run: |
+    bundle exec fake_data_dsl generate User \
+      -d schemas -c 100 \
+      -s 12345  # Explicit seed
+```
+
+---
+
+## Next Steps
+
+- [RSpec Integration](rspec.md) - Testing patterns
+- [Rails Integration](rails.md) - Rails setup
+- [Streaming Generation](../advanced/streaming.md) - Large datasets
+