agentic-flow 2.0.1-alpha.9 → 2.0.2-alpha

package/packages/agentic-jujutsu/benchmarks/docker/Dockerfile.jujutsu

@@ -0,0 +1,67 @@
+# Jujutsu Benchmark Container
+FROM rust:1.75-bookworm
+
+# Avoid interactive prompts
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    build-essential \
+    pkg-config \
+    libssl-dev \
+    curl \
+    time \
+    jq \
+    bc \
+    htop \
+    sysstat \
+    python3 \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Jujutsu from source (latest version)
+RUN cargo install --locked jj-cli --version "^0.21"
+
+# Verify installation
+RUN jj --version
+
+# Install Node.js for coordination scripts
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create workspace
+WORKDIR /benchmark
+
+# Copy benchmark scripts
+COPY scripts/setup-repos.sh /benchmark/scripts/
+COPY scripts/run-benchmarks.sh /benchmark/scripts/
+COPY scripts/collect-metrics.sh /benchmark/scripts/
+COPY config/jj-config.json /benchmark/config/
+
+# Make scripts executable
+RUN chmod +x /benchmark/scripts/*.sh
+
+# Configure Jujutsu
+RUN mkdir -p ~/.config/jj \
+    && echo '[user]' > ~/.config/jj/config.toml \
+    && echo 'name = "Benchmark Agent"' >> ~/.config/jj/config.toml \
+    && echo 'email = "benchmark@agentic-flow.dev"' >> ~/.config/jj/config.toml
+
+# Set up test repository structure
+RUN mkdir -p /repos /results
+
+# Install Python benchmarking tools
+RUN pip3 install --no-cache-dir \
+    psutil \
+    matplotlib \
+    numpy \
+    pandas
+
+# Install agentic-jujutsu WASM (for enhanced features)
+COPY ../../target/wasm32-unknown-unknown/release/*.wasm /benchmark/wasm/ 2>/dev/null || true
+
+VOLUME ["/repos", "/results"]
+
+CMD ["/benchmark/scripts/run-benchmarks.sh", "jj"]

package/packages/agentic-jujutsu/benchmarks/docker/Dockerfile.swarm-coordinator

@@ -0,0 +1,45 @@
+# Swarm Coordinator Container
+FROM node:20-bookworm
+
+# Avoid interactive prompts
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    jq \
+    bc \
+    python3 \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create workspace
+WORKDIR /coordinator
+
+# Install agentic-flow and dependencies
+RUN npm install -g \
+    claude-flow@alpha \
+    agentic-flow@latest \
+    pnpm
+
+# Copy coordinator scripts
+COPY scripts/swarm-coordinator.js /coordinator/
+COPY config/swarm-config.json /coordinator/config/
+
+# Install Python analysis tools
+RUN pip3 install --no-cache-dir \
+    psutil \
+    matplotlib \
+    numpy \
+    pandas \
+    scipy
+
+# Create directories
+RUN mkdir -p /data/results /data/metrics /data/logs
+
+VOLUME ["/data"]
+
+# Expose coordination API
+EXPOSE 8080
+
+CMD ["node", "/coordinator/swarm-coordinator.js"]

package/packages/agentic-jujutsu/benchmarks/docker/config/prometheus.yml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'swarm-coordinator'
+    static_configs:
+      - targets: ['swarm-coordinator:8080']
+
+  - job_name: 'agentdb'
+    static_configs:
+      - targets: ['agentdb:5432']
+
+  - job_name: 'git-benchmark'
+    static_configs:
+      - targets: ['git-benchmark:9100']
+
+  - job_name: 'jj-benchmark'
+    static_configs:
+      - targets: ['jj-benchmark:9100']

package/packages/agentic-jujutsu/benchmarks/docker/docker-compose.yml

@@ -0,0 +1,152 @@
+version: '3.8'
+
+services:
+  # Git baseline benchmark container
+  git-benchmark:
+    build:
+      context: .
+      dockerfile: Dockerfile.git
+    container_name: agentic-jj-git-bench
+    hostname: git-bench
+    volumes:
+      - ./data/repos:/repos
+      - ./data/results:/results
+      - ./scripts:/benchmark/scripts:ro
+      - ./config:/benchmark/config:ro
+    environment:
+      - TEST_SCENARIO=${TEST_SCENARIO:-concurrent}
+      - AGENT_COUNT=${AGENT_COUNT:-10}
+      - COMMIT_COUNT=${COMMIT_COUNT:-100}
+      - CONFLICT_RATE=${CONFLICT_RATE:-10}
+      - ENABLE_METRICS=true
+      - BENCHMARK_DURATION=${BENCHMARK_DURATION:-600}
+    networks:
+      - benchmark-net
+    depends_on:
+      - swarm-coordinator
+    mem_limit: 4g
+    cpus: 2
+
+  # Jujutsu benchmark container
+  jj-benchmark:
+    build:
+      context: .
+      dockerfile: Dockerfile.jujutsu
+    container_name: agentic-jj-bench
+    hostname: jj-bench
+    volumes:
+      - ./data/repos:/repos
+      - ./data/results:/results
+      - ./scripts:/benchmark/scripts:ro
+      - ./config:/benchmark/config:ro
+    environment:
+      - TEST_SCENARIO=${TEST_SCENARIO:-concurrent}
+      - AGENT_COUNT=${AGENT_COUNT:-10}
+      - COMMIT_COUNT=${COMMIT_COUNT:-100}
+      - CONFLICT_RATE=${CONFLICT_RATE:-10}
+      - ENABLE_METRICS=true
+      - BENCHMARK_DURATION=${BENCHMARK_DURATION:-600}
+      - JJ_WORKSPACE_MODE=${JJ_WORKSPACE_MODE:-shared-log}
+    networks:
+      - benchmark-net
+    depends_on:
+      - swarm-coordinator
+    mem_limit: 4g
+    cpus: 2
+
+  # Swarm coordinator for orchestration
+  swarm-coordinator:
+    build:
+      context: .
+      dockerfile: Dockerfile.swarm-coordinator
+    container_name: agentic-jj-coordinator
+    hostname: coordinator
+    volumes:
+      - ./data:/data
+      - ./scripts:/coordinator/scripts:ro
+      - ./config:/coordinator/config:ro
+    environment:
+      - COORDINATOR_MODE=benchmark
+      - ENABLE_LOGGING=true
+      - LOG_LEVEL=info
+      - API_PORT=8080
+    networks:
+      - benchmark-net
+    ports:
+      - "8080:8080"
+    mem_limit: 2g
+    cpus: 1
+
+  # AgentDB for pattern learning and metrics storage
+  agentdb:
+    image: agentdb:latest
+    container_name: agentic-jj-agentdb
+    hostname: agentdb
+    volumes:
+      - agentdb-data:/data
+      - ./data/metrics:/metrics
+    environment:
+      - AGENTDB_MODE=benchmark
+      - ENABLE_VECTOR_SEARCH=true
+      - ENABLE_HNSW_INDEX=true
+      - QUANTIZATION_LEVEL=8
+    networks:
+      - benchmark-net
+    ports:
+      - "5432:5432"
+    mem_limit: 4g
+    cpus: 2
+
+  # Metrics collection and visualization
+  metrics-collector:
+    image: prom/prometheus:latest
+    container_name: agentic-jj-metrics
+    hostname: metrics
+    volumes:
+      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - metrics-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=30d'
+    networks:
+      - benchmark-net
+    ports:
+      - "9090:9090"
+    mem_limit: 1g
+    cpus: 0.5
+
+  # Visualization dashboard
+  grafana:
+    image: grafana/grafana:latest
+    container_name: agentic-jj-grafana
+    hostname: grafana
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./config/grafana-dashboards:/etc/grafana/provisioning/dashboards:ro
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    networks:
+      - benchmark-net
+    ports:
+      - "3000:3000"
+    depends_on:
+      - metrics-collector
+    mem_limit: 512m
+    cpus: 0.5
+
+networks:
+  benchmark-net:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.28.0.0/16
+
+volumes:
+  agentdb-data:
+    driver: local
+  metrics-data:
+    driver: local
+  grafana-data:
+    driver: local

package/packages/agentic-jujutsu/benchmarks/docker/scripts/collect-metrics.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+set -e
+
+VCS_TYPE=${1:-git}
+TEST_SCENARIO=${2:-concurrent}
+
+RESULTS_DIR="/results/${VCS_TYPE}/${TEST_SCENARIO}"
+METRICS_FILE="${RESULTS_DIR}/metrics.json"
+
+echo "=== Collecting Metrics ==="
+echo "VCS: ${VCS_TYPE}"
+echo "Scenario: ${TEST_SCENARIO}"
+echo ""
+
+# Initialize metrics file
+cat > "${METRICS_FILE}" <<EOF
+{
+  "vcs_type": "${VCS_TYPE}",
+  "test_scenario": "${TEST_SCENARIO}",
+  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+  "system": {},
+  "performance": {},
+  "throughput": {},
+  "latency": {},
+  "resource_usage": {}
+}
+EOF
+
+# Collect system information
+echo "Collecting system information..."
+cat >> "${RESULTS_DIR}/system-info.txt" <<EOF
+=== System Information ===
+Hostname: $(hostname)
+Kernel: $(uname -r)
+CPU: $(nproc) cores
+Memory: $(free -h | grep Mem | awk '{print $2}')
+Disk: $(df -h / | tail -1 | awk '{print $2}')
+
+=== Software Versions ===
+EOF
+
+if [ "$VCS_TYPE" = "git" ]; then
+    git --version >> "${RESULTS_DIR}/system-info.txt"
+else
+    jj --version >> "${RESULTS_DIR}/system-info.txt"
+fi
+
+# Collect resource usage statistics
+echo "Collecting resource usage..."
+{
+    echo "=== CPU Usage (%) ==="
+    mpstat 1 1 | tail -1 | awk '{print 100 - $12}'
+
+    echo ""
+    echo "=== Memory Usage (MB) ==="
+    free -m | grep Mem | awk '{print $3}'
+
+    echo ""
+    echo "=== Disk I/O ==="
+    iostat -x 1 1 | tail -n +4
+} > "${RESULTS_DIR}/resource-usage.txt"
+
+# Parse benchmark timing results
+if [ -f "${RESULTS_DIR}/timings.csv" ]; then
+    echo "Processing timing data..."
+
+    # Calculate statistics using Python
+    python3 -c "
+import csv
+import json
+import statistics
+
+timings = []
+with open('${RESULTS_DIR}/timings.csv', 'r') as f:
+    reader = csv.DictReader(f)
+    for row in reader:
+        timings.append(float(row['duration_ms']))
+
+if timings:
+    stats = {
+        'count': len(timings),
+        'mean': statistics.mean(timings),
+        'median': statistics.median(timings),
+        'stdev': statistics.stdev(timings) if len(timings) > 1 else 0,
+        'min': min(timings),
+        'max': max(timings),
+        'p50': statistics.median(timings),
+        'p95': sorted(timings)[int(len(timings) * 0.95)] if len(timings) > 20 else max(timings),
+        'p99': sorted(timings)[int(len(timings) * 0.99)] if len(timings) > 100 else max(timings),
+    }
+
+    with open('${RESULTS_DIR}/timing-stats.json', 'w') as f:
+        json.dump(stats, f, indent=2)
+
+    print(json.dumps(stats, indent=2))
+"
+fi
+
+# Calculate throughput
+if [ -f "${RESULTS_DIR}/throughput.txt" ]; then
+    echo "Calculating throughput metrics..."
+
+    TOTAL_OPS=$(cat "${RESULTS_DIR}/throughput.txt" | wc -l)
+    TOTAL_TIME=$(tail -1 "${RESULTS_DIR}/throughput.txt" | awk '{print $2}')
+
+    if [ -n "$TOTAL_TIME" ] && [ "$TOTAL_TIME" != "0" ]; then
+        OPS_PER_SEC=$(echo "scale=2; $TOTAL_OPS / $TOTAL_TIME" | bc)
+        echo "Operations per second: ${OPS_PER_SEC}" >> "${RESULTS_DIR}/metrics-summary.txt"
+    fi
+fi
+
+# Merge all metrics into final JSON
+python3 -c "
+import json
+import os
+
+metrics = {}
+results_dir = '${RESULTS_DIR}'
+
+# Load existing metrics
+if os.path.exists('${METRICS_FILE}'):
+    with open('${METRICS_FILE}', 'r') as f:
+        metrics = json.load(f)
+
+# Load timing stats if available
+timing_stats_path = os.path.join(results_dir, 'timing-stats.json')
+if os.path.exists(timing_stats_path):
+    with open(timing_stats_path, 'r') as f:
+        metrics['latency'] = json.load(f)
+
+# Save merged metrics
+with open('${METRICS_FILE}', 'w') as f:
+    json.dump(metrics, f, indent=2)
+
+print('Metrics collection complete!')
+print(f'Results saved to: ${METRICS_FILE}')
+"
+
+echo ""
+echo "=== Metrics Collection Complete ==="
+echo "Metrics file: ${METRICS_FILE}"
+echo ""
+cat "${METRICS_FILE}"

package/packages/agentic-jujutsu/benchmarks/docker/scripts/generate-reports.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+set -e
+
+RESULTS_BASE="/results"
+REPORT_DIR="/results/reports"
+
+mkdir -p "${REPORT_DIR}"
+
+echo "========================================="
+echo "  Generating Benchmark Reports"
+echo "========================================="
+echo ""
+
+# Generate comparison report
+python3 <<EOF
+import json
+import os
+from pathlib import Path
+
+results_base = Path("${RESULTS_BASE}")
+report_dir = Path("${REPORT_DIR}")
+
+# Collect all metrics
+all_metrics = {}
+
+for vcs_dir in results_base.glob("*/"):
+    vcs_type = vcs_dir.name
+    if vcs_type == "reports":
+        continue
+
+    all_metrics[vcs_type] = {}
+
+    for scenario_dir in vcs_dir.glob("*/"):
+        scenario = scenario_dir.name
+        metrics_file = scenario_dir / "metrics.json"
+
+        if metrics_file.exists():
+            with open(metrics_file) as f:
+                all_metrics[vcs_type][scenario] = json.load(f)
+
+# Generate comparison report
+with open(report_dir / "comparison.md", "w") as f:
+    f.write("# Benchmark Comparison Report\n\n")
+    f.write("## Overview\n\n")
+
+    for vcs_type, scenarios in all_metrics.items():
+        f.write(f"### {vcs_type.upper()}\n\n")
+
+        for scenario, metrics in scenarios.items():
+            f.write(f"#### Scenario: {scenario}\n\n")
+
+            if "latency" in metrics:
+                lat = metrics["latency"]
+                f.write("**Latency Statistics:**\n\n")
+                f.write(f"- Mean: {lat.get('mean', 0):.2f}ms\n")
+                f.write(f"- Median (P50): {lat.get('p50', 0):.2f}ms\n")
+                f.write(f"- P95: {lat.get('p95', 0):.2f}ms\n")
+                f.write(f"- P99: {lat.get('p99', 0):.2f}ms\n")
+                f.write(f"- Min: {lat.get('min', 0):.2f}ms\n")
+                f.write(f"- Max: {lat.get('max', 0):.2f}ms\n\n")
+
+            f.write("---\n\n")
+
+    # Comparison table
+    f.write("## Performance Comparison\n\n")
+    f.write("| VCS | Scenario | Mean Latency | P95 Latency | Throughput |\n")
+    f.write("|-----|----------|--------------|-------------|------------|\n")
+
+    for vcs_type, scenarios in all_metrics.items():
+        for scenario, metrics in scenarios.items():
+            lat = metrics.get("latency", {})
+            mean = lat.get("mean", 0)
+            p95 = lat.get("p95", 0)
+            throughput = metrics.get("throughput", {}).get("ops_per_sec", 0)
+
+            f.write(f"| {vcs_type} | {scenario} | {mean:.2f}ms | {p95:.2f}ms | {throughput:.2f} ops/s |\n")
+
+print("Comparison report generated: ${REPORT_DIR}/comparison.md")
+
+# Generate detailed JSON report
+with open(report_dir / "full-results.json", "w") as f:
+    json.dump(all_metrics, f, indent=2)
+
+print("Full results JSON: ${REPORT_DIR}/full-results.json")
+EOF
+
+# Generate visualizations
+python3 <<EOF
+import json
+import matplotlib.pyplot as plt
+import matplotlib
+matplotlib.use('Agg')
+from pathlib import Path
+
+results_base = Path("${RESULTS_BASE}")
+report_dir = Path("${REPORT_DIR}")
+
+# Load full results
+with open(report_dir / "full-results.json") as f:
+    all_metrics = json.load(f)
+
+# Plot latency comparison
+fig, ax = plt.subplots(figsize=(12, 6))
+
+vcs_types = []
+scenarios = set()
+latency_data = {}
+
+for vcs_type, vcs_scenarios in all_metrics.items():
+    vcs_types.append(vcs_type)
+    for scenario, metrics in vcs_scenarios.items():
+        scenarios.add(scenario)
+        key = f"{vcs_type}_{scenario}"
+        latency_data[key] = metrics.get("latency", {}).get("mean", 0)
+
+# Create bar chart
+x_pos = range(len(latency_data))
+plt.bar(x_pos, latency_data.values())
+plt.xticks(x_pos, latency_data.keys(), rotation=45, ha='right')
+plt.ylabel('Mean Latency (ms)')
+plt.title('Latency Comparison Across VCS and Scenarios')
+plt.tight_layout()
+plt.savefig(report_dir / "latency-comparison.png", dpi=300)
+print("Latency comparison chart: ${REPORT_DIR}/latency-comparison.png")
+
+# Plot throughput comparison
+fig, ax = plt.subplots(figsize=(12, 6))
+throughput_data = {}
+
+for vcs_type, vcs_scenarios in all_metrics.items():
+    for scenario, metrics in vcs_scenarios.items():
+        key = f"{vcs_type}_{scenario}"
+        throughput_data[key] = metrics.get("throughput", {}).get("ops_per_sec", 0)
+
+x_pos = range(len(throughput_data))
+plt.bar(x_pos, throughput_data.values(), color='green')
+plt.xticks(x_pos, throughput_data.keys(), rotation=45, ha='right')
+plt.ylabel('Operations per Second')
+plt.title('Throughput Comparison Across VCS and Scenarios')
+plt.tight_layout()
+plt.savefig(report_dir / "throughput-comparison.png", dpi=300)
+print("Throughput comparison chart: ${REPORT_DIR}/throughput-comparison.png")
+EOF
+
+echo ""
+echo "========================================="
+echo "  Report Generation Complete!"
+echo "========================================="
+echo "Reports available at: ${REPORT_DIR}/"
+ls -lh "${REPORT_DIR}/"

package/packages/agentic-jujutsu/benchmarks/docker/scripts/run-benchmarks.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -e
+
+VCS_TYPE=${1:-git}
+TEST_SCENARIO=${TEST_SCENARIO:-concurrent}
+AGENT_COUNT=${AGENT_COUNT:-10}
+COMMIT_COUNT=${COMMIT_COUNT:-100}
+CONFLICT_RATE=${CONFLICT_RATE:-10}
+BENCHMARK_DURATION=${BENCHMARK_DURATION:-600}
+
+RESULTS_DIR="/results/${VCS_TYPE}/${TEST_SCENARIO}"
+mkdir -p "${RESULTS_DIR}"
+
+echo "========================================="
+echo "  Agentic-Jujutsu Benchmark Suite"
+echo "========================================="
+echo "VCS Type: ${VCS_TYPE}"
+echo "Scenario: ${TEST_SCENARIO}"
+echo "Agents: ${AGENT_COUNT}"
+echo "Commits per agent: ${COMMIT_COUNT}"
+echo "Conflict rate: ${CONFLICT_RATE}%"
+echo "Max duration: ${BENCHMARK_DURATION}s"
+echo "========================================="
+echo ""
+
+# Set up repository
+bash /benchmark/scripts/setup-repos.sh "${VCS_TYPE}" "/repos/test-repo"
+
+# Run appropriate benchmark scenario
+case "${TEST_SCENARIO}" in
+    concurrent)
+        echo "Running Concurrent Commit Storm benchmark..."
+        bash /benchmark/tests/concurrent-commits.sh "${VCS_TYPE}"
+        ;;
+
+    workspace-isolation)
+        echo "Running Workspace Isolation benchmark..."
+        bash /benchmark/tests/workspace-isolation.sh "${VCS_TYPE}"
+        ;;
+
+    conflict-resolution)
+        echo "Running Conflict Resolution benchmark..."
+        bash /benchmark/tests/conflict-resolution.sh "${VCS_TYPE}"
+        ;;
+
+    operation-log-sync)
+        echo "Running Operation Log Synchronization benchmark..."
+        bash /benchmark/tests/operation-log-sync.sh "${VCS_TYPE}"
+        ;;
+
+    real-world)
+        echo "Running Real-World Development Workflow benchmark..."
+        bash /benchmark/tests/real-world-workflow.sh "${VCS_TYPE}"
+        ;;
+
+    *)
+        echo "Unknown scenario: ${TEST_SCENARIO}"
+        exit 1
+        ;;
+esac
+
+# Collect metrics
+echo ""
+echo "Collecting metrics..."
+bash /benchmark/scripts/collect-metrics.sh "${VCS_TYPE}" "${TEST_SCENARIO}"
+
+# Generate report
+echo ""
+echo "Generating report..."
+python3 /benchmark/scripts/generate-report.py \
+    --vcs "${VCS_TYPE}" \
+    --scenario "${TEST_SCENARIO}" \
+    --results-dir "${RESULTS_DIR}"
+
+echo ""
+echo "========================================="
+echo "  Benchmark Complete!"
+echo "========================================="
+echo "Results saved to: ${RESULTS_DIR}"
+ls -lh "${RESULTS_DIR}"