dialectic 0.1.0

package/examples/kata1/debate-config-kata1.json

@@ -0,0 +1,54 @@
+{
+  "agents": [
+    {
+      "id": "agent-architect",
+      "name": "System Architect",
+      "role": "architect",
+      "model": "anthropic/claude-sonnet-4.5",
+      "provider": "openrouter",
+      "temperature": 0.5,
+      "enabled": true
+    },
+    {
+      "id": "agent-performance",
+      "name": "Performance Engineer",
+      "role": "performance",
+      "model": "anthropic/claude-sonnet-4.5",
+      "provider": "openrouter",
+      "temperature": 0.6,
+      "enabled": true
+    },
+    {
+      "id": "agent-security",
+      "name": "Security Specialist",
+      "role": "security",
+      "model": "gpt-4",
+      "provider": "openai",
+      "temperature": 0.4,
+      "enabled": true
+    }
+  ],
+  "judge": {
+    "id": "judge-main",
+    "name": "Technical Judge",
+    "role": "generalist",
+    "model": "anthropic/claude-3-opus",
+    "provider": "openrouter",
+    "temperature": 0.5
+  },
+  "debate": {
+    "rounds": 3,
+    "terminationCondition": {
+      "type": "fixed"
+    },
+    "synthesisMethod": "judge",
+    "includeFullHistory": true,
+    "timeoutPerRound": 300000,
+    "summarization": {
+      "enabled": true,
+      "threshold": 2500,
+      "maxLength": 1250,
+      "method": "length-based"
+    }
+  }
+}

package/examples/kata1/eval_architect-only_2-rounds.json

@@ -0,0 +1,97 @@
+{
+  "evaluation": {
+    "functional_completeness": {
+      "average_score": 10
+    },
+    "non_functional": {
+      "performance_scalability": {
+        "average_score": 10
+      },
+      "security": {
+        "average_score": 10
+      },
+      "maintainability_evolvability": {
+        "average_score": 10
+      },
+      "regulatory_compliance": {
+        "average_score": 9
+      },
+      "testability": {
+        "average_score": 9
+      }
+    }
+  },
+  "overall_score": 10,
+  "agents": {
+    "eval-1": {
+      "evaluation": {
+        "functional_completeness": {
+          "score": 10,
+          "reasoning": "The solution addresses all explicit requirements: student code upload, automated running/grading, persistence/auditability (PostgreSQL, immutable logs), plagiarism detection (MinHash + TurnItIn), and LMS integration (LTI 1.3). It also covers implied requirements like handling different programming languages and managing submission spikes."
+        },
+        "non_functional": {
+          "performance_scalability": {
+            "score": 10,
+            "reasoning": "The design is highly scalable, utilizing an event-driven architecture (Kafka/NATS), Kubernetes HPA, auto-scaling node pools, and presigned S3 uploads to handle deadline spikes (150-300 submissions in minutes). It explicitly details capacity planning and cost optimization (spot instances)."
+          },
+          "security": {
+            "score": 10,
+            "reasoning": "Security is a core design principle, employing defense-in-depth. It uses gVisor/Firecracker for kernel-level isolation of untrusted code, Kubernetes NetworkPolicies, TLS everywhere, Vault for secrets, and RBAC. This is a robust security posture for running arbitrary user code."
+          },
+          "maintainability_evolvability": {
+            "score": 10,
+            "reasoning": "The solution is built on event-driven microservices with clear component boundaries, which maximizes maintainability and reduces the blast radius of changes. The use of GitOps (ArgoCD) and a detailed operational model with runbooks and observability (OpenTelemetry, Prometheus) ensures high evolvability and ease of troubleshooting."
+          },
+          "regulatory_compliance": {
+            "score": 9,
+            "reasoning": "The solution explicitly mentions GDPR compliance via data anonymization and aligns retention policies with university governance (7 years). It includes an immutable audit log, which is crucial for compliance and auditing requirements."
+          },
+          "testability": {
+            "score": 9,
+            "reasoning": "The microservice architecture and clear interfaces (Kafka events) inherently support isolated testing. The GitOps pipeline includes CI/CD with unit and integration tests, and the roadmap includes automated load testing. The use of standard cloud components also simplifies mocking and environment setup."
+          }
+        }
+      },
+      "overall_summary": {
+        "strengths": "Exceptional depth in security (gVisor/Firecracker), scalability (event-driven, Kubernetes), and operations (GitOps, detailed observability stack). All requirements are met with production-ready, battle-tested patterns.",
+        "weaknesses": "The complexity of the proposed stack (Kafka, NATS, Kubernetes, gVisor) presents a high initial operational learning curve for the university's IT team, as acknowledged in the summary.",
+        "overall_score": 10
+      }
+    },
+    "eval-2": {
+      "evaluation": {
+        "functional_completeness": {
+          "score": 10,
+          "reasoning": "The solution addresses all explicit requirements: student code upload, automated running and grading, persistence and auditability (immutable audit log, retention policy), plagiarism detection (two-tier MinHash and TurnItIn integration), and LMS integration (LTI 1.3 Adapter). Edge cases like large files (presigned URLs) and resource contention (resource classes, quotas) are also covered."
+        },
+        "non_functional": {
+          "performance_scalability": {
+            "score": 10,
+            "reasoning": "Scalability is a core design principle. The architecture uses horizontal scaling (HPA, auto-scaled components), an event-driven model (Kafka/NATS) to buffer load spikes, and specific strategies for high-load scenarios (pre-warming node pools, increased Kafka partitions). Presigned uploads prevent the platform from becoming a network bottleneck. Cost optimization (Spot instances) is also explicitly addressed."
+          },
+          "security": {
+            "score": 10,
+            "reasoning": "Security is addressed with a robust defense-in-depth strategy. This includes kernel-level isolation for untrusted code (gVisor/Firecracker), network segmentation (Kubernetes NetworkPolicies), resource protection (PodSecurityPolicies, ResourceQuotas), data encryption (TLS, SSE-S3), and strong access control (Vault, RBAC). The solution explicitly mitigates the critical risk of sandbox escape."
+          },
+          "maintainability_evolvability": {
+            "score": 10,
+            "reasoning": "The solution is highly maintainable and evolvable due to its microservices architecture, clear component boundaries, and event-driven communication. GitOps (ArgoCD) ensures declarative, repeatable deployments and easy rollbacks. The roadmap explicitly plans for future language support (Java, C++, JS), demonstrating foresight for evolution. Observability (OpenTelemetry, Prometheus, Loki) is built-in, aiding troubleshooting."
+          },
+          "regulatory_compliance": {
+            "score": 9,
+            "reasoning": "The solution explicitly mentions GDPR compliance via data anonymization after course completion and aligns retention policies with university governance (7 years). It also includes an immutable audit log of state transitions, which is crucial for compliance and auditing requirements. The use of LTI 1.3 standard for LMS integration also promotes secure data handling."
+          },
+          "testability": {
+            "score": 9,
+            "reasoning": "The microservices architecture and clear interfaces (Kafka events) allow for isolated testing of components (e.g., LTI Adapter, Grading Engine). The GitOps pipeline includes automated unit and integration tests, and the deployment strategy uses Canary rollouts, which is excellent for testing changes in production safely. The roadmap includes automated load testing, indicating a strong focus on non-functional testing."
+          }
+        }
+      },
+      "overall_summary": {
+        "strengths": "Exceptional depth in security (gVisor/Firecracker), scalability (event-driven, HPA), and operational maturity (GitOps, Observability). All functional requirements are met with robust, production-ready patterns.",
+        "weaknesses": "The complexity of the proposed stack (Kubernetes, Kafka, NATS, gVisor) implies a high initial operational learning curve for the university's IT team, which is acknowledged as a risk.",
+        "overall_score": 10
+      }
+    }
+  }
+}

package/examples/kata1/eval_architect-perf_2-rounds.json

@@ -0,0 +1,97 @@
+{
+  "evaluation": {
+    "functional_completeness": {
+      "average_score": 10
+    },
+    "non_functional": {
+      "performance_scalability": {
+        "average_score": 10
+      },
+      "security": {
+        "average_score": 10
+      },
+      "maintainability_evolvability": {
+        "average_score": 9
+      },
+      "regulatory_compliance": {
+        "average_score": 9.5
+      },
+      "testability": {
+        "average_score": 10
+      }
+    }
+  },
+  "overall_score": 10,
+  "agents": {
+    "eval-1": {
+      "evaluation": {
+        "functional_completeness": {
+          "score": 10,
+          "reasoning": "The solution addresses all stated requirements: student upload, running/grading, persistent/auditable grades/runs (WORM storage, immutable audit trail), plagiarism detection (MinHash-LSH, TurnItIn integration), and LMS integration (implied via API Gateway/LMS/SPA block). It also covers implied requirements like performance, security, and operational resilience."
+        },
+        "non_functional": {
+          "performance_scalability": {
+            "score": 10,
+            "reasoning": "The design explicitly targets high performance and scalability using event-driven architecture (Kafka), presigned URLs to offload the gateway, warm container pools for low latency execution, detailed throughput targets (10x baseline), and specific optimizations like weighted Kafka lag and multi-level caching. It provides concrete SLA targets and a validation plan."
+          },
+          "security": {
+            "score": 10,
+            "reasoning": "Security is thoroughly addressed, focusing on sandbox hardening (gVisor/Firecracker, seccomp, network isolation), data protection (mTLS, encryption at rest, IAM), and image integrity (Docker Content Trust). The use of WORM storage also enhances audit security."
+          },
+          "maintainability_evolvability": {
+            "score": 9,
+            "reasoning": "The microservices and event-driven architecture promote clear separation of concerns (Submission, Sandbox, Grading, Plagiarism services). The use of Kubernetes, GitOps, and clear component specifications enhances maintainability. The design explicitly considers future enhancements (Nice-to-Have section), demonstrating evolvability, though the complexity of the Kafka/Outbox pattern adds a slight operational burden."
+          },
+          "regulatory_compliance": {
+            "score": 9,
+            "reasoning": "The solution explicitly mentions GDPR/FERPA compliance, addressing data hashing, subject access tooling, and configurable retention. The use of WORM storage (S3 Object Lock) directly addresses the auditability and non-tampering requirements essential for accreditation/compliance."
+          },
+          "testability": {
+            "score": 10,
+            "reasoning": "Testability is a core consideration, with clear component boundaries, a detailed performance testing plan (load, burst, mixed payload), and a dedicated section on Chaos Engineering to validate resilience. The use of a dedicated worker pool with a circuit breaker for the 3rd party TurnItIn integration ensures isolated testing and graceful degradation."
+          }
+        }
+      },
+      "overall_summary": {
+        "strengths": "Exceptional detail in addressing non-functional requirements, particularly performance, security, and auditability. The use of advanced patterns like weighted Kafka lag, warm container pools, and MinHash-LSH demonstrates a deep understanding of the problem domain and operational excellence. The solution is highly structured and includes a comprehensive implementation roadmap and risk mitigation plan.",
+        "weaknesses": "The complexity introduced by the Outbox pattern and managing a highly tuned Kafka cluster might require specialized expertise, potentially exceeding the 'operational complexity budget' of a typical university IT department, despite the claim.",
+        "overall_score": 10
+      }
+    },
+    "eval-2": {
+      "evaluation": {
+        "functional_completeness": {
+          "score": 10,
+          "reasoning": "The solution addresses all explicit requirements: student upload, persistent/auditable grades and runs (WORM storage, immutable audit trail), plagiarism detection (MinHash-LSH, TurnItIn integration), and LMS integration (implied via API Gateway/LMS/SPA component). It also covers implied requirements like resource isolation (sandbox) and high availability (HA PostgreSQL, Kafka replication)."
+        },
+        "non_functional": {
+          "performance_scalability": {
+            "score": 10,
+            "reasoning": "The solution is highly optimized for performance and scalability. Key decisions like Presigned URL uploads, Event-Driven architecture (Kafka), Warm container pools, Weighted Kafka Lag scaling, and MinHash-LSH for plagiarism directly address potential bottlenecks and provide significant headroom (10x peak capacity). Concrete SLA targets and throughput targets are provided."
+          },
+          "security": {
+            "score": 10,
+            "reasoning": "Security is comprehensively addressed, particularly the critical sandbox isolation (gVisor/Firecracker, seccomp, Falco monitoring, network isolation). Data protection includes mTLS, AES-256 encryption at rest, and granular IAM access control. Image integrity via Docker Content Trust is also included."
+          },
+          "maintainability_evolvability": {
+            "score": 9,
+            "reasoning": "The microservices and event-driven architecture promote high maintainability and low coupling. The use of GitOps (ArgoCD), clear component boundaries, and an observability stack (Prometheus, Grafana, OpenTelemetry) supports troubleshooting and evolution. The Outbox pattern ensures reliable state transitions. The only minor deduction is that the daily partitioning of the grades table, while good for performance, might require occasional maintenance for schema evolution."
+          },
+          "regulatory_compliance": {
+            "score": 10,
+            "reasoning": "The solution explicitly addresses regulatory compliance (GDPR/FERPA) through data hashing, configurable retention policies (5 years minimum), WORM storage (S3 Object Lock) for auditability, and tooling for data subject access. This is a strong and explicit coverage of compliance needs."
+          },
+          "testability": {
+            "score": 10,
+            "reasoning": "Testability is excellent. The component decomposition allows for isolated testing. The solution explicitly details performance testing (sustained, burst, mixed payload), security testing, and includes a robust plan for Chaos Engineering to validate resilience. The use of clear SLAs and metrics facilitates continuous validation."
+          }
+        }
+      },
+      "overall_summary": {
+        "strengths": "The solution is exceptionally detailed, covering all functional and non-functional requirements with high-confidence, production-ready patterns (Event-Driven, Outbox, Warm Pools). It provides concrete performance targets, robust security measures (sandbox hardening), and a clear, phased implementation roadmap.",
+        "weaknesses": "The complexity of integrating MinHash-LSH and the reliance on specific Kafka expertise are minor operational challenges, but these are acknowledged and mitigated in the plan.",
+        "overall_score": 10
+      }
+    }
+  }
+}