@nbardy/oompa 0.1.0

package/agentnet/src/agentnet/orchestrator.clj

@@ -0,0 +1,367 @@
+(ns agentnet.orchestrator
+  "Main orchestration loop coordinating agents, worktrees, and merges.
+
+   The orchestrator manages the full lifecycle:
+     1. Initialize worktree pool
+     2. Load task queue
+     3. Dispatch tasks to workers
+     4. Run propose/review loops
+     5. Merge approved changes
+     6. Log results
+
+   Design:
+     - Workers are lightweight (just state, not threads)
+     - core.async for parallelism
+     - Each worker owns one worktree
+     - Tasks flow: queue -> worker -> review -> merge -> done
+
+   Concurrency Model:
+     - N workers, N worktrees
+     - Each worker processes one task at a time
+     - Workers can run in parallel (different worktrees)
+     - Merges are serialized (one at a time to main)"
+  (:require [agentnet.schema :as schema]
+            [agentnet.core :as core]
+            [agentnet.agent :as agent]
+            [agentnet.worktree :as worktree]
+            [agentnet.review :as review]
+            [agentnet.merge :as merge]
+            [clojure.core.async :as async]
+            [clojure.edn :as edn]
+            [clojure.java.io :as io]
+            [cheshire.core :as json]))
+
+;; =============================================================================
+;; Function Specs
+;; =============================================================================
+
+;; create-orchestrator : OrchestratorConfig -> OrchestratorState
+;; Initialize orchestrator with workers and worktree pool
+
+;; run! : OrchestratorState -> OrchestratorState
+;; Process all tasks to completion
+
+;; process-task! : Worker, Task, Context -> TaskResult
+;; Full lifecycle for one task (propose, review, merge)
+
+;; shutdown! : OrchestratorState -> nil
+;; Clean up resources
+
+;; =============================================================================
+;; Constants
+;; =============================================================================
+
+(def ^:const TASKS_PATH "config/tasks.edn")
+(def ^:const POLICY_PATH "config/policy.edn")
+(def ^:const RUNS_DIR "runs")
+
+;; =============================================================================
+;; State Management
+;; =============================================================================
+
+(defn- now-ms []
+  (System/currentTimeMillis))
+
+(defn- read-edn [path]
+  (let [f (io/file path)]
+    (when (.exists f)
+      (with-open [r (java.io.PushbackReader. (io/reader f))]
+        (edn/read {:eof nil} r)))))
+
+(defn- load-tasks []
+  (or (read-edn TASKS_PATH) []))
+
+(defn- load-policy []
+  (or (read-edn POLICY_PATH)
+      {:allow ["src/**" "tests/**"]
+       :deny ["secrets/**" "**/*.pem"]
+       :limits {:max-lines-added 800
+                :max-review-attempts 5}}))
+
+;; =============================================================================
+;; Worker Management
+;; =============================================================================
+
+(defn- create-worker [id worktree-instance]
+  {:id id
+   :status :idle
+   :worktree worktree-instance
+   :current-task nil
+   :review-loop nil})
+
+(defn- assign-task [worker task]
+  (assoc worker
+         :status :proposing
+         :current-task task))
+
+(defn- complete-task [worker result]
+  (assoc worker
+         :status :idle
+         :current-task nil
+         :review-loop nil
+         :last-result result))
+
+;; =============================================================================
+;; Task Processing
+;; =============================================================================
+
+(defn- build-context [tasks]
+  (core/build-context {:tasks tasks
+                       :policy (load-policy)
+                       :recent-sec 180}))
+
+(defn process-task!
+  "Process a single task through full lifecycle.
+
+   Steps:
+     1. Run propose/review loop until approved
+     2. Merge approved changes to main
+     3. Clean up worktree
+
+   Returns TaskResult map."
+  [agent-config task context worktree opts]
+  (let [start-time (now-ms)
+        max-attempts (get-in opts [:limits :max-review-attempts] 5)
+
+        ;; Run review loop
+        loop-result (review/review-task!
+                      agent-config
+                      task
+                      context
+                      worktree
+                      {:max-attempts max-attempts})
+
+        _ (println (format "[%s] Review: %s (%d attempts)"
+                           (:id task)
+                           (name (:status loop-result))
+                           (review/attempt-count loop-result)))
+
+        ;; Merge if approved
+        merge-result (when (review/approved? loop-result)
+                       (println (format "[%s] Merging..." (:id task)))
+                       (merge/merge-worktree! worktree
+                                              {:strategy :no-ff
+                                               :dry-run (:dry-run opts)}))]
+
+    {:task-id (:id task)
+     :status (cond
+               (and merge-result (= :merged (:status merge-result))) :merged
+               (review/approved? loop-result) :merge-failed
+               (review/exhausted? loop-result) :review-exhausted
+               :else :failed)
+     :worker-id (:id worktree)
+     :started-at start-time
+     :completed-at (now-ms)
+     :review-attempts (review/attempt-count loop-result)
+     :merge-result merge-result
+     :error (or (:error loop-result)
+                (:error merge-result))}))
+
+;; =============================================================================
+;; Orchestrator Lifecycle
+;; =============================================================================
+
+(defn create-orchestrator
+  "Initialize orchestrator with config.
+
+   Arguments:
+     config - OrchestratorConfig
+
+   Returns OrchestratorState"
+  [{:keys [worker-count harness model worktree-root dry-run] :as config}]
+  (schema/assert-valid schema/valid-orchestrator-config? config "OrchestratorConfig")
+
+  (let [model-str (if model (format " (model: %s)" model) "")]
+    (println (format "Initializing %d workers with %s harness%s..."
+                     worker-count (name harness) model-str)))
+
+  ;; Initialize worktree pool
+  (let [pool (worktree/init-pool! config)
+        workers (mapv (fn [wt]
+                        (create-worker (:id wt) wt))
+                      pool)
+        tasks (load-tasks)]
+
+    (println (format "Loaded %d tasks, %d worktrees ready"
+                     (count tasks) (count workers)))
+
+    {:config config
+     :workers workers
+     :worktree-pool pool
+     :task-queue (vec tasks)
+     :completed []
+     :failed []
+     :started-at (now-ms)}))
+
+(defn run!
+  "Run orchestrator: process all tasks with parallel workers.
+
+   Uses core.async to parallelize across workers.
+   Returns updated OrchestratorState."
+  [state]
+  (let [{:keys [config workers worktree-pool task-queue]} state
+        {:keys [worker-count harness model review-harness review-model custom-prompt dry-run]} config
+        policy (load-policy)
+
+        ;; Proposer config
+        agent-config {:type harness
+                      :model model
+                      :sandbox :workspace-write
+                      :timeout-seconds 300}
+
+        ;; Reviewer config (defaults to same as proposer)
+        reviewer-config {:type (or review-harness harness)
+                         :model (or review-model model)
+                         :sandbox :workspace-write
+                         :timeout-seconds 300}
+
+        context (build-context task-queue)
+
+        ;; Channels for task distribution
+        task-ch (async/chan)
+        result-ch (async/chan)
+        merge-ch (async/chan)  ; Serialize merges
+
+        ;; Worker processes
+        worker-procs
+        (doall
+          (for [worker workers]
+            (async/go-loop []
+              (when-let [task (async/<! task-ch)]
+                (let [wt (:worktree worker)
+                      result (try
+                               (process-task! agent-config task context wt
+                                              {:dry-run dry-run
+                                               :limits (:limits policy)})
+                               (catch Exception e
+                                 {:task-id (:id task)
+                                  :status :error
+                                  :worker-id (:id worker)
+                                  :error (.getMessage e)}))]
+                  (async/>! result-ch result)
+                  ;; Reset worktree for next task
+                  (worktree/release! worktree-pool (:id wt) {:reset? true})
+                  (recur))))))
+
+        ;; Feed tasks to workers
+        _ (async/go
+            (doseq [task task-queue]
+              (async/>! task-ch task))
+            (async/close! task-ch))
+
+        ;; Collect results
+        results (loop [remaining (count task-queue)
+                       acc []]
+                  (if (zero? remaining)
+                    acc
+                    (if-let [result (async/<!! result-ch)]
+                      (do
+                        (println (format "[%s] -> %s"
+                                         (:task-id result)
+                                         (name (:status result))))
+                        (recur (dec remaining) (conj acc result)))
+                      acc)))]
+
+    ;; Close channels
+    (async/close! result-ch)
+
+    ;; Update state
+    (assoc state
+           :completed (filterv #(= :merged (:status %)) results)
+           :failed (filterv #(not= :merged (:status %)) results)
+           :run-log results
+           :finished-at (now-ms))))
+
+(defn shutdown!
+  "Clean up orchestrator resources"
+  [state]
+  (println "Shutting down orchestrator...")
+  (worktree/cleanup-pool! (:worktree-pool state))
+  nil)
+
+;; =============================================================================
+;; Logging
+;; =============================================================================
+
+(defn- ensure-dir! [path]
+  (.mkdirs (io/file path)))
+
+(defn save-run-log!
+  "Save run results to JSONL file"
+  [state]
+  (ensure-dir! RUNS_DIR)
+  (let [timestamp (java.time.format.DateTimeFormatter/ofPattern "yyyyMMdd-HHmmss")
+        fname (format "run-%s.jsonl"
+                      (.format timestamp (java.time.LocalDateTime/now)))
+        path (io/file RUNS_DIR fname)]
+    (doseq [entry (:run-log state)]
+      (spit path (str (json/generate-string entry) "\n") :append true))
+    (println (format "Run log: %s" (.getPath path)))
+    (.getPath path)))
+
+(defn print-summary
+  "Print run summary to stdout"
+  [state]
+  (let [{:keys [completed failed started-at finished-at]} state
+        duration-sec (/ (- (or finished-at (now-ms)) started-at) 1000.0)]
+    (println)
+    (println "=== Run Summary ===")
+    (println (format "Duration: %.1fs" duration-sec))
+    (println (format "Completed: %d" (count completed)))
+    (println (format "Failed: %d" (count failed)))
+    (when (seq failed)
+      (println "Failed tasks:")
+      (doseq [{:keys [task-id status error]} failed]
+        (println (format "  - %s: %s%s"
+                         task-id
+                         (name status)
+                         (if error (str " (" error ")") "")))))))
+
+;; =============================================================================
+;; Convenience API
+;; =============================================================================
+
+(defn run-once!
+  "Convenience: create orchestrator, run, save log, shutdown.
+
+   Arguments:
+     opts - {:workers N, :harness :codex|:claude, :model string,
+             :review-harness keyword, :review-model string,
+             :custom-prompt string, :dry-run bool}
+
+   Returns run log path"
+  [opts]
+  (let [config {:worker-count (or (:workers opts) 2)
+                :harness (or (:harness opts) :codex)
+                :model (:model opts)
+                :review-harness (:review-harness opts)
+                :review-model (:review-model opts)
+                :custom-prompt (:custom-prompt opts)
+                :worktree-root ".workers"
+                :dry-run (:dry-run opts false)
+                :policy (load-policy)}
+        state (-> (create-orchestrator config)
+                  (run!))]
+    (print-summary state)
+    (let [log-path (save-run-log! state)]
+      (when-not (:keep-worktrees opts)
+        (shutdown! state))
+      log-path)))
+
+(defn run-loop!
+  "Run orchestrator in a loop N times (like the bash loop).
+
+   Arguments:
+     iterations - Number of times to run
+     opts       - Same as run-once!
+
+   Returns vector of log paths"
+  [iterations opts]
+  (loop [i 0
+         logs []]
+    (if (< i iterations)
+      (do
+        (println (format "\n=== Iteration %d/%d ===" (inc i) iterations))
+        (let [log (run-once! opts)]
+          (recur (inc i) (conj logs log))))
+      logs)))

package/agentnet/src/agentnet/review.clj

@@ -0,0 +1,263 @@
+(ns agentnet.review
+  "Propose-review-fix loop management.
+
+   The review loop orchestrates the back-and-forth between proposer and
+   reviewer agents until changes are approved or max attempts exhausted.
+
+   Flow:
+     1. Proposer makes changes
+     2. Reviewer evaluates changes
+     3. If approved -> exit loop, ready to merge
+     4. If needs-changes -> proposer fixes based on feedback
+     5. Repeat until approved or max attempts
+
+   Design:
+     - Loop state is a pure data structure (ReviewLoop)
+     - Each step returns new state (functional)
+     - Side effects (agent calls) isolated to step functions
+     - Configurable max attempts prevents infinite loops"
+  (:require [agentnet.schema :as schema]
+            [agentnet.agent :as agent]
+            [agentnet.worktree :as worktree]
+            [clojure.string :as str]))
+
+;; =============================================================================
+;; Function Specs
+;; =============================================================================
+
+;; create-loop : Task, WorktreeId, MaxAttempts -> ReviewLoop
+;; Initialize a new review loop
+
+;; step! : ReviewLoop, AgentConfig, Context, Worktree -> ReviewLoop
+;; Execute one iteration of the loop (propose or review)
+
+;; run-loop! : ReviewLoop, AgentConfig, Context, Worktree -> ReviewLoop
+;; Run loop to completion (approved or exhausted)
+
+;; approved? : ReviewLoop -> Boolean
+;; Check if loop ended with approval
+
+;; =============================================================================
+;; Constants
+;; =============================================================================
+
+(def ^:const DEFAULT_MAX_ATTEMPTS 5)
+
+;; =============================================================================
+;; State Management
+;; =============================================================================
+
+(defn- now-ms []
+  (System/currentTimeMillis))
+
+(defn create-loop
+  "Create a new review loop for a task"
+  [task-id worktree-id {:keys [max-attempts] :or {max-attempts DEFAULT_MAX_ATTEMPTS}}]
+  {:task-id task-id
+   :worktree-id worktree-id
+   :max-attempts max-attempts
+   :attempts []
+   :status :in-progress})
+
+(defn approved?
+  "Check if loop completed with approval"
+  [loop-state]
+  (= :approved (:status loop-state)))
+
+(defn exhausted?
+  "Check if loop exhausted max attempts"
+  [loop-state]
+  (= :exhausted (:status loop-state)))
+
+(defn attempt-count
+  "Get number of attempts so far"
+  [loop-state]
+  (count (:attempts loop-state)))
+
+(defn last-feedback
+  "Get feedback from most recent attempt"
+  [loop-state]
+  (-> loop-state :attempts last :feedback))
+
+(defn can-continue?
+  "Check if loop can continue (not done, not exhausted)"
+  [loop-state]
+  (and (= :in-progress (:status loop-state))
+       (< (attempt-count loop-state) (:max-attempts loop-state))))
+
+;; =============================================================================
+;; Feedback Formatting
+;; =============================================================================
+
+(defn- format-feedback-for-proposer
+  "Format review feedback as instructions for proposer to fix"
+  [feedback]
+  (let [{:keys [verdict comments violations suggested-fixes]} feedback
+        sections (cond-> []
+                   (seq violations)
+                   (conj (str "Violations:\n"
+                              (str/join "\n" (map #(str "- " %) violations))))
+
+                   (seq comments)
+                   (conj (str "Feedback:\n"
+                              (str/join "\n" (map #(str "- " %) comments))))
+
+                   (seq suggested-fixes)
+                   (conj (str "Suggested fixes:\n"
+                              (str/join "\n" (map #(str "- " %) suggested-fixes)))))]
+    (if (seq sections)
+      (str "Previous review feedback:\n\n"
+           (str/join "\n\n" sections)
+           "\n\nPlease address the above issues.")
+      "Please review and improve the implementation.")))
+
+;; =============================================================================
+;; Step Execution
+;; =============================================================================
+
+(defn- run-proposer!
+  "Execute proposer agent, commit changes"
+  [agent-config task context worktree feedback]
+  (let [;; Augment context with feedback if this is a retry
+        augmented-context (if feedback
+                            (assoc context :review_feedback
+                                   (format-feedback-for-proposer feedback))
+                            context)
+        result (agent/propose! agent-config task augmented-context worktree)]
+    (when (zero? (:exit result))
+      ;; Commit changes in worktree
+      (let [msg (if feedback
+                  (format "fix: address review feedback for %s" (:id task))
+                  (format "feat: implement %s" (:id task)))]
+        (worktree/commit-in-worktree! worktree msg)))
+    result))
+
+(defn- run-reviewer!
+  "Execute reviewer agent on current worktree state"
+  [agent-config task context worktree]
+  (agent/review! agent-config task context worktree))
+
+(defn- create-attempt
+  "Create an attempt record from review result"
+  [attempt-number review-result worktree]
+  {:attempt-number attempt-number
+   :timestamp (now-ms)
+   :feedback (or (:parsed review-result)
+                 {:verdict :needs-changes
+                  :comments ["Review failed to produce structured output"]})
+   :patch-hash nil})  ; TODO: compute hash of current diff
+
+(defn step!
+  "Execute one iteration of propose->review.
+
+   Returns updated ReviewLoop with new attempt recorded."
+  [loop-state agent-config task context worktree]
+  (when-not (can-continue? loop-state)
+    (throw (ex-info "Cannot continue loop" {:loop loop-state})))
+
+  (let [attempt-num (inc (attempt-count loop-state))
+        prev-feedback (last-feedback loop-state)
+
+        ;; Step 1: Proposer makes/fixes changes
+        propose-result (run-proposer! agent-config task context worktree prev-feedback)
+
+        _ (when-not (zero? (:exit propose-result))
+            (throw (ex-info "Proposer failed"
+                            {:exit (:exit propose-result)
+                             :stderr (:stderr propose-result)})))
+
+        ;; Step 2: Reviewer evaluates
+        review-result (run-reviewer! agent-config task context worktree)
+
+        ;; Record attempt
+        attempt (create-attempt attempt-num review-result worktree)
+        verdict (get-in attempt [:feedback :verdict])
+
+        ;; Update loop state
+        new-attempts (conj (:attempts loop-state) attempt)
+        new-status (cond
+                     (= verdict :approved) :approved
+                     (>= attempt-num (:max-attempts loop-state)) :exhausted
+                     :else :in-progress)]
+
+    (assoc loop-state
+           :attempts new-attempts
+           :status new-status)))
+
+;; =============================================================================
+;; Full Loop Execution
+;; =============================================================================
+
+(defn run-loop!
+  "Run the review loop to completion.
+
+   Continues until:
+     - Approved: reviewer accepts changes
+     - Exhausted: max attempts reached
+     - Aborted: unrecoverable error
+
+   Returns final ReviewLoop state."
+  [loop-state agent-config task context worktree]
+  (loop [state loop-state]
+    (if (can-continue? state)
+      (let [new-state (try
+                        (step! state agent-config task context worktree)
+                        (catch Exception e
+                          (assoc state
+                                 :status :aborted
+                                 :error (.getMessage e))))]
+        (if (= :in-progress (:status new-state))
+          (recur new-state)
+          new-state))
+      state)))
+
+;; =============================================================================
+;; Convenience API
+;; =============================================================================
+
+(defn review-task!
+  "Complete review flow for a task: create loop, run to completion.
+
+   Arguments:
+     agent-config - AgentConfig for proposer/reviewer
+     task         - Task to implement
+     context      - Context map for prompts
+     worktree     - Worktree to work in
+     opts         - {:max-attempts N}
+
+   Returns final ReviewLoop state"
+  [agent-config task context worktree opts]
+  (let [loop-state (create-loop (:id task) (:id worktree) opts)]
+    (run-loop! loop-state agent-config task context worktree)))
+
+(defn summarize-loop
+  "Generate human-readable summary of review loop"
+  [loop-state]
+  (let [{:keys [task-id status attempts max-attempts]} loop-state
+        attempt-summaries (map (fn [{:keys [attempt-number feedback]}]
+                                 (format "  Attempt %d: %s"
+                                         attempt-number
+                                         (name (:verdict feedback :unknown))))
+                               attempts)]
+    (str (format "Task: %s\n" task-id)
+         (format "Status: %s\n" (name status))
+         (format "Attempts: %d/%d\n" (count attempts) max-attempts)
+         (when (seq attempt-summaries)
+           (str "History:\n" (str/join "\n" attempt-summaries))))))
+
+;; =============================================================================
+;; Logging / Persistence
+;; =============================================================================
+
+(defn loop->log-entry
+  "Convert loop state to log entry format"
+  [loop-state]
+  {:task-id (:task-id loop-state)
+   :status (case (:status loop-state)
+             :approved :merged  ; will become merged after actual merge
+             :exhausted :failed
+             :aborted :failed
+             :in-progress)
+   :review-attempts (attempt-count loop-state)
+   :final-verdict (get-in (last-feedback loop-state) [:verdict])
+   :error (:error loop-state)})