draft-board 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/backend/.env.example +9 -0
- package/app/backend/.smartkanban/evidence/8b383839-cbec-45af-86ee-c7708d075cbe/bddf2ed5-2e21-4d46-a62b-10b87f1642a6_patch.txt +195 -0
- package/app/backend/.smartkanban/evidence/8b383839-cbec-45af-86ee-c7708d075cbe/bddf2ed5-2e21-4d46-a62b-10b87f1642a6_stat.txt +6 -0
- package/app/backend/CURL_EXAMPLES.md +335 -0
- package/app/backend/ENV_SETUP.md +65 -0
- package/app/backend/alembic/env.py +71 -0
- package/app/backend/alembic/script.py.mako +28 -0
- package/app/backend/alembic/versions/001_initial_schema.py +104 -0
- package/app/backend/alembic/versions/002_add_jobs_table.py +52 -0
- package/app/backend/alembic/versions/003_add_workspace_table.py +48 -0
- package/app/backend/alembic/versions/004_add_evidence_table.py +56 -0
- package/app/backend/alembic/versions/005_add_verification_commands.py +32 -0
- package/app/backend/alembic/versions/006_add_planner_lock_table.py +39 -0
- package/app/backend/alembic/versions/007_add_revision_review_tables.py +126 -0
- package/app/backend/alembic/versions/008_add_revision_idempotency_and_traceability.py +52 -0
- package/app/backend/alembic/versions/009_add_job_health_fields.py +46 -0
- package/app/backend/alembic/versions/010_add_review_comment_line_content.py +36 -0
- package/app/backend/alembic/versions/011_add_analysis_cache.py +47 -0
- package/app/backend/alembic/versions/012_add_boards_table.py +102 -0
- package/app/backend/alembic/versions/013_add_ticket_blocking.py +45 -0
- package/app/backend/alembic/versions/014_add_agent_sessions.py +220 -0
- package/app/backend/alembic/versions/015_add_ticket_sort_order.py +33 -0
- package/app/backend/alembic/versions/03220f0b93ae_add_pr_fields_to_ticket.py +49 -0
- package/app/backend/alembic/versions/0c2d89fff3b1_seed_board_configs_from_yaml.py +206 -0
- package/app/backend/alembic/versions/3348e5cf54c1_add_merge_checklist_table.py +67 -0
- package/app/backend/alembic/versions/357c780ee445_add_goal_status.py +34 -0
- package/app/backend/alembic/versions/553340b7e26c_add_autonomy_fields_to_goal.py +65 -0
- package/app/backend/alembic/versions/774dc335c679_merge_migration_heads.py +23 -0
- package/app/backend/alembic/versions/7b307e847cbd_merge_heads.py +23 -0
- package/app/backend/alembic/versions/82ecd978cc70_add_missing_indexes.py +48 -0
- package/app/backend/alembic/versions/8ef5054dc280_add_normalized_log_entries.py +173 -0
- package/app/backend/alembic/versions/8f3e2bd8ea3b_merge_migration_heads.py +23 -0
- package/app/backend/alembic/versions/9d17f0698d3b_add_config_column_to_boards_table.py +30 -0
- package/app/backend/alembic/versions/add_agent_conversation_history.py +72 -0
- package/app/backend/alembic/versions/add_job_variant.py +34 -0
- package/app/backend/alembic/versions/add_performance_indexes.py +95 -0
- package/app/backend/alembic/versions/add_repos_and_board_repos.py +174 -0
- package/app/backend/alembic/versions/add_session_id_to_jobs.py +27 -0
- package/app/backend/alembic/versions/add_sqlite_backend_tables.py +104 -0
- package/app/backend/alembic/versions/b10fb0b62240_add_diff_content_to_revisions.py +34 -0
- package/app/backend/alembic.ini +89 -0
- package/app/backend/app/__init__.py +3 -0
- package/app/backend/app/data_dir.py +85 -0
- package/app/backend/app/database.py +70 -0
- package/app/backend/app/database_sync.py +64 -0
- package/app/backend/app/dependencies/__init__.py +5 -0
- package/app/backend/app/dependencies/auth.py +80 -0
- package/app/backend/app/dependencies.py +43 -0
- package/app/backend/app/exceptions.py +178 -0
- package/app/backend/app/executors/__init__.py +1 -0
- package/app/backend/app/executors/adapters/__init__.py +1 -0
- package/app/backend/app/executors/adapters/aider.py +152 -0
- package/app/backend/app/executors/adapters/amazon_q.py +103 -0
- package/app/backend/app/executors/adapters/amp.py +123 -0
- package/app/backend/app/executors/adapters/claude.py +177 -0
- package/app/backend/app/executors/adapters/cline.py +127 -0
- package/app/backend/app/executors/adapters/codex.py +167 -0
- package/app/backend/app/executors/adapters/copilot.py +202 -0
- package/app/backend/app/executors/adapters/cursor.py +87 -0
- package/app/backend/app/executors/adapters/droid.py +123 -0
- package/app/backend/app/executors/adapters/gemini.py +132 -0
- package/app/backend/app/executors/adapters/goose.py +131 -0
- package/app/backend/app/executors/adapters/opencode.py +123 -0
- package/app/backend/app/executors/adapters/qwen.py +123 -0
- package/app/backend/app/executors/plugins/__init__.py +1 -0
- package/app/backend/app/executors/registry.py +202 -0
- package/app/backend/app/executors/spec.py +226 -0
- package/app/backend/app/main.py +486 -0
- package/app/backend/app/middleware/__init__.py +13 -0
- package/app/backend/app/middleware/idempotency.py +426 -0
- package/app/backend/app/middleware/rate_limit.py +312 -0
- package/app/backend/app/middleware/security_headers.py +43 -0
- package/app/backend/app/middleware/timeout.py +37 -0
- package/app/backend/app/models/__init__.py +56 -0
- package/app/backend/app/models/agent_conversation_history.py +56 -0
- package/app/backend/app/models/agent_session.py +127 -0
- package/app/backend/app/models/analysis_cache.py +49 -0
- package/app/backend/app/models/base.py +9 -0
- package/app/backend/app/models/board.py +79 -0
- package/app/backend/app/models/board_repo.py +68 -0
- package/app/backend/app/models/cost_budget.py +42 -0
- package/app/backend/app/models/enums.py +40 -0
- package/app/backend/app/models/evidence.py +132 -0
- package/app/backend/app/models/goal.py +102 -0
- package/app/backend/app/models/idempotency_entry.py +30 -0
- package/app/backend/app/models/job.py +163 -0
- package/app/backend/app/models/job_queue.py +39 -0
- package/app/backend/app/models/kv_store.py +28 -0
- package/app/backend/app/models/merge_checklist.py +87 -0
- package/app/backend/app/models/normalized_log.py +100 -0
- package/app/backend/app/models/planner_lock.py +43 -0
- package/app/backend/app/models/rate_limit_entry.py +25 -0
- package/app/backend/app/models/repo.py +66 -0
- package/app/backend/app/models/review_comment.py +91 -0
- package/app/backend/app/models/review_summary.py +69 -0
- package/app/backend/app/models/revision.py +130 -0
- package/app/backend/app/models/ticket.py +223 -0
- package/app/backend/app/models/ticket_event.py +83 -0
- package/app/backend/app/models/user.py +47 -0
- package/app/backend/app/models/workspace.py +71 -0
- package/app/backend/app/redis_client.py +119 -0
- package/app/backend/app/routers/__init__.py +29 -0
- package/app/backend/app/routers/agents.py +296 -0
- package/app/backend/app/routers/auth.py +94 -0
- package/app/backend/app/routers/board.py +885 -0
- package/app/backend/app/routers/dashboard.py +351 -0
- package/app/backend/app/routers/debug.py +528 -0
- package/app/backend/app/routers/evidence.py +96 -0
- package/app/backend/app/routers/executors.py +324 -0
- package/app/backend/app/routers/goals.py +574 -0
- package/app/backend/app/routers/jobs.py +448 -0
- package/app/backend/app/routers/maintenance.py +172 -0
- package/app/backend/app/routers/merge.py +360 -0
- package/app/backend/app/routers/planner.py +537 -0
- package/app/backend/app/routers/pull_requests.py +382 -0
- package/app/backend/app/routers/repos.py +263 -0
- package/app/backend/app/routers/revisions.py +939 -0
- package/app/backend/app/routers/settings.py +267 -0
- package/app/backend/app/routers/tickets.py +2003 -0
- package/app/backend/app/routers/webhooks.py +143 -0
- package/app/backend/app/routers/websocket.py +249 -0
- package/app/backend/app/schemas/__init__.py +109 -0
- package/app/backend/app/schemas/board.py +87 -0
- package/app/backend/app/schemas/common.py +33 -0
- package/app/backend/app/schemas/evidence.py +87 -0
- package/app/backend/app/schemas/goal.py +90 -0
- package/app/backend/app/schemas/job.py +97 -0
- package/app/backend/app/schemas/merge.py +139 -0
- package/app/backend/app/schemas/planner.py +500 -0
- package/app/backend/app/schemas/repo.py +187 -0
- package/app/backend/app/schemas/review.py +137 -0
- package/app/backend/app/schemas/revision.py +114 -0
- package/app/backend/app/schemas/ticket.py +238 -0
- package/app/backend/app/schemas/ticket_event.py +72 -0
- package/app/backend/app/schemas/workspace.py +19 -0
- package/app/backend/app/services/__init__.py +31 -0
- package/app/backend/app/services/agent_memory_service.py +223 -0
- package/app/backend/app/services/agent_registry.py +346 -0
- package/app/backend/app/services/agent_session_manager.py +318 -0
- package/app/backend/app/services/agent_session_service.py +219 -0
- package/app/backend/app/services/agent_tools.py +379 -0
- package/app/backend/app/services/auth_service.py +98 -0
- package/app/backend/app/services/autonomy_service.py +380 -0
- package/app/backend/app/services/board_repo_service.py +201 -0
- package/app/backend/app/services/board_service.py +326 -0
- package/app/backend/app/services/cleanup_service.py +1085 -0
- package/app/backend/app/services/config_service.py +908 -0
- package/app/backend/app/services/context_gatherer.py +557 -0
- package/app/backend/app/services/cost_tracking_service.py +293 -0
- package/app/backend/app/services/cursor_log_normalizer.py +536 -0
- package/app/backend/app/services/delivery_pipeline.py +440 -0
- package/app/backend/app/services/executor_service.py +634 -0
- package/app/backend/app/services/git_host/__init__.py +11 -0
- package/app/backend/app/services/git_host/factory.py +87 -0
- package/app/backend/app/services/git_host/github.py +270 -0
- package/app/backend/app/services/git_host/gitlab.py +194 -0
- package/app/backend/app/services/git_host/protocol.py +75 -0
- package/app/backend/app/services/git_merge_simple.py +346 -0
- package/app/backend/app/services/git_ops.py +384 -0
- package/app/backend/app/services/github_service.py +233 -0
- package/app/backend/app/services/goal_service.py +113 -0
- package/app/backend/app/services/job_service.py +423 -0
- package/app/backend/app/services/job_watchdog_service.py +424 -0
- package/app/backend/app/services/langchain_adapter.py +122 -0
- package/app/backend/app/services/llm_provider_clients.py +351 -0
- package/app/backend/app/services/llm_service.py +285 -0
- package/app/backend/app/services/log_normalizer.py +342 -0
- package/app/backend/app/services/log_stream_service.py +276 -0
- package/app/backend/app/services/merge_checklist_service.py +264 -0
- package/app/backend/app/services/merge_service.py +784 -0
- package/app/backend/app/services/orchestrator_log.py +84 -0
- package/app/backend/app/services/planner_service.py +1662 -0
- package/app/backend/app/services/planner_tick_sync.py +1040 -0
- package/app/backend/app/services/queued_message_service.py +156 -0
- package/app/backend/app/services/reliability_wrapper.py +389 -0
- package/app/backend/app/services/repo_discovery_service.py +318 -0
- package/app/backend/app/services/review_service.py +334 -0
- package/app/backend/app/services/revision_service.py +389 -0
- package/app/backend/app/services/safe_autopilot.py +510 -0
- package/app/backend/app/services/sqlite_worker.py +372 -0
- package/app/backend/app/services/task_dispatch.py +135 -0
- package/app/backend/app/services/ticket_generation_service.py +1781 -0
- package/app/backend/app/services/ticket_service.py +486 -0
- package/app/backend/app/services/udar_planner_service.py +1007 -0
- package/app/backend/app/services/webhook_service.py +126 -0
- package/app/backend/app/services/workspace_service.py +465 -0
- package/app/backend/app/services/worktree_file_service.py +92 -0
- package/app/backend/app/services/worktree_validator.py +213 -0
- package/app/backend/app/sqlite_kv.py +278 -0
- package/app/backend/app/state_machine.py +128 -0
- package/app/backend/app/templates/__init__.py +5 -0
- package/app/backend/app/templates/registry.py +243 -0
- package/app/backend/app/utils/__init__.py +5 -0
- package/app/backend/app/utils/artifact_reader.py +87 -0
- package/app/backend/app/utils/circuit_breaker.py +229 -0
- package/app/backend/app/utils/db_retry.py +136 -0
- package/app/backend/app/utils/ignored_fields.py +123 -0
- package/app/backend/app/utils/validators.py +54 -0
- package/app/backend/app/websocket/__init__.py +5 -0
- package/app/backend/app/websocket/manager.py +179 -0
- package/app/backend/app/websocket/state_tracker.py +113 -0
- package/app/backend/app/worker.py +3190 -0
- package/app/backend/calculator_tickets.json +40 -0
- package/app/backend/canary_tests.sh +591 -0
- package/app/backend/celerybeat-schedule +0 -0
- package/app/backend/celerybeat-schedule-shm +0 -0
- package/app/backend/celerybeat-schedule-wal +0 -0
- package/app/backend/logs/.gitkeep +3 -0
- package/app/backend/multiplication_division_implementation_tickets.json +55 -0
- package/app/backend/multiplication_division_tickets.json +42 -0
- package/app/backend/pyproject.toml +45 -0
- package/app/backend/requirements-dev.txt +8 -0
- package/app/backend/requirements.txt +20 -0
- package/app/backend/run.sh +30 -0
- package/app/backend/run_with_logs.sh +10 -0
- package/app/backend/scientific_calculator_tickets.json +40 -0
- package/app/backend/scripts/extract_openapi.py +21 -0
- package/app/backend/scripts/seed_demo.py +187 -0
- package/app/backend/setup_demo_review.py +302 -0
- package/app/backend/test_actual_parse.py +41 -0
- package/app/backend/test_agent_streaming.py +61 -0
- package/app/backend/test_parse.py +51 -0
- package/app/backend/test_streaming.py +51 -0
- package/app/backend/test_subprocess_streaming.py +50 -0
- package/app/backend/tests/__init__.py +1 -0
- package/app/backend/tests/conftest.py +46 -0
- package/app/backend/tests/test_auth.py +341 -0
- package/app/backend/tests/test_autonomy_service.py +391 -0
- package/app/backend/tests/test_cleanup_service_safety.py +417 -0
- package/app/backend/tests/test_middleware.py +279 -0
- package/app/backend/tests/test_planner_providers.py +290 -0
- package/app/backend/tests/test_planner_unblock.py +183 -0
- package/app/backend/tests/test_revision_invariants.py +618 -0
- package/app/backend/tests/test_sqlite_kv.py +290 -0
- package/app/backend/tests/test_sqlite_worker.py +353 -0
- package/app/backend/tests/test_task_dispatch.py +100 -0
- package/app/backend/tests/test_ticket_validation.py +304 -0
- package/app/backend/tests/test_udar_agent.py +693 -0
- package/app/backend/tests/test_webhook_service.py +184 -0
- package/app/backend/tickets_output.json +59 -0
- package/app/backend/user_management_tickets.json +50 -0
- package/app/backend/uvicorn.log +0 -0
- package/app/draft.yaml +313 -0
- package/app/frontend/dist/assets/index-LcjCczu5.js +155 -0
- package/app/frontend/dist/assets/index-_FP_279e.css +1 -0
- package/app/frontend/dist/index.html +14 -0
- package/app/frontend/dist/vite.svg +1 -0
- package/app/frontend/package.json +101 -0
- package/bin/cli.js +527 -0
- package/package.json +37 -0
|
@@ -0,0 +1,3190 @@
|
|
|
1
|
+
"""Worker task implementations for Draft.
|
|
2
|
+
|
|
3
|
+
These functions are called by the SQLiteWorker (in-process background job runner).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import subprocess
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
import uuid
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
|
|
20
|
+
from sqlalchemy.orm import selectinload
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from app.services.config_service import PlannerConfig
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
from app.data_dir import get_evidence_dir as central_evidence_dir
|
|
27
|
+
from app.data_dir import get_log_path
|
|
28
|
+
from app.database_sync import get_sync_db
|
|
29
|
+
from app.exceptions import (
|
|
30
|
+
ExecutorNotFoundError,
|
|
31
|
+
NotAGitRepositoryError,
|
|
32
|
+
WorkspaceError,
|
|
33
|
+
)
|
|
34
|
+
from app.models.evidence import Evidence, EvidenceKind
|
|
35
|
+
from app.models.job import Job, JobStatus
|
|
36
|
+
from app.models.ticket import Ticket
|
|
37
|
+
from app.models.ticket_event import TicketEvent
|
|
38
|
+
from app.services.config_service import DraftConfig, YoloStatus
|
|
39
|
+
from app.services.cursor_log_normalizer import CursorLogNormalizer
|
|
40
|
+
from app.services.executor_service import (
|
|
41
|
+
ExecutorService,
|
|
42
|
+
ExecutorType,
|
|
43
|
+
PromptBundleBuilder,
|
|
44
|
+
)
|
|
45
|
+
from app.services.log_stream_service import LogLevel, log_stream_publisher
|
|
46
|
+
from app.services.workspace_service import WorkspaceService
|
|
47
|
+
from app.services.worktree_validator import WorktreeValidator
|
|
48
|
+
from app.state_machine import ActorType, EventType, TicketState
|
|
49
|
+
|
|
50
|
+
# Thread-local storage for job context (THREAD-SAFE)
|
|
51
|
+
_job_context = threading.local()
|
|
52
|
+
|
|
53
|
+
# Track active subprocesses for cancellation (THREAD-SAFE with lock)
|
|
54
|
+
_active_processes: dict[str, subprocess.Popen] = {}
|
|
55
|
+
_active_processes_lock = threading.Lock()
|
|
56
|
+
|
|
57
|
+
# Executor retry configuration
|
|
58
|
+
EXECUTOR_MAX_RETRIES = 2 # Retry up to 2 times (3 total attempts)
|
|
59
|
+
EXECUTOR_RETRY_DELAY_BASE = 5 # Base delay in seconds (exponential backoff)
|
|
60
|
+
|
|
61
|
+
# Retry-eligible exit codes (transient failures)
|
|
62
|
+
RETRYABLE_EXIT_CODES = {
|
|
63
|
+
-1, # Timeout or general failure
|
|
64
|
+
124, # Timeout (from timeout command)
|
|
65
|
+
137, # SIGKILL (OOM or external kill)
|
|
66
|
+
143, # SIGTERM
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
# Retry-eligible error patterns in stderr (network/API issues)
|
|
70
|
+
RETRYABLE_ERROR_PATTERNS = [
|
|
71
|
+
"connection reset",
|
|
72
|
+
"connection refused",
|
|
73
|
+
"network error",
|
|
74
|
+
"rate limit",
|
|
75
|
+
"503 service unavailable",
|
|
76
|
+
"502 bad gateway",
|
|
77
|
+
"504 gateway timeout",
|
|
78
|
+
"temporary failure",
|
|
79
|
+
"timeout",
|
|
80
|
+
"timed out",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def get_current_job() -> str | None:
|
|
85
|
+
"""Get the current job ID for this thread."""
|
|
86
|
+
return getattr(_job_context, "job_id", None)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def write_log(log_path: Path, message: str, job_id: str | None = None) -> None:
|
|
90
|
+
"""Write a timestamped message to the log file AND stream via Redis.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
log_path: Path to the log file
|
|
94
|
+
message: The log message
|
|
95
|
+
job_id: Optional job ID for real-time streaming (uses thread-local if not set)
|
|
96
|
+
"""
|
|
97
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
timestamp = datetime.now(UTC).isoformat()
|
|
99
|
+
with open(log_path, "a") as f:
|
|
100
|
+
f.write(f"[{timestamp}] {message}\n")
|
|
101
|
+
|
|
102
|
+
# Also stream to Redis for real-time SSE (THREAD-SAFE)
|
|
103
|
+
stream_job_id = job_id or get_current_job()
|
|
104
|
+
if stream_job_id:
|
|
105
|
+
try:
|
|
106
|
+
log_stream_publisher.push_info(stream_job_id, message)
|
|
107
|
+
except Exception:
|
|
108
|
+
pass # Don't fail job if streaming fails
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def set_current_job(job_id: str | None) -> None:
|
|
112
|
+
"""Set the current job ID for this thread (THREAD-SAFE)."""
|
|
113
|
+
_job_context.job_id = job_id
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def is_retryable_error(exit_code: int, stderr: str) -> bool:
|
|
117
|
+
"""Check if an executor error is retryable (transient failure).
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
exit_code: The process exit code
|
|
121
|
+
stderr: The stderr output from the process
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
True if the error is likely transient and worth retrying
|
|
125
|
+
"""
|
|
126
|
+
# Check exit code
|
|
127
|
+
if exit_code in RETRYABLE_EXIT_CODES:
|
|
128
|
+
return True
|
|
129
|
+
|
|
130
|
+
# Check stderr for known transient error patterns
|
|
131
|
+
stderr_lower = stderr.lower()
|
|
132
|
+
for pattern in RETRYABLE_ERROR_PATTERNS:
|
|
133
|
+
if pattern in stderr_lower:
|
|
134
|
+
return True
|
|
135
|
+
|
|
136
|
+
return False
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def register_active_process(job_id: str, process: subprocess.Popen) -> None:
|
|
140
|
+
"""Register an active subprocess for potential cancellation."""
|
|
141
|
+
with _active_processes_lock:
|
|
142
|
+
_active_processes[job_id] = process
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def unregister_active_process(job_id: str) -> None:
|
|
146
|
+
"""Unregister an active subprocess."""
|
|
147
|
+
with _active_processes_lock:
|
|
148
|
+
_active_processes.pop(job_id, None)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def kill_job_process(job_id: str) -> bool:
|
|
152
|
+
"""Kill the subprocess for a job (for cancellation).
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
True if process was found and killed, False otherwise
|
|
156
|
+
"""
|
|
157
|
+
with _active_processes_lock:
|
|
158
|
+
process = _active_processes.get(job_id)
|
|
159
|
+
if process and process.poll() is None: # Still running
|
|
160
|
+
logger.info(f"Killing process {process.pid} for job {job_id}")
|
|
161
|
+
try:
|
|
162
|
+
process.kill()
|
|
163
|
+
process.wait(timeout=5)
|
|
164
|
+
return True
|
|
165
|
+
except Exception as e:
|
|
166
|
+
logger.error(f"Failed to kill process for job {job_id}: {e}")
|
|
167
|
+
return False
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def stream_finished(job_id: str) -> None:
|
|
172
|
+
"""Signal that job has finished streaming."""
|
|
173
|
+
try:
|
|
174
|
+
log_stream_publisher.push_finished(job_id)
|
|
175
|
+
except Exception:
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def get_job_with_ticket(job_id: str) -> tuple[Job, Ticket] | None:
|
|
180
|
+
"""Get a job and its associated ticket."""
|
|
181
|
+
with get_sync_db() as db:
|
|
182
|
+
job = (
|
|
183
|
+
db.query(Job)
|
|
184
|
+
.options(selectinload(Job.ticket).selectinload(Ticket.goal))
|
|
185
|
+
.filter(Job.id == job_id)
|
|
186
|
+
.first()
|
|
187
|
+
)
|
|
188
|
+
if job and job.ticket:
|
|
189
|
+
# Expunge to use outside session
|
|
190
|
+
db.expunge(job)
|
|
191
|
+
db.expunge(job.ticket)
|
|
192
|
+
if job.ticket.goal:
|
|
193
|
+
db.expunge(job.ticket.goal)
|
|
194
|
+
return job, job.ticket
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def ensure_workspace_for_ticket(
|
|
199
|
+
ticket_id: str, goal_id: str
|
|
200
|
+
) -> tuple[Path | None, str | None]:
|
|
201
|
+
"""
|
|
202
|
+
Ensure a workspace exists for a ticket and return paths.
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
Tuple of (worktree_path, error_message).
|
|
206
|
+
If successful, worktree_path is set and error_message is None.
|
|
207
|
+
If failed, worktree_path is None and error_message describes the error.
|
|
208
|
+
"""
|
|
209
|
+
with get_sync_db() as db:
|
|
210
|
+
workspace_service = WorkspaceService(db)
|
|
211
|
+
try:
|
|
212
|
+
workspace = workspace_service.ensure_workspace(ticket_id, goal_id)
|
|
213
|
+
return Path(workspace.worktree_path), None
|
|
214
|
+
except NotAGitRepositoryError as e:
|
|
215
|
+
return None, e.message
|
|
216
|
+
except WorkspaceError as e:
|
|
217
|
+
return None, e.message
|
|
218
|
+
except Exception as e:
|
|
219
|
+
return None, f"Failed to create workspace: {str(e)}"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def get_log_path_for_job(job_id: str, worktree_path: Path | None) -> tuple[Path, str]:
|
|
223
|
+
"""
|
|
224
|
+
Get the log path for a job using central data directory.
|
|
225
|
+
|
|
226
|
+
Returns:
|
|
227
|
+
Tuple of (full_log_path, path_stored_in_db).
|
|
228
|
+
"""
|
|
229
|
+
full_path = get_log_path(job_id)
|
|
230
|
+
# Store the full path in DB since logs are now in a central location
|
|
231
|
+
return full_path, str(full_path)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def update_job_started(
|
|
235
|
+
job_id: str, log_path: str, timeout_seconds: int | None = None
|
|
236
|
+
) -> bool:
|
|
237
|
+
"""Mark job as running. Returns False if job was canceled."""
|
|
238
|
+
with get_sync_db() as db:
|
|
239
|
+
job = db.query(Job).filter(Job.id == job_id).first()
|
|
240
|
+
if not job:
|
|
241
|
+
return False
|
|
242
|
+
|
|
243
|
+
# Check if job was canceled before it started
|
|
244
|
+
if job.status == JobStatus.CANCELED.value:
|
|
245
|
+
return False
|
|
246
|
+
|
|
247
|
+
now = datetime.now(UTC)
|
|
248
|
+
job.status = JobStatus.RUNNING.value
|
|
249
|
+
job.started_at = now
|
|
250
|
+
job.last_heartbeat_at = now
|
|
251
|
+
job.log_path = log_path
|
|
252
|
+
if timeout_seconds:
|
|
253
|
+
job.timeout_seconds = timeout_seconds
|
|
254
|
+
db.commit()
|
|
255
|
+
return True
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def update_job_heartbeat(job_id: str) -> bool:
|
|
259
|
+
"""Update job heartbeat timestamp. Returns False if job was canceled."""
|
|
260
|
+
with get_sync_db() as db:
|
|
261
|
+
job = db.query(Job).filter(Job.id == job_id).first()
|
|
262
|
+
if not job:
|
|
263
|
+
return False
|
|
264
|
+
|
|
265
|
+
# Check if job was canceled
|
|
266
|
+
if job.status == JobStatus.CANCELED.value:
|
|
267
|
+
return False
|
|
268
|
+
|
|
269
|
+
job.last_heartbeat_at = datetime.now(UTC)
|
|
270
|
+
db.commit()
|
|
271
|
+
return True
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def update_job_finished(job_id: str, status: JobStatus, exit_code: int = 0) -> None:
|
|
275
|
+
"""Mark job as finished with given status and exit code."""
|
|
276
|
+
with get_sync_db() as db:
|
|
277
|
+
job = db.query(Job).filter(Job.id == job_id).first()
|
|
278
|
+
if job:
|
|
279
|
+
job.status = status.value
|
|
280
|
+
job.finished_at = datetime.now(UTC)
|
|
281
|
+
job.exit_code = exit_code
|
|
282
|
+
db.commit()
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def check_canceled(job_id: str) -> bool:
|
|
286
|
+
"""Check if the job has been canceled."""
|
|
287
|
+
with get_sync_db() as db:
|
|
288
|
+
job = db.query(Job).filter(Job.id == job_id).first()
|
|
289
|
+
return job is not None and job.status == JobStatus.CANCELED.value
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def get_evidence_dir(
|
|
293
|
+
worktree_path: Path | None, job_id: str, repo_root: Path | None = None
|
|
294
|
+
) -> Path:
|
|
295
|
+
"""Get the directory for storing evidence files.
|
|
296
|
+
|
|
297
|
+
Evidence is stored in a central location at ~/.draft/evidence/{job_id}/
|
|
298
|
+
so it survives worktree cleanup and doesn't pollute target repos.
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
worktree_path: Unused (kept for API compat)
|
|
302
|
+
job_id: UUID of the job
|
|
303
|
+
repo_root: Unused (kept for API compat)
|
|
304
|
+
|
|
305
|
+
Returns:
|
|
306
|
+
Path to evidence directory
|
|
307
|
+
"""
|
|
308
|
+
return central_evidence_dir(job_id)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def run_verification_command(
|
|
312
|
+
command: str,
|
|
313
|
+
cwd: Path | None,
|
|
314
|
+
evidence_dir: Path,
|
|
315
|
+
evidence_id: str,
|
|
316
|
+
repo_root: Path,
|
|
317
|
+
timeout: int = 300,
|
|
318
|
+
extra_allowed_commands: list[str] | None = None,
|
|
319
|
+
) -> tuple[int, str, str]:
|
|
320
|
+
"""
|
|
321
|
+
Run a verification command and capture output (SECURE - no shell injection).
|
|
322
|
+
|
|
323
|
+
SECURITY: Uses shlex.split() to safely parse commands without shell=True.
|
|
324
|
+
Only allows commands from a predefined allowlist to prevent arbitrary execution.
|
|
325
|
+
|
|
326
|
+
Args:
|
|
327
|
+
command: The command string to parse and execute
|
|
328
|
+
cwd: Working directory for the command
|
|
329
|
+
evidence_dir: Directory to store stdout/stderr files
|
|
330
|
+
evidence_id: UUID for naming evidence files
|
|
331
|
+
repo_root: Path to repo root (for computing relative paths)
|
|
332
|
+
timeout: Command timeout in seconds
|
|
333
|
+
extra_allowed_commands: Additional commands to allow (from config)
|
|
334
|
+
|
|
335
|
+
Returns:
|
|
336
|
+
Tuple of (exit_code, stdout_relpath, stderr_relpath) - paths are relative to repo_root
|
|
337
|
+
|
|
338
|
+
Raises:
|
|
339
|
+
ValueError: If command is not in allowlist
|
|
340
|
+
"""
|
|
341
|
+
import shlex
|
|
342
|
+
import shutil as _shutil
|
|
343
|
+
|
|
344
|
+
stdout_path = evidence_dir / f"{evidence_id}.stdout"
|
|
345
|
+
stderr_path = evidence_dir / f"{evidence_id}.stderr"
|
|
346
|
+
|
|
347
|
+
# Allowlist of permitted commands (prevents arbitrary code execution)
|
|
348
|
+
ALLOWED_COMMANDS = {
|
|
349
|
+
"pytest",
|
|
350
|
+
"python",
|
|
351
|
+
"python3",
|
|
352
|
+
"ruff",
|
|
353
|
+
"mypy",
|
|
354
|
+
"black",
|
|
355
|
+
"isort",
|
|
356
|
+
"npm",
|
|
357
|
+
"yarn",
|
|
358
|
+
"pnpm",
|
|
359
|
+
"node",
|
|
360
|
+
"cargo",
|
|
361
|
+
"rustc",
|
|
362
|
+
"go",
|
|
363
|
+
"make",
|
|
364
|
+
"eslint",
|
|
365
|
+
"tsc",
|
|
366
|
+
"jest",
|
|
367
|
+
"vitest",
|
|
368
|
+
"flake8",
|
|
369
|
+
"pylint",
|
|
370
|
+
}
|
|
371
|
+
if extra_allowed_commands:
|
|
372
|
+
ALLOWED_COMMANDS.update(extra_allowed_commands)
|
|
373
|
+
|
|
374
|
+
try:
|
|
375
|
+
# SECURE: Parse command string into argv array (prevents injection)
|
|
376
|
+
# shlex.split() handles quotes and escaping properly
|
|
377
|
+
cmd_argv = shlex.split(command)
|
|
378
|
+
|
|
379
|
+
if not cmd_argv:
|
|
380
|
+
raise ValueError("Empty command")
|
|
381
|
+
|
|
382
|
+
# SECURITY CHECK: Validate first argument is in allowlist
|
|
383
|
+
# Resolve the command to its full path to prevent PATH manipulation attacks
|
|
384
|
+
# (e.g., a malicious "pytest" script in the worktree shadowing the real one)
|
|
385
|
+
raw_cmd = cmd_argv[0]
|
|
386
|
+
base_command = Path(raw_cmd).name # Strip path, get command name
|
|
387
|
+
if base_command not in ALLOWED_COMMANDS:
|
|
388
|
+
raise ValueError(
|
|
389
|
+
f"Command '{base_command}' not in allowlist. "
|
|
390
|
+
f"Allowed: {', '.join(sorted(ALLOWED_COMMANDS))}"
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
# Resolve to absolute path via PATH lookup to ensure we run the real binary
|
|
394
|
+
resolved_path = _shutil.which(base_command)
|
|
395
|
+
if resolved_path:
|
|
396
|
+
cmd_argv[0] = resolved_path
|
|
397
|
+
|
|
398
|
+
# Run WITHOUT shell=True (SECURE - no command injection possible)
|
|
399
|
+
result = subprocess.run(
|
|
400
|
+
cmd_argv, # List, not string
|
|
401
|
+
shell=False, # CRITICAL: No shell metacharacter interpretation
|
|
402
|
+
cwd=cwd,
|
|
403
|
+
capture_output=True,
|
|
404
|
+
text=True,
|
|
405
|
+
timeout=timeout,
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
# Write stdout/stderr to files
|
|
409
|
+
stdout_path.write_text(result.stdout or "")
|
|
410
|
+
stderr_path.write_text(result.stderr or "")
|
|
411
|
+
|
|
412
|
+
# Return relative paths for DB storage (security: no absolute paths in DB)
|
|
413
|
+
stdout_rel = str(stdout_path)
|
|
414
|
+
stderr_rel = str(stderr_path)
|
|
415
|
+
|
|
416
|
+
return result.returncode, stdout_rel, stderr_rel
|
|
417
|
+
|
|
418
|
+
except subprocess.TimeoutExpired as e:
|
|
419
|
+
# Write partial output if available
|
|
420
|
+
stdout_path.write_text(e.stdout.decode() if e.stdout else "Command timed out")
|
|
421
|
+
stderr_path.write_text(e.stderr.decode() if e.stderr else "")
|
|
422
|
+
stdout_rel = str(stdout_path)
|
|
423
|
+
stderr_rel = str(stderr_path)
|
|
424
|
+
return -1, stdout_rel, stderr_rel
|
|
425
|
+
|
|
426
|
+
except ValueError as e:
|
|
427
|
+
# Command validation failed (not in allowlist or empty)
|
|
428
|
+
stdout_path.write_text("")
|
|
429
|
+
stderr_path.write_text(f"Command validation failed: {str(e)}")
|
|
430
|
+
stdout_rel = str(stdout_path)
|
|
431
|
+
stderr_rel = str(stderr_path)
|
|
432
|
+
return -1, stdout_rel, stderr_rel
|
|
433
|
+
|
|
434
|
+
except FileNotFoundError:
|
|
435
|
+
# Command not found in PATH
|
|
436
|
+
stdout_path.write_text("")
|
|
437
|
+
stderr_path.write_text(f"Command not found: {cmd_argv[0]}")
|
|
438
|
+
stdout_rel = str(stdout_path)
|
|
439
|
+
stderr_rel = str(stderr_path)
|
|
440
|
+
return -1, stdout_rel, stderr_rel
|
|
441
|
+
|
|
442
|
+
except Exception as e:
|
|
443
|
+
stdout_path.write_text("")
|
|
444
|
+
stderr_path.write_text(f"Error running command: {str(e)}")
|
|
445
|
+
|
|
446
|
+
stdout_rel = str(stdout_path)
|
|
447
|
+
stderr_rel = str(stderr_path)
|
|
448
|
+
return -1, stdout_rel, stderr_rel
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def create_evidence_record(
|
|
452
|
+
ticket_id: str,
|
|
453
|
+
job_id: str,
|
|
454
|
+
command: str,
|
|
455
|
+
exit_code: int,
|
|
456
|
+
stdout_path: str,
|
|
457
|
+
stderr_path: str,
|
|
458
|
+
evidence_id: str,
|
|
459
|
+
kind: EvidenceKind = EvidenceKind.COMMAND_LOG,
|
|
460
|
+
) -> Evidence:
|
|
461
|
+
"""Create an Evidence record in the database.
|
|
462
|
+
|
|
463
|
+
Args:
|
|
464
|
+
ticket_id: UUID of the ticket
|
|
465
|
+
job_id: UUID of the job
|
|
466
|
+
command: Command that was executed
|
|
467
|
+
exit_code: Exit code from the command
|
|
468
|
+
stdout_path: Path to stdout file
|
|
469
|
+
stderr_path: Path to stderr file
|
|
470
|
+
evidence_id: UUID for this evidence record
|
|
471
|
+
kind: Type of evidence (executor_stdout, git_diff_stat, etc.)
|
|
472
|
+
|
|
473
|
+
Returns:
|
|
474
|
+
The created Evidence record
|
|
475
|
+
"""
|
|
476
|
+
with get_sync_db() as db:
|
|
477
|
+
evidence = Evidence(
|
|
478
|
+
id=evidence_id,
|
|
479
|
+
ticket_id=ticket_id,
|
|
480
|
+
job_id=job_id,
|
|
481
|
+
kind=kind.value,
|
|
482
|
+
command=command,
|
|
483
|
+
exit_code=exit_code,
|
|
484
|
+
stdout_path=stdout_path,
|
|
485
|
+
stderr_path=stderr_path,
|
|
486
|
+
)
|
|
487
|
+
db.add(evidence)
|
|
488
|
+
db.commit()
|
|
489
|
+
db.refresh(evidence)
|
|
490
|
+
return evidence
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def create_revision_for_job(
|
|
494
|
+
ticket_id: str,
|
|
495
|
+
job_id: str,
|
|
496
|
+
diff_stat_evidence_id: str | None = None,
|
|
497
|
+
diff_patch_evidence_id: str | None = None,
|
|
498
|
+
) -> str | None:
|
|
499
|
+
"""Create a Revision record for a job that produced changes.
|
|
500
|
+
|
|
501
|
+
This function is IDEMPOTENT - if the same job_id is retried, returns existing revision.
|
|
502
|
+
Automatically supersedes any existing open revisions for the ticket.
|
|
503
|
+
|
|
504
|
+
Args:
|
|
505
|
+
ticket_id: UUID of the ticket
|
|
506
|
+
job_id: UUID of the job
|
|
507
|
+
diff_stat_evidence_id: Optional evidence ID for git diff stat
|
|
508
|
+
diff_patch_evidence_id: Optional evidence ID for git diff patch
|
|
509
|
+
|
|
510
|
+
Returns:
|
|
511
|
+
The revision ID if created/found, None on error
|
|
512
|
+
"""
|
|
513
|
+
from app.models.revision import Revision, RevisionStatus
|
|
514
|
+
|
|
515
|
+
with get_sync_db() as db:
|
|
516
|
+
try:
|
|
517
|
+
# IDEMPOTENCY CHECK: Return existing revision if job was already processed
|
|
518
|
+
existing = (
|
|
519
|
+
db.query(Revision)
|
|
520
|
+
.filter(
|
|
521
|
+
Revision.ticket_id == ticket_id,
|
|
522
|
+
Revision.job_id == job_id,
|
|
523
|
+
)
|
|
524
|
+
.first()
|
|
525
|
+
)
|
|
526
|
+
if existing:
|
|
527
|
+
import logging
|
|
528
|
+
|
|
529
|
+
logging.getLogger(__name__).info(
|
|
530
|
+
f"Revision already exists for job {job_id}: {existing.id}"
|
|
531
|
+
)
|
|
532
|
+
return existing.id
|
|
533
|
+
|
|
534
|
+
# Supersede any open revisions (in same transaction)
|
|
535
|
+
open_revisions = (
|
|
536
|
+
db.query(Revision)
|
|
537
|
+
.filter(
|
|
538
|
+
Revision.ticket_id == ticket_id,
|
|
539
|
+
Revision.status == RevisionStatus.OPEN.value,
|
|
540
|
+
)
|
|
541
|
+
.all()
|
|
542
|
+
)
|
|
543
|
+
for rev in open_revisions:
|
|
544
|
+
rev.status = RevisionStatus.SUPERSEDED.value
|
|
545
|
+
|
|
546
|
+
# Get next revision number
|
|
547
|
+
last_revision = (
|
|
548
|
+
db.query(Revision)
|
|
549
|
+
.filter(Revision.ticket_id == ticket_id)
|
|
550
|
+
.order_by(Revision.number.desc())
|
|
551
|
+
.first()
|
|
552
|
+
)
|
|
553
|
+
next_number = (last_revision.number if last_revision else 0) + 1
|
|
554
|
+
|
|
555
|
+
# Create new revision
|
|
556
|
+
revision = Revision(
|
|
557
|
+
ticket_id=ticket_id,
|
|
558
|
+
job_id=job_id,
|
|
559
|
+
number=next_number,
|
|
560
|
+
status=RevisionStatus.OPEN.value,
|
|
561
|
+
diff_stat_evidence_id=diff_stat_evidence_id,
|
|
562
|
+
diff_patch_evidence_id=diff_patch_evidence_id,
|
|
563
|
+
)
|
|
564
|
+
db.add(revision)
|
|
565
|
+
db.commit()
|
|
566
|
+
db.refresh(revision)
|
|
567
|
+
return revision.id
|
|
568
|
+
except Exception as e:
|
|
569
|
+
db.rollback()
|
|
570
|
+
# Log but don't fail the job
|
|
571
|
+
import logging
|
|
572
|
+
|
|
573
|
+
logging.getLogger(__name__).error(f"Failed to create revision: {e}")
|
|
574
|
+
return None
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def get_feedback_bundle_for_ticket(ticket_id: str) -> dict | None:
|
|
578
|
+
"""Get the feedback bundle from the most recent changes_requested revision.
|
|
579
|
+
|
|
580
|
+
This is used when re-running an execute job after changes were requested.
|
|
581
|
+
|
|
582
|
+
Args:
|
|
583
|
+
ticket_id: UUID of the ticket
|
|
584
|
+
|
|
585
|
+
Returns:
|
|
586
|
+
Feedback bundle dict if found, None otherwise
|
|
587
|
+
"""
|
|
588
|
+
from app.models.review_comment import ReviewComment
|
|
589
|
+
from app.models.review_summary import ReviewSummary
|
|
590
|
+
from app.models.revision import Revision, RevisionStatus
|
|
591
|
+
|
|
592
|
+
with get_sync_db() as db:
|
|
593
|
+
try:
|
|
594
|
+
# Find the most recent revision with changes_requested status
|
|
595
|
+
revision = (
|
|
596
|
+
db.query(Revision)
|
|
597
|
+
.filter(
|
|
598
|
+
Revision.ticket_id == ticket_id,
|
|
599
|
+
Revision.status == RevisionStatus.CHANGES_REQUESTED.value,
|
|
600
|
+
)
|
|
601
|
+
.order_by(Revision.number.desc())
|
|
602
|
+
.first()
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
if not revision:
|
|
606
|
+
return None
|
|
607
|
+
|
|
608
|
+
# Get review summary
|
|
609
|
+
review_summary = (
|
|
610
|
+
db.query(ReviewSummary)
|
|
611
|
+
.filter(ReviewSummary.revision_id == revision.id)
|
|
612
|
+
.first()
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
# Get unresolved comments
|
|
616
|
+
comments = (
|
|
617
|
+
db.query(ReviewComment)
|
|
618
|
+
.filter(
|
|
619
|
+
ReviewComment.revision_id == revision.id,
|
|
620
|
+
ReviewComment.resolved == False, # noqa: E712
|
|
621
|
+
)
|
|
622
|
+
.order_by(ReviewComment.created_at)
|
|
623
|
+
.all()
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
# Build feedback bundle
|
|
627
|
+
return {
|
|
628
|
+
"ticket_id": ticket_id,
|
|
629
|
+
"revision_id": revision.id,
|
|
630
|
+
"revision_number": revision.number,
|
|
631
|
+
"decision": "changes_requested",
|
|
632
|
+
"summary": review_summary.body if review_summary else "",
|
|
633
|
+
"comments": [
|
|
634
|
+
{
|
|
635
|
+
"file_path": c.file_path,
|
|
636
|
+
"line_number": c.line_number,
|
|
637
|
+
"anchor": c.anchor,
|
|
638
|
+
"body": c.body,
|
|
639
|
+
"line_content": c.line_content,
|
|
640
|
+
}
|
|
641
|
+
for c in comments
|
|
642
|
+
],
|
|
643
|
+
}
|
|
644
|
+
except Exception as e:
|
|
645
|
+
import logging
|
|
646
|
+
|
|
647
|
+
logging.getLogger(__name__).error(f"Failed to get feedback bundle: {e}")
|
|
648
|
+
return None
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def transition_ticket_sync(
|
|
652
|
+
ticket_id: str,
|
|
653
|
+
to_state: TicketState,
|
|
654
|
+
reason: str | None = None,
|
|
655
|
+
payload: dict | None = None,
|
|
656
|
+
actor_id: str = "worker",
|
|
657
|
+
auto_verify: bool = True,
|
|
658
|
+
) -> None:
|
|
659
|
+
"""
|
|
660
|
+
Transition a ticket to a new state synchronously.
|
|
661
|
+
|
|
662
|
+
Args:
|
|
663
|
+
ticket_id: The UUID of the ticket
|
|
664
|
+
to_state: The target state
|
|
665
|
+
reason: Optional reason for the transition
|
|
666
|
+
payload: Optional payload for the event
|
|
667
|
+
actor_id: The ID of the actor performing the transition
|
|
668
|
+
auto_verify: If True, auto-enqueue verify job when entering verifying state
|
|
669
|
+
"""
|
|
670
|
+
with get_sync_db() as db:
|
|
671
|
+
ticket = db.query(Ticket).filter(Ticket.id == ticket_id).first()
|
|
672
|
+
if not ticket:
|
|
673
|
+
return
|
|
674
|
+
|
|
675
|
+
from_state = ticket.state
|
|
676
|
+
|
|
677
|
+
# Skip no-op transitions (already in target state)
|
|
678
|
+
if from_state == to_state.value:
|
|
679
|
+
logger.info(
|
|
680
|
+
f"Skipping no-op transition for ticket {ticket_id}: already in {from_state}"
|
|
681
|
+
)
|
|
682
|
+
return
|
|
683
|
+
|
|
684
|
+
# Validate state transition against the state machine
|
|
685
|
+
from app.state_machine import validate_transition as sm_validate
|
|
686
|
+
|
|
687
|
+
try:
|
|
688
|
+
from_state_enum = TicketState(from_state)
|
|
689
|
+
except ValueError:
|
|
690
|
+
logger.error(f"Invalid current state '{from_state}' for ticket {ticket_id}")
|
|
691
|
+
return
|
|
692
|
+
|
|
693
|
+
if not sm_validate(from_state_enum, to_state):
|
|
694
|
+
logger.error(
|
|
695
|
+
f"Invalid state transition {from_state} -> {to_state.value} "
|
|
696
|
+
f"for ticket {ticket_id}, skipping"
|
|
697
|
+
)
|
|
698
|
+
return
|
|
699
|
+
|
|
700
|
+
ticket.state = to_state.value
|
|
701
|
+
board_id = ticket.board_id
|
|
702
|
+
|
|
703
|
+
# Create transition event
|
|
704
|
+
event = TicketEvent(
|
|
705
|
+
ticket_id=ticket_id,
|
|
706
|
+
event_type=EventType.TRANSITIONED.value,
|
|
707
|
+
from_state=from_state,
|
|
708
|
+
to_state=to_state.value,
|
|
709
|
+
actor_type=ActorType.EXECUTOR.value,
|
|
710
|
+
actor_id=actor_id,
|
|
711
|
+
reason=reason,
|
|
712
|
+
payload_json=json.dumps(payload) if payload else None,
|
|
713
|
+
)
|
|
714
|
+
db.add(event)
|
|
715
|
+
db.commit()
|
|
716
|
+
|
|
717
|
+
# Broadcast board invalidation via WebSocket
|
|
718
|
+
if board_id:
|
|
719
|
+
from app.websocket.manager import broadcast_sync
|
|
720
|
+
|
|
721
|
+
broadcast_sync(
|
|
722
|
+
f"board:{board_id}",
|
|
723
|
+
{"type": "invalidate", "reason": "ticket_transition"},
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
# Auto-trigger verification when entering verifying state
|
|
727
|
+
if auto_verify and to_state == TicketState.VERIFYING:
|
|
728
|
+
_enqueue_verify_job_sync(ticket_id)
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _enqueue_verify_job_sync(ticket_id: str) -> str | None:
|
|
732
|
+
"""
|
|
733
|
+
Synchronously enqueue a verify job for a ticket (idempotent).
|
|
734
|
+
|
|
735
|
+
Idempotency: Only creates a new verify job if there is no active
|
|
736
|
+
(queued or running) verify job for this ticket. This prevents
|
|
737
|
+
duplicate verify jobs from race conditions or retries.
|
|
738
|
+
|
|
739
|
+
Returns:
|
|
740
|
+
The job ID if created, None if skipped (already active).
|
|
741
|
+
"""
|
|
742
|
+
from app.models.job import Job, JobKind, JobStatus
|
|
743
|
+
|
|
744
|
+
with get_sync_db() as db:
|
|
745
|
+
# IDEMPOTENCY CHECK: Is there already an active verify job?
|
|
746
|
+
active_verify = (
|
|
747
|
+
db.query(Job)
|
|
748
|
+
.filter(
|
|
749
|
+
Job.ticket_id == ticket_id,
|
|
750
|
+
Job.kind == JobKind.VERIFY.value,
|
|
751
|
+
Job.status.in_([JobStatus.QUEUED.value, JobStatus.RUNNING.value]),
|
|
752
|
+
)
|
|
753
|
+
.first()
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
if active_verify:
|
|
757
|
+
# Already has an active verify job - skip to avoid duplicates
|
|
758
|
+
return None
|
|
759
|
+
|
|
760
|
+
# Get the ticket to inherit board_id
|
|
761
|
+
ticket = db.query(Ticket).filter(Ticket.id == ticket_id).first()
|
|
762
|
+
board_id = ticket.board_id if ticket else None
|
|
763
|
+
|
|
764
|
+
# Create the job record with board_id for permission scoping
|
|
765
|
+
job = Job(
|
|
766
|
+
ticket_id=ticket_id,
|
|
767
|
+
board_id=board_id,
|
|
768
|
+
kind=JobKind.VERIFY.value,
|
|
769
|
+
status=JobStatus.QUEUED.value,
|
|
770
|
+
)
|
|
771
|
+
db.add(job)
|
|
772
|
+
db.flush()
|
|
773
|
+
job_id = job.id
|
|
774
|
+
# Commit BEFORE enqueue_task to release the SQLite write lock.
|
|
775
|
+
# enqueue_task opens a separate sqlite3 connection which would
|
|
776
|
+
# deadlock if this session still holds the write lock.
|
|
777
|
+
db.commit()
|
|
778
|
+
|
|
779
|
+
# Enqueue the verify task (outside the db session to avoid deadlock)
|
|
780
|
+
from app.services.task_dispatch import enqueue_task
|
|
781
|
+
|
|
782
|
+
task = enqueue_task("verify_ticket", args=[job_id])
|
|
783
|
+
|
|
784
|
+
# Update the job with the task ID
|
|
785
|
+
with get_sync_db() as db:
|
|
786
|
+
job = db.query(Job).filter(Job.id == job_id).first()
|
|
787
|
+
if job:
|
|
788
|
+
job.celery_task_id = task.id
|
|
789
|
+
|
|
790
|
+
return job_id
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def run_executor_cli(
|
|
794
|
+
command: list[str],
|
|
795
|
+
cwd: Path,
|
|
796
|
+
evidence_dir: Path,
|
|
797
|
+
evidence_id: str,
|
|
798
|
+
repo_root: Path,
|
|
799
|
+
timeout: int = 600,
|
|
800
|
+
job_id: str | None = None,
|
|
801
|
+
normalize_logs: bool = False,
|
|
802
|
+
stdin_content: str | None = None,
|
|
803
|
+
) -> tuple[int, str, str]:
|
|
804
|
+
"""
|
|
805
|
+
Run the executor CLI and capture output with real-time streaming.
|
|
806
|
+
|
|
807
|
+
Args:
|
|
808
|
+
command: The CLI command to run as a list of arguments
|
|
809
|
+
cwd: Working directory for the command
|
|
810
|
+
evidence_dir: Directory to store stdout/stderr files
|
|
811
|
+
evidence_id: UUID for naming evidence files
|
|
812
|
+
repo_root: Path to repo root (for computing relative paths)
|
|
813
|
+
timeout: Command timeout in seconds
|
|
814
|
+
job_id: Optional job ID for real-time log streaming
|
|
815
|
+
normalize_logs: If True, parse cursor-agent JSON and stream normalized entries
|
|
816
|
+
stdin_content: Optional content to pipe to the process via stdin
|
|
817
|
+
|
|
818
|
+
Returns:
|
|
819
|
+
Tuple of (exit_code, stdout_relpath, stderr_relpath) - paths are relative to repo_root
|
|
820
|
+
"""
|
|
821
|
+
import json as json_module
|
|
822
|
+
import threading
|
|
823
|
+
|
|
824
|
+
stdout_path = evidence_dir / f"{evidence_id}.stdout"
|
|
825
|
+
stderr_path = evidence_dir / f"{evidence_id}.stderr"
|
|
826
|
+
|
|
827
|
+
stdout_lines: list[str] = []
|
|
828
|
+
stderr_lines: list[str] = []
|
|
829
|
+
|
|
830
|
+
# Create normalizer if needed
|
|
831
|
+
normalizer = CursorLogNormalizer(str(cwd)) if normalize_logs else None
|
|
832
|
+
|
|
833
|
+
# Strip Claude Code session env vars to avoid "nested session" errors
|
|
834
|
+
# when spawning claude CLI from within a Claude Code session
|
|
835
|
+
clean_env = {
|
|
836
|
+
k: v
|
|
837
|
+
for k, v in os.environ.items()
|
|
838
|
+
if k not in ("CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT")
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
try:
|
|
842
|
+
# Use Popen for real-time streaming instead of blocking run()
|
|
843
|
+
process = subprocess.Popen(
|
|
844
|
+
command,
|
|
845
|
+
cwd=cwd,
|
|
846
|
+
stdin=subprocess.PIPE if stdin_content else None,
|
|
847
|
+
stdout=subprocess.PIPE,
|
|
848
|
+
stderr=subprocess.PIPE,
|
|
849
|
+
text=True,
|
|
850
|
+
bufsize=1, # Line buffered
|
|
851
|
+
env=clean_env,
|
|
852
|
+
)
|
|
853
|
+
|
|
854
|
+
# Write stdin content and close to signal EOF
|
|
855
|
+
if stdin_content and process.stdin:
|
|
856
|
+
process.stdin.write(stdin_content)
|
|
857
|
+
process.stdin.close()
|
|
858
|
+
|
|
859
|
+
def stream_output(pipe, lines_list, is_stderr=False, stop_event=None):
|
|
860
|
+
"""Read and stream output line by line with stop event support."""
|
|
861
|
+
try:
|
|
862
|
+
for line in iter(pipe.readline, ""):
|
|
863
|
+
# Check if we should stop
|
|
864
|
+
if stop_event and stop_event.is_set():
|
|
865
|
+
logger.debug(
|
|
866
|
+
f"Stream thread stopping due to stop_event ({'stderr' if is_stderr else 'stdout'})"
|
|
867
|
+
)
|
|
868
|
+
break
|
|
869
|
+
if not line:
|
|
870
|
+
break
|
|
871
|
+
line = line.rstrip("\n")
|
|
872
|
+
lines_list.append(line)
|
|
873
|
+
|
|
874
|
+
# Stream to Redis for real-time SSE
|
|
875
|
+
if job_id:
|
|
876
|
+
try:
|
|
877
|
+
if is_stderr:
|
|
878
|
+
log_stream_publisher.push_stderr(job_id, line)
|
|
879
|
+
elif normalizer:
|
|
880
|
+
# Parse and stream normalized entries
|
|
881
|
+
entries = normalizer.process_line(line)
|
|
882
|
+
for entry in entries:
|
|
883
|
+
# Serialize normalized entry as JSON
|
|
884
|
+
entry_data = {
|
|
885
|
+
"entry_type": entry.entry_type.value,
|
|
886
|
+
"content": entry.content,
|
|
887
|
+
"sequence": entry.sequence,
|
|
888
|
+
"tool_name": entry.tool_name,
|
|
889
|
+
"action_type": entry.action_type.value
|
|
890
|
+
if entry.action_type
|
|
891
|
+
else None,
|
|
892
|
+
"tool_status": entry.tool_status.value
|
|
893
|
+
if entry.tool_status
|
|
894
|
+
else None,
|
|
895
|
+
"metadata": entry.metadata,
|
|
896
|
+
}
|
|
897
|
+
log_stream_publisher.push(
|
|
898
|
+
job_id,
|
|
899
|
+
LogLevel.NORMALIZED,
|
|
900
|
+
json_module.dumps(entry_data),
|
|
901
|
+
)
|
|
902
|
+
else:
|
|
903
|
+
log_stream_publisher.push_stdout(job_id, line)
|
|
904
|
+
except Exception:
|
|
905
|
+
pass # Don't fail execution if streaming fails
|
|
906
|
+
finally:
|
|
907
|
+
pipe.close()
|
|
908
|
+
|
|
909
|
+
# Create stop events for graceful thread termination
|
|
910
|
+
stdout_stop_event = threading.Event()
|
|
911
|
+
stderr_stop_event = threading.Event()
|
|
912
|
+
|
|
913
|
+
# Stream stdout and stderr in parallel threads
|
|
914
|
+
# Use daemon=True so threads don't block process exit if they get stuck
|
|
915
|
+
stdout_thread = threading.Thread(
|
|
916
|
+
target=stream_output,
|
|
917
|
+
args=(process.stdout, stdout_lines, False, stdout_stop_event),
|
|
918
|
+
daemon=True,
|
|
919
|
+
name=f"stdout-{job_id[:8] if job_id else 'unknown'}",
|
|
920
|
+
)
|
|
921
|
+
stderr_thread = threading.Thread(
|
|
922
|
+
target=stream_output,
|
|
923
|
+
args=(process.stderr, stderr_lines, True, stderr_stop_event),
|
|
924
|
+
daemon=True,
|
|
925
|
+
name=f"stderr-{job_id[:8] if job_id else 'unknown'}",
|
|
926
|
+
)
|
|
927
|
+
|
|
928
|
+
stdout_thread.start()
|
|
929
|
+
stderr_thread.start()
|
|
930
|
+
|
|
931
|
+
# Register process for cancellation support
|
|
932
|
+
if job_id:
|
|
933
|
+
register_active_process(job_id, process)
|
|
934
|
+
|
|
935
|
+
# Wait for process with timeout AND poll for cancellation
|
|
936
|
+
try:
|
|
937
|
+
start_time = time.time()
|
|
938
|
+
last_heartbeat_time = start_time
|
|
939
|
+
while True:
|
|
940
|
+
# Check if process finished
|
|
941
|
+
exit_code = process.poll()
|
|
942
|
+
if exit_code is not None:
|
|
943
|
+
break
|
|
944
|
+
|
|
945
|
+
# Update heartbeat every 30 seconds to prevent watchdog from killing long-running jobs
|
|
946
|
+
now = time.time()
|
|
947
|
+
if job_id and now - last_heartbeat_time >= 30:
|
|
948
|
+
update_job_heartbeat(job_id)
|
|
949
|
+
last_heartbeat_time = now
|
|
950
|
+
|
|
951
|
+
# Check if job was canceled
|
|
952
|
+
if job_id and check_canceled(job_id):
|
|
953
|
+
logger.info(
|
|
954
|
+
f"Job {job_id} canceled, attempting graceful shutdown of process {process.pid}"
|
|
955
|
+
)
|
|
956
|
+
# Try SIGTERM first for graceful shutdown
|
|
957
|
+
process.terminate()
|
|
958
|
+
try:
|
|
959
|
+
process.wait(timeout=5)
|
|
960
|
+
logger.info(f"Process {process.pid} terminated gracefully")
|
|
961
|
+
except subprocess.TimeoutExpired:
|
|
962
|
+
logger.warning(
|
|
963
|
+
f"Process {process.pid} did not terminate gracefully, using SIGKILL"
|
|
964
|
+
)
|
|
965
|
+
process.kill() # Force kill if still alive
|
|
966
|
+
process.wait()
|
|
967
|
+
stdout_lines.append("\n[CANCELED] Job canceled by user")
|
|
968
|
+
exit_code = -2 # Special exit code for cancellation
|
|
969
|
+
break
|
|
970
|
+
|
|
971
|
+
# Check timeout
|
|
972
|
+
if time.time() - start_time > timeout:
|
|
973
|
+
logger.warning(
|
|
974
|
+
f"Process {process.pid} exceeded timeout {timeout}s, attempting graceful shutdown"
|
|
975
|
+
)
|
|
976
|
+
# Try SIGTERM first for graceful shutdown
|
|
977
|
+
process.terminate()
|
|
978
|
+
try:
|
|
979
|
+
process.wait(timeout=5)
|
|
980
|
+
logger.info(
|
|
981
|
+
f"Process {process.pid} terminated gracefully after timeout"
|
|
982
|
+
)
|
|
983
|
+
except subprocess.TimeoutExpired:
|
|
984
|
+
logger.warning(
|
|
985
|
+
f"Process {process.pid} did not terminate gracefully, using SIGKILL"
|
|
986
|
+
)
|
|
987
|
+
process.kill()
|
|
988
|
+
process.wait()
|
|
989
|
+
stdout_lines.append(
|
|
990
|
+
f"\n[TIMEOUT] Process killed after {timeout} seconds"
|
|
991
|
+
)
|
|
992
|
+
exit_code = -1
|
|
993
|
+
break
|
|
994
|
+
|
|
995
|
+
# Poll every second
|
|
996
|
+
time.sleep(1)
|
|
997
|
+
|
|
998
|
+
finally:
|
|
999
|
+
# Unregister process
|
|
1000
|
+
if job_id:
|
|
1001
|
+
unregister_active_process(job_id)
|
|
1002
|
+
|
|
1003
|
+
# Signal threads to stop gracefully
|
|
1004
|
+
stdout_stop_event.set()
|
|
1005
|
+
stderr_stop_event.set()
|
|
1006
|
+
|
|
1007
|
+
# Close pipes to unblock threads waiting on readline()
|
|
1008
|
+
if process.stdout:
|
|
1009
|
+
try:
|
|
1010
|
+
process.stdout.close()
|
|
1011
|
+
except Exception:
|
|
1012
|
+
pass
|
|
1013
|
+
if process.stderr:
|
|
1014
|
+
try:
|
|
1015
|
+
process.stderr.close()
|
|
1016
|
+
except Exception:
|
|
1017
|
+
pass
|
|
1018
|
+
|
|
1019
|
+
# Wait for output threads to finish (with timeout to prevent blocking)
|
|
1020
|
+
# Try longer timeout first (10s)
|
|
1021
|
+
stdout_thread.join(timeout=10)
|
|
1022
|
+
stderr_thread.join(timeout=10)
|
|
1023
|
+
|
|
1024
|
+
# CRITICAL: If threads didn't stop, force cleanup
|
|
1025
|
+
threads_stuck = False
|
|
1026
|
+
if stdout_thread.is_alive():
|
|
1027
|
+
logger.error(
|
|
1028
|
+
f"stdout thread for job {job_id[:8] if job_id else 'unknown'} did not stop after 10s - forcing cleanup"
|
|
1029
|
+
)
|
|
1030
|
+
threads_stuck = True
|
|
1031
|
+
# Thread will be GC'd because daemon=True, but we lost some output
|
|
1032
|
+
# This is acceptable - better than blocking indefinitely
|
|
1033
|
+
|
|
1034
|
+
if stderr_thread.is_alive():
|
|
1035
|
+
logger.error(
|
|
1036
|
+
f"stderr thread for job {job_id[:8] if job_id else 'unknown'} did not stop after 10s - forcing cleanup"
|
|
1037
|
+
)
|
|
1038
|
+
threads_stuck = True
|
|
1039
|
+
|
|
1040
|
+
if threads_stuck:
|
|
1041
|
+
# Add warning to output that some logs may be incomplete
|
|
1042
|
+
stdout_lines.append(
|
|
1043
|
+
"\n[WARNING] Output streaming threads did not terminate cleanly - some logs may be incomplete"
|
|
1044
|
+
)
|
|
1045
|
+
|
|
1046
|
+
# Write captured output to files
|
|
1047
|
+
stdout_path.write_text("\n".join(stdout_lines))
|
|
1048
|
+
stderr_path.write_text("\n".join(stderr_lines))
|
|
1049
|
+
|
|
1050
|
+
# Return relative paths for secure DB storage
|
|
1051
|
+
stdout_rel = str(stdout_path)
|
|
1052
|
+
stderr_rel = str(stderr_path)
|
|
1053
|
+
return exit_code, stdout_rel, stderr_rel
|
|
1054
|
+
|
|
1055
|
+
except subprocess.TimeoutExpired as e:
|
|
1056
|
+
# Write partial output if available
|
|
1057
|
+
stdout_path.write_text(
|
|
1058
|
+
e.stdout.decode()
|
|
1059
|
+
if e.stdout
|
|
1060
|
+
else f"Command timed out after {timeout} seconds"
|
|
1061
|
+
)
|
|
1062
|
+
stderr_path.write_text(e.stderr.decode() if e.stderr else "")
|
|
1063
|
+
stdout_rel = str(stdout_path)
|
|
1064
|
+
stderr_rel = str(stderr_path)
|
|
1065
|
+
return -1, stdout_rel, stderr_rel
|
|
1066
|
+
|
|
1067
|
+
except FileNotFoundError as e:
|
|
1068
|
+
stdout_path.write_text("")
|
|
1069
|
+
stderr_path.write_text(f"Executor CLI not found: {str(e)}")
|
|
1070
|
+
stdout_rel = str(stdout_path)
|
|
1071
|
+
stderr_rel = str(stderr_path)
|
|
1072
|
+
return -1, stdout_rel, stderr_rel
|
|
1073
|
+
|
|
1074
|
+
except Exception as e:
|
|
1075
|
+
stdout_path.write_text("")
|
|
1076
|
+
stderr_path.write_text(f"Error running executor CLI: {str(e)}")
|
|
1077
|
+
stdout_rel = str(stdout_path)
|
|
1078
|
+
stderr_rel = str(stderr_path)
|
|
1079
|
+
return -1, stdout_rel, stderr_rel
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def run_executor_cli_with_retry(
|
|
1083
|
+
command: list[str],
|
|
1084
|
+
cwd: Path,
|
|
1085
|
+
evidence_dir: Path,
|
|
1086
|
+
evidence_id: str,
|
|
1087
|
+
repo_root: Path,
|
|
1088
|
+
timeout: int = 600,
|
|
1089
|
+
job_id: str | None = None,
|
|
1090
|
+
normalize_logs: bool = False,
|
|
1091
|
+
stdin_content: str | None = None,
|
|
1092
|
+
log_path: Path | None = None,
|
|
1093
|
+
) -> tuple[int, str, str]:
|
|
1094
|
+
"""
|
|
1095
|
+
Wrapper for run_executor_cli with retry logic for transient failures.
|
|
1096
|
+
|
|
1097
|
+
Retries up to EXECUTOR_MAX_RETRIES times with exponential backoff when:
|
|
1098
|
+
- Exit code is in RETRYABLE_EXIT_CODES (timeout, kill, term)
|
|
1099
|
+
- Stderr contains known transient error patterns (network, rate limit, etc.)
|
|
1100
|
+
|
|
1101
|
+
Args:
|
|
1102
|
+
Same as run_executor_cli, plus:
|
|
1103
|
+
log_path: Optional path to write retry messages
|
|
1104
|
+
|
|
1105
|
+
Returns:
|
|
1106
|
+
Same as run_executor_cli: (exit_code, stdout_relpath, stderr_relpath)
|
|
1107
|
+
"""
|
|
1108
|
+
last_exit_code = -1
|
|
1109
|
+
last_stdout_path = ""
|
|
1110
|
+
last_stderr_path = ""
|
|
1111
|
+
|
|
1112
|
+
for attempt in range(EXECUTOR_MAX_RETRIES + 1): # 0, 1, 2 = 3 total attempts
|
|
1113
|
+
if attempt > 0:
|
|
1114
|
+
# This is a retry - calculate backoff delay
|
|
1115
|
+
delay = EXECUTOR_RETRY_DELAY_BASE * (
|
|
1116
|
+
2 ** (attempt - 1)
|
|
1117
|
+
) # Exponential backoff: 5s, 10s
|
|
1118
|
+
retry_msg = f"Retry attempt {attempt}/{EXECUTOR_MAX_RETRIES} after {delay}s delay..."
|
|
1119
|
+
logger.info(retry_msg)
|
|
1120
|
+
if log_path:
|
|
1121
|
+
write_log(log_path, retry_msg, job_id)
|
|
1122
|
+
time.sleep(delay)
|
|
1123
|
+
|
|
1124
|
+
# Run the executor
|
|
1125
|
+
exit_code, stdout_rel, stderr_rel = run_executor_cli(
|
|
1126
|
+
command=command,
|
|
1127
|
+
cwd=cwd,
|
|
1128
|
+
evidence_dir=evidence_dir,
|
|
1129
|
+
evidence_id=f"{evidence_id}_attempt{attempt}"
|
|
1130
|
+
if attempt > 0
|
|
1131
|
+
else evidence_id,
|
|
1132
|
+
repo_root=repo_root,
|
|
1133
|
+
timeout=timeout,
|
|
1134
|
+
job_id=job_id,
|
|
1135
|
+
normalize_logs=normalize_logs,
|
|
1136
|
+
stdin_content=stdin_content,
|
|
1137
|
+
)
|
|
1138
|
+
|
|
1139
|
+
last_exit_code = exit_code
|
|
1140
|
+
last_stdout_path = stdout_rel
|
|
1141
|
+
last_stderr_path = stderr_rel
|
|
1142
|
+
|
|
1143
|
+
# Success - return immediately
|
|
1144
|
+
if exit_code == 0:
|
|
1145
|
+
if attempt > 0:
|
|
1146
|
+
success_msg = f"Executor succeeded on retry attempt {attempt}"
|
|
1147
|
+
logger.info(success_msg)
|
|
1148
|
+
if log_path:
|
|
1149
|
+
write_log(log_path, success_msg, job_id)
|
|
1150
|
+
return exit_code, stdout_rel, stderr_rel
|
|
1151
|
+
|
|
1152
|
+
# Check if this is the last attempt
|
|
1153
|
+
if attempt >= EXECUTOR_MAX_RETRIES:
|
|
1154
|
+
break
|
|
1155
|
+
|
|
1156
|
+
# Check if error is retryable
|
|
1157
|
+
try:
|
|
1158
|
+
stderr_content = (repo_root / stderr_rel).read_text()
|
|
1159
|
+
except Exception:
|
|
1160
|
+
stderr_content = ""
|
|
1161
|
+
|
|
1162
|
+
if is_retryable_error(exit_code, stderr_content):
|
|
1163
|
+
retry_reason = (
|
|
1164
|
+
f"Executor failed with retryable error (exit_code={exit_code})"
|
|
1165
|
+
)
|
|
1166
|
+
logger.warning(retry_reason)
|
|
1167
|
+
if log_path:
|
|
1168
|
+
write_log(log_path, retry_reason, job_id)
|
|
1169
|
+
# Continue to next iteration (retry)
|
|
1170
|
+
else:
|
|
1171
|
+
# Not retryable - fail immediately
|
|
1172
|
+
non_retry_msg = f"Executor failed with non-retryable error (exit_code={exit_code}) - not retrying"
|
|
1173
|
+
logger.info(non_retry_msg)
|
|
1174
|
+
if log_path:
|
|
1175
|
+
write_log(log_path, non_retry_msg, job_id)
|
|
1176
|
+
break
|
|
1177
|
+
|
|
1178
|
+
# All retries exhausted or non-retryable error
|
|
1179
|
+
if attempt > 0:
|
|
1180
|
+
exhausted_msg = f"Executor failed after {attempt + 1} attempts (final exit_code={last_exit_code})"
|
|
1181
|
+
logger.error(exhausted_msg)
|
|
1182
|
+
if log_path:
|
|
1183
|
+
write_log(log_path, exhausted_msg, job_id)
|
|
1184
|
+
|
|
1185
|
+
return last_exit_code, last_stdout_path, last_stderr_path
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
def capture_git_diff(
|
|
1189
|
+
cwd: Path,
|
|
1190
|
+
evidence_dir: Path,
|
|
1191
|
+
evidence_id: str,
|
|
1192
|
+
repo_root: Path,
|
|
1193
|
+
) -> tuple[int, str, str, str, bool]:
|
|
1194
|
+
"""
|
|
1195
|
+
Capture git diff output for changes made in the worktree.
|
|
1196
|
+
|
|
1197
|
+
This function captures both:
|
|
1198
|
+
1. Changes to tracked files (via git diff)
|
|
1199
|
+
2. New untracked files (via git status --porcelain)
|
|
1200
|
+
|
|
1201
|
+
Args:
|
|
1202
|
+
cwd: Working directory (worktree path)
|
|
1203
|
+
evidence_dir: Directory to store diff files
|
|
1204
|
+
evidence_id: UUID for naming evidence files
|
|
1205
|
+
repo_root: Path to repo root (for computing relative paths)
|
|
1206
|
+
|
|
1207
|
+
Returns:
|
|
1208
|
+
Tuple of (exit_code, diff_stat_relpath, diff_patch_relpath, diff_stat_text, has_changes)
|
|
1209
|
+
has_changes is True if there are uncommitted changes OR new untracked files.
|
|
1210
|
+
Paths are relative to repo_root.
|
|
1211
|
+
"""
|
|
1212
|
+
diff_stat_path = evidence_dir / f"{evidence_id}.diff_stat"
|
|
1213
|
+
diff_patch_path = evidence_dir / f"{evidence_id}.diff_patch"
|
|
1214
|
+
stderr_path = evidence_dir / f"{evidence_id}.stderr"
|
|
1215
|
+
|
|
1216
|
+
diff_stat = ""
|
|
1217
|
+
has_changes = False
|
|
1218
|
+
has_tracked_changes = False
|
|
1219
|
+
has_untracked_files = False
|
|
1220
|
+
untracked_files: list[str] = []
|
|
1221
|
+
|
|
1222
|
+
try:
|
|
1223
|
+
# First get the diff stat for tracked file changes (both staged and unstaged)
|
|
1224
|
+
stat_result = subprocess.run(
|
|
1225
|
+
["git", "diff", "HEAD", "--stat"],
|
|
1226
|
+
cwd=cwd,
|
|
1227
|
+
capture_output=True,
|
|
1228
|
+
text=True,
|
|
1229
|
+
timeout=60,
|
|
1230
|
+
)
|
|
1231
|
+
diff_stat = stat_result.stdout.strip() if stat_result.stdout else ""
|
|
1232
|
+
|
|
1233
|
+
# Then get the full patch for tracked files (both staged and unstaged)
|
|
1234
|
+
patch_result = subprocess.run(
|
|
1235
|
+
["git", "diff", "HEAD"],
|
|
1236
|
+
cwd=cwd,
|
|
1237
|
+
capture_output=True,
|
|
1238
|
+
text=True,
|
|
1239
|
+
timeout=60,
|
|
1240
|
+
)
|
|
1241
|
+
diff_patch = patch_result.stdout.strip() if patch_result.stdout else ""
|
|
1242
|
+
has_tracked_changes = bool(diff_patch)
|
|
1243
|
+
|
|
1244
|
+
# Also check for untracked files (new files created by executor)
|
|
1245
|
+
# These won't show up in git diff but represent real work done
|
|
1246
|
+
status_result = subprocess.run(
|
|
1247
|
+
["git", "status", "--porcelain"],
|
|
1248
|
+
cwd=cwd,
|
|
1249
|
+
capture_output=True,
|
|
1250
|
+
text=True,
|
|
1251
|
+
timeout=60,
|
|
1252
|
+
)
|
|
1253
|
+
if status_result.stdout:
|
|
1254
|
+
for line in status_result.stdout.strip().split("\n"):
|
|
1255
|
+
if line.startswith("??"):
|
|
1256
|
+
# Untracked file - extract path (skip "?? " prefix)
|
|
1257
|
+
file_path = line[3:].strip()
|
|
1258
|
+
# Skip .draft directory (internal files)
|
|
1259
|
+
if not file_path.startswith(".draft"):
|
|
1260
|
+
untracked_files.append(file_path)
|
|
1261
|
+
has_untracked_files = len(untracked_files) > 0
|
|
1262
|
+
|
|
1263
|
+
# Determine if there are actual changes (tracked OR untracked)
|
|
1264
|
+
has_changes = has_tracked_changes or has_untracked_files
|
|
1265
|
+
|
|
1266
|
+
# Build comprehensive diff stat that includes untracked files
|
|
1267
|
+
stat_parts = []
|
|
1268
|
+
if diff_stat:
|
|
1269
|
+
stat_parts.append(diff_stat)
|
|
1270
|
+
if untracked_files:
|
|
1271
|
+
stat_parts.append("\nNew files (untracked):")
|
|
1272
|
+
for f in untracked_files[:20]: # Limit to 20 files in summary
|
|
1273
|
+
stat_parts.append(f" + {f}")
|
|
1274
|
+
if len(untracked_files) > 20:
|
|
1275
|
+
stat_parts.append(f" ... and {len(untracked_files) - 20} more")
|
|
1276
|
+
|
|
1277
|
+
final_diff_stat = "\n".join(stat_parts) if stat_parts else "(no changes)"
|
|
1278
|
+
diff_stat_path.write_text(final_diff_stat)
|
|
1279
|
+
|
|
1280
|
+
# For patch, also include untracked file contents if any
|
|
1281
|
+
patch_parts = []
|
|
1282
|
+
if diff_patch:
|
|
1283
|
+
patch_parts.append(diff_patch)
|
|
1284
|
+
if untracked_files:
|
|
1285
|
+
patch_parts.append("\n\n# === New untracked files ===\n")
|
|
1286
|
+
for f in untracked_files[:10]: # Limit to 10 files to avoid huge patches
|
|
1287
|
+
file_full_path = cwd / f
|
|
1288
|
+
if file_full_path.is_file():
|
|
1289
|
+
try:
|
|
1290
|
+
content = file_full_path.read_text()
|
|
1291
|
+
# Truncate large files
|
|
1292
|
+
if len(content) > 5000:
|
|
1293
|
+
content = content[:5000] + "\n... (truncated)"
|
|
1294
|
+
patch_parts.append(f"\n# +++ {f}\n{content}")
|
|
1295
|
+
except Exception:
|
|
1296
|
+
patch_parts.append(f"\n# +++ {f} (could not read)")
|
|
1297
|
+
|
|
1298
|
+
final_patch = "\n".join(patch_parts) if patch_parts else "(no changes)"
|
|
1299
|
+
diff_patch_path.write_text(final_patch)
|
|
1300
|
+
|
|
1301
|
+
# Combine stderr from commands
|
|
1302
|
+
combined_stderr = ""
|
|
1303
|
+
if stat_result.stderr:
|
|
1304
|
+
combined_stderr += f"git diff --stat stderr:\n{stat_result.stderr}\n"
|
|
1305
|
+
if patch_result.stderr:
|
|
1306
|
+
combined_stderr += f"git diff stderr:\n{patch_result.stderr}\n"
|
|
1307
|
+
if status_result.stderr:
|
|
1308
|
+
combined_stderr += f"git status stderr:\n{status_result.stderr}\n"
|
|
1309
|
+
stderr_path.write_text(combined_stderr)
|
|
1310
|
+
|
|
1311
|
+
# Return relative paths for secure DB storage
|
|
1312
|
+
diff_stat_rel = str(diff_stat_path)
|
|
1313
|
+
diff_patch_rel = str(diff_patch_path)
|
|
1314
|
+
return 0, diff_stat_rel, diff_patch_rel, final_diff_stat, has_changes
|
|
1315
|
+
|
|
1316
|
+
except subprocess.TimeoutExpired:
|
|
1317
|
+
diff_stat_path.write_text("Git diff timed out")
|
|
1318
|
+
diff_patch_path.write_text("")
|
|
1319
|
+
stderr_path.write_text("Git diff command timed out after 60 seconds")
|
|
1320
|
+
diff_stat_rel = str(diff_stat_path)
|
|
1321
|
+
diff_patch_rel = str(diff_patch_path)
|
|
1322
|
+
return -1, diff_stat_rel, diff_patch_rel, "(timeout)", False
|
|
1323
|
+
|
|
1324
|
+
except Exception as e:
|
|
1325
|
+
diff_stat_path.write_text("")
|
|
1326
|
+
diff_patch_path.write_text("")
|
|
1327
|
+
stderr_path.write_text(f"Error running git diff: {str(e)}")
|
|
1328
|
+
diff_stat_rel = str(diff_stat_path)
|
|
1329
|
+
diff_patch_rel = str(diff_patch_path)
|
|
1330
|
+
return -1, diff_stat_rel, diff_patch_rel, "(error)", False
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
# =============================================================================
|
|
1334
|
+
# NO-CHANGES ANALYSIS (LLM-powered)
|
|
1335
|
+
# =============================================================================
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
@dataclass
|
|
1339
|
+
class NoChangesAnalysis:
|
|
1340
|
+
"""Result of analyzing why no code changes were produced."""
|
|
1341
|
+
|
|
1342
|
+
reason: str # Human-readable explanation
|
|
1343
|
+
needs_code_changes: bool # True if code changes are actually needed
|
|
1344
|
+
requires_manual_work: bool # True if manual human intervention is required
|
|
1345
|
+
manual_work_description: str | None # Description of manual work needed (if any)
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
def analyze_no_changes_reason(
|
|
1349
|
+
ticket_title: str,
|
|
1350
|
+
ticket_description: str | None,
|
|
1351
|
+
executor_stdout: str,
|
|
1352
|
+
planner_config: PlannerConfig,
|
|
1353
|
+
) -> NoChangesAnalysis:
|
|
1354
|
+
"""
|
|
1355
|
+
Analyze executor output to determine why no code changes were produced.
|
|
1356
|
+
|
|
1357
|
+
Uses LLM to understand the executor's reasoning and categorize the result:
|
|
1358
|
+
1. No changes needed - the task doesn't require code modifications
|
|
1359
|
+
2. Manual work required - needs human intervention (config, external tools, etc.)
|
|
1360
|
+
3. Unclear/error - couldn't determine, needs investigation
|
|
1361
|
+
|
|
1362
|
+
Args:
|
|
1363
|
+
ticket_title: Title of the ticket being executed
|
|
1364
|
+
ticket_description: Description of the ticket
|
|
1365
|
+
executor_stdout: The stdout output from the executor
|
|
1366
|
+
planner_config: Planner configuration for LLM settings
|
|
1367
|
+
|
|
1368
|
+
Returns:
|
|
1369
|
+
NoChangesAnalysis with categorized result
|
|
1370
|
+
"""
|
|
1371
|
+
import logging
|
|
1372
|
+
|
|
1373
|
+
from app.services.llm_service import LLMService
|
|
1374
|
+
|
|
1375
|
+
logger = logging.getLogger(__name__)
|
|
1376
|
+
|
|
1377
|
+
# Truncate executor output to avoid token limits
|
|
1378
|
+
max_output_chars = 8000
|
|
1379
|
+
truncated_stdout = executor_stdout[:max_output_chars]
|
|
1380
|
+
if len(executor_stdout) > max_output_chars:
|
|
1381
|
+
truncated_stdout += "\n... (output truncated)"
|
|
1382
|
+
|
|
1383
|
+
system_prompt = """You are a technical analyst reviewing why a coding agent completed a task without making any code changes.
|
|
1384
|
+
|
|
1385
|
+
Analyze the executor output and categorize the result into ONE of these categories:
|
|
1386
|
+
|
|
1387
|
+
1. NO_CHANGES_NEEDED - The task genuinely doesn't require code changes. Examples:
|
|
1388
|
+
- The requested functionality already exists
|
|
1389
|
+
- The code is already correct as-is
|
|
1390
|
+
- The task was a review/analysis that doesn't need modifications
|
|
1391
|
+
|
|
1392
|
+
2. MANUAL_WORK_REQUIRED - The task requires human intervention that the agent cannot do. Examples:
|
|
1393
|
+
- Configuration changes in external systems
|
|
1394
|
+
- Running commands that require special permissions
|
|
1395
|
+
- Setting up environment variables or secrets
|
|
1396
|
+
- Installing system packages
|
|
1397
|
+
- Deploying or running external services
|
|
1398
|
+
- Manual testing or verification steps
|
|
1399
|
+
|
|
1400
|
+
3. NEEDS_INVESTIGATION - Unable to determine clearly, needs human review
|
|
1401
|
+
|
|
1402
|
+
Your response MUST be valid JSON with this exact structure:
|
|
1403
|
+
{
|
|
1404
|
+
"category": "NO_CHANGES_NEEDED" | "MANUAL_WORK_REQUIRED" | "NEEDS_INVESTIGATION",
|
|
1405
|
+
"reason": "Brief explanation of why no code changes were made",
|
|
1406
|
+
"manual_work_description": "If MANUAL_WORK_REQUIRED, describe exactly what needs to be done manually. Otherwise null."
|
|
1407
|
+
}"""
|
|
1408
|
+
|
|
1409
|
+
user_prompt = f"""A coding agent was asked to work on this ticket but produced no code changes.
|
|
1410
|
+
|
|
1411
|
+
TICKET TITLE: {ticket_title}
|
|
1412
|
+
|
|
1413
|
+
TICKET DESCRIPTION:
|
|
1414
|
+
{ticket_description or "(no description)"}
|
|
1415
|
+
|
|
1416
|
+
EXECUTOR OUTPUT:
|
|
1417
|
+
{truncated_stdout}
|
|
1418
|
+
|
|
1419
|
+
Analyze why no code changes were produced and categorize the result."""
|
|
1420
|
+
|
|
1421
|
+
try:
|
|
1422
|
+
llm_service = LLMService(planner_config)
|
|
1423
|
+
response = llm_service.call_completion(
|
|
1424
|
+
messages=[{"role": "user", "content": user_prompt}],
|
|
1425
|
+
max_tokens=500,
|
|
1426
|
+
system_prompt=system_prompt,
|
|
1427
|
+
timeout=30,
|
|
1428
|
+
)
|
|
1429
|
+
data = llm_service.safe_parse_json(response.content, {})
|
|
1430
|
+
|
|
1431
|
+
category = data.get("category", "NEEDS_INVESTIGATION")
|
|
1432
|
+
reason = data.get("reason", "Unable to determine why no changes were produced")
|
|
1433
|
+
manual_work_desc = data.get("manual_work_description")
|
|
1434
|
+
|
|
1435
|
+
return NoChangesAnalysis(
|
|
1436
|
+
reason=reason,
|
|
1437
|
+
needs_code_changes=(category == "NEEDS_INVESTIGATION"),
|
|
1438
|
+
requires_manual_work=(category == "MANUAL_WORK_REQUIRED"),
|
|
1439
|
+
manual_work_description=manual_work_desc
|
|
1440
|
+
if category == "MANUAL_WORK_REQUIRED"
|
|
1441
|
+
else None,
|
|
1442
|
+
)
|
|
1443
|
+
|
|
1444
|
+
except Exception as e:
|
|
1445
|
+
logger.error(f"Failed to analyze no-changes reason: {e}")
|
|
1446
|
+
# Fallback: treat as needs investigation
|
|
1447
|
+
return NoChangesAnalysis(
|
|
1448
|
+
reason=f"Analysis failed: {str(e)}",
|
|
1449
|
+
needs_code_changes=True,
|
|
1450
|
+
requires_manual_work=False,
|
|
1451
|
+
manual_work_description=None,
|
|
1452
|
+
)
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
def create_manual_work_followup_sync(
|
|
1456
|
+
parent_ticket_id: str,
|
|
1457
|
+
parent_ticket_title: str,
|
|
1458
|
+
manual_work_description: str,
|
|
1459
|
+
goal_id: str,
|
|
1460
|
+
board_id: str | None = None,
|
|
1461
|
+
) -> str | None:
|
|
1462
|
+
"""
|
|
1463
|
+
Create a follow-up ticket for manual work that the agent cannot perform.
|
|
1464
|
+
|
|
1465
|
+
The ticket is created in PROPOSED state with a [Manual Work] prefix.
|
|
1466
|
+
|
|
1467
|
+
Args:
|
|
1468
|
+
parent_ticket_id: ID of the blocked ticket
|
|
1469
|
+
parent_ticket_title: Title of the blocked ticket
|
|
1470
|
+
manual_work_description: Description of the manual work needed
|
|
1471
|
+
goal_id: Goal ID to link the follow-up ticket to
|
|
1472
|
+
board_id: Optional board ID for permission scoping
|
|
1473
|
+
|
|
1474
|
+
Returns:
|
|
1475
|
+
The ID of the created follow-up ticket, or None if creation failed
|
|
1476
|
+
"""
|
|
1477
|
+
import logging
|
|
1478
|
+
|
|
1479
|
+
logger = logging.getLogger(__name__)
|
|
1480
|
+
|
|
1481
|
+
try:
|
|
1482
|
+
with get_sync_db() as db:
|
|
1483
|
+
# Get the parent ticket for priority inheritance
|
|
1484
|
+
parent_ticket = (
|
|
1485
|
+
db.query(Ticket).filter(Ticket.id == parent_ticket_id).first()
|
|
1486
|
+
)
|
|
1487
|
+
priority = parent_ticket.priority if parent_ticket else None
|
|
1488
|
+
|
|
1489
|
+
# Create follow-up ticket with [Manual Work] prefix
|
|
1490
|
+
followup_title = f"[Manual Work] {parent_ticket_title}"
|
|
1491
|
+
# Truncate if too long (max 255 chars)
|
|
1492
|
+
if len(followup_title) > 255:
|
|
1493
|
+
followup_title = followup_title[:252] + "..."
|
|
1494
|
+
|
|
1495
|
+
followup_description = f"""This ticket requires manual human intervention that the automated agent cannot perform.
|
|
1496
|
+
|
|
1497
|
+
**Original Ticket:** {parent_ticket_title}
|
|
1498
|
+
|
|
1499
|
+
**Manual Work Required:**
|
|
1500
|
+
{manual_work_description}
|
|
1501
|
+
|
|
1502
|
+
**Instructions:**
|
|
1503
|
+
1. Review the manual work description above
|
|
1504
|
+
2. Perform the required actions manually
|
|
1505
|
+
3. Mark this ticket as done when complete
|
|
1506
|
+
"""
|
|
1507
|
+
|
|
1508
|
+
followup_ticket = Ticket(
|
|
1509
|
+
goal_id=goal_id,
|
|
1510
|
+
board_id=board_id,
|
|
1511
|
+
title=followup_title,
|
|
1512
|
+
description=followup_description,
|
|
1513
|
+
state=TicketState.PROPOSED.value,
|
|
1514
|
+
priority=priority,
|
|
1515
|
+
)
|
|
1516
|
+
db.add(followup_ticket)
|
|
1517
|
+
db.flush()
|
|
1518
|
+
followup_id = followup_ticket.id
|
|
1519
|
+
|
|
1520
|
+
# Create creation event for follow-up ticket
|
|
1521
|
+
creation_event = TicketEvent(
|
|
1522
|
+
ticket_id=followup_id,
|
|
1523
|
+
event_type=EventType.CREATED.value,
|
|
1524
|
+
from_state=None,
|
|
1525
|
+
to_state=TicketState.PROPOSED.value,
|
|
1526
|
+
actor_type=ActorType.EXECUTOR.value,
|
|
1527
|
+
actor_id="execute_worker",
|
|
1528
|
+
reason=f"Manual work follow-up for blocked ticket: {parent_ticket_title}",
|
|
1529
|
+
payload_json=json.dumps(
|
|
1530
|
+
{
|
|
1531
|
+
"parent_ticket_id": parent_ticket_id,
|
|
1532
|
+
"manual_work": True,
|
|
1533
|
+
"auto_generated": True,
|
|
1534
|
+
}
|
|
1535
|
+
),
|
|
1536
|
+
)
|
|
1537
|
+
db.add(creation_event)
|
|
1538
|
+
|
|
1539
|
+
# Create link event on the parent ticket
|
|
1540
|
+
link_event = TicketEvent(
|
|
1541
|
+
ticket_id=parent_ticket_id,
|
|
1542
|
+
event_type=EventType.COMMENT.value,
|
|
1543
|
+
from_state=TicketState.BLOCKED.value,
|
|
1544
|
+
to_state=TicketState.BLOCKED.value,
|
|
1545
|
+
actor_type=ActorType.EXECUTOR.value,
|
|
1546
|
+
actor_id="execute_worker",
|
|
1547
|
+
reason=f"Created manual work follow-up ticket: {followup_title}",
|
|
1548
|
+
payload_json=json.dumps(
|
|
1549
|
+
{
|
|
1550
|
+
"followup_ticket_id": followup_id,
|
|
1551
|
+
"manual_work_followup": True,
|
|
1552
|
+
}
|
|
1553
|
+
),
|
|
1554
|
+
)
|
|
1555
|
+
db.add(link_event)
|
|
1556
|
+
|
|
1557
|
+
db.commit()
|
|
1558
|
+
|
|
1559
|
+
logger.info(
|
|
1560
|
+
f"Created manual work follow-up ticket {followup_id} for blocked ticket {parent_ticket_id}"
|
|
1561
|
+
)
|
|
1562
|
+
return followup_id
|
|
1563
|
+
|
|
1564
|
+
except Exception as e:
|
|
1565
|
+
logger.error(f"Failed to create manual work follow-up ticket: {e}")
|
|
1566
|
+
return None
|
|
1567
|
+
|
|
1568
|
+
|
|
1569
|
+
def _get_related_tickets_context_sync(ticket_id: str) -> dict | None:
|
|
1570
|
+
"""
|
|
1571
|
+
Get context about related tickets for better prompt building.
|
|
1572
|
+
|
|
1573
|
+
Returns dict with:
|
|
1574
|
+
- dependencies: list of tickets this ticket depends on
|
|
1575
|
+
- completed_tickets: list of DONE tickets in the same goal
|
|
1576
|
+
- goal_title: title of the goal this ticket belongs to
|
|
1577
|
+
"""
|
|
1578
|
+
from sqlalchemy.orm import Session, selectinload
|
|
1579
|
+
|
|
1580
|
+
from app.database_sync import sync_engine
|
|
1581
|
+
from app.models.ticket import Ticket
|
|
1582
|
+
from app.state_machine import TicketState
|
|
1583
|
+
|
|
1584
|
+
# Use the shared sync_engine instead of creating a new one each time
|
|
1585
|
+
# This prevents connection pool exhaustion
|
|
1586
|
+
with Session(sync_engine) as db:
|
|
1587
|
+
# Get the current ticket with its goal and dependencies
|
|
1588
|
+
ticket = (
|
|
1589
|
+
db.query(Ticket)
|
|
1590
|
+
.options(selectinload(Ticket.blocked_by), selectinload(Ticket.goal))
|
|
1591
|
+
.filter(Ticket.id == ticket_id)
|
|
1592
|
+
.first()
|
|
1593
|
+
)
|
|
1594
|
+
|
|
1595
|
+
if not ticket or not ticket.goal_id:
|
|
1596
|
+
return None
|
|
1597
|
+
|
|
1598
|
+
context = {
|
|
1599
|
+
"goal_title": ticket.goal.title if ticket.goal else None,
|
|
1600
|
+
"dependencies": [],
|
|
1601
|
+
"completed_tickets": [],
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
# Add dependency information
|
|
1605
|
+
if ticket.blocked_by:
|
|
1606
|
+
context["dependencies"].append(
|
|
1607
|
+
{"title": ticket.blocked_by.title, "state": ticket.blocked_by.state}
|
|
1608
|
+
)
|
|
1609
|
+
|
|
1610
|
+
# Get completed tickets in the same goal (for context)
|
|
1611
|
+
completed_tickets = (
|
|
1612
|
+
db.query(Ticket)
|
|
1613
|
+
.filter(
|
|
1614
|
+
Ticket.goal_id == ticket.goal_id,
|
|
1615
|
+
Ticket.state == TicketState.DONE.value,
|
|
1616
|
+
Ticket.id != ticket_id,
|
|
1617
|
+
)
|
|
1618
|
+
.order_by(Ticket.created_at.asc())
|
|
1619
|
+
.limit(5)
|
|
1620
|
+
.all()
|
|
1621
|
+
)
|
|
1622
|
+
|
|
1623
|
+
for comp_ticket in completed_tickets:
|
|
1624
|
+
context["completed_tickets"].append(
|
|
1625
|
+
{"title": comp_ticket.title, "description": comp_ticket.description}
|
|
1626
|
+
)
|
|
1627
|
+
|
|
1628
|
+
return (
|
|
1629
|
+
context
|
|
1630
|
+
if (context["dependencies"] or context["completed_tickets"])
|
|
1631
|
+
else None
|
|
1632
|
+
)
|
|
1633
|
+
|
|
1634
|
+
|
|
1635
|
+
def execute_ticket_task(job_id: str) -> dict:
|
|
1636
|
+
"""
|
|
1637
|
+
Execute task for a ticket using Claude Code CLI (headless) or Cursor CLI (interactive).
|
|
1638
|
+
|
|
1639
|
+
Execution Modes:
|
|
1640
|
+
- Claude CLI (headless): Runs automatically, transitions based on result.
|
|
1641
|
+
- Cursor CLI (interactive): Prepares workspace + prompt, then hands off to user.
|
|
1642
|
+
|
|
1643
|
+
State Transitions:
|
|
1644
|
+
- Headless success with diff → verifying
|
|
1645
|
+
- Headless success with NO diff → blocked (reason: no changes produced)
|
|
1646
|
+
- Headless failure → blocked
|
|
1647
|
+
- Interactive (Cursor) → needs_human immediately
|
|
1648
|
+
|
|
1649
|
+
YOLO Mode:
|
|
1650
|
+
If yolo_mode is enabled in config AND the repo is in the allowlist,
|
|
1651
|
+
Claude CLI runs with --dangerously-skip-permissions. Otherwise it runs
|
|
1652
|
+
in permissioned mode (may require user approval for certain operations).
|
|
1653
|
+
"""
|
|
1654
|
+
# Enable real-time log streaming for this job
|
|
1655
|
+
set_current_job(job_id)
|
|
1656
|
+
|
|
1657
|
+
try:
|
|
1658
|
+
return _execute_ticket_task_impl(job_id)
|
|
1659
|
+
except Exception as e:
|
|
1660
|
+
# Catch-all: if _execute_ticket_task_impl crashes with an unhandled
|
|
1661
|
+
# exception, properly fail the job and block the ticket instead of
|
|
1662
|
+
# leaving them in a zombie RUNNING/EXECUTING state.
|
|
1663
|
+
import logging
|
|
1664
|
+
|
|
1665
|
+
logger = logging.getLogger(__name__)
|
|
1666
|
+
logger.error(
|
|
1667
|
+
f"execute_ticket_task crashed for job {job_id}: {e}",
|
|
1668
|
+
exc_info=True,
|
|
1669
|
+
)
|
|
1670
|
+
try:
|
|
1671
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
1672
|
+
except Exception:
|
|
1673
|
+
pass
|
|
1674
|
+
try:
|
|
1675
|
+
# Try to find the ticket_id from the job to transition it
|
|
1676
|
+
result = get_job_with_ticket(job_id)
|
|
1677
|
+
if result:
|
|
1678
|
+
_, ticket = result
|
|
1679
|
+
transition_ticket_sync(
|
|
1680
|
+
ticket.id,
|
|
1681
|
+
TicketState.BLOCKED,
|
|
1682
|
+
reason=f"Execution crashed: {e}",
|
|
1683
|
+
actor_id="execute_worker",
|
|
1684
|
+
)
|
|
1685
|
+
except Exception:
|
|
1686
|
+
pass
|
|
1687
|
+
return {
|
|
1688
|
+
"job_id": job_id,
|
|
1689
|
+
"status": "failed",
|
|
1690
|
+
"error": f"Unexpected error: {e}",
|
|
1691
|
+
}
|
|
1692
|
+
finally:
|
|
1693
|
+
# Signal streaming finished and clean up
|
|
1694
|
+
stream_finished(job_id)
|
|
1695
|
+
set_current_job(None)
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
def _execute_ticket_task_impl(job_id: str) -> dict:
|
|
1699
|
+
"""Implementation of execute_ticket_task (separated for streaming wrapper)."""
|
|
1700
|
+
# Get job and ticket info
|
|
1701
|
+
result = get_job_with_ticket(job_id)
|
|
1702
|
+
if not result:
|
|
1703
|
+
return {
|
|
1704
|
+
"job_id": job_id,
|
|
1705
|
+
"status": "failed",
|
|
1706
|
+
"error": "Job or ticket not found",
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
job, ticket = result
|
|
1710
|
+
goal_id = ticket.goal_id
|
|
1711
|
+
ticket_id = ticket.id
|
|
1712
|
+
|
|
1713
|
+
# Check if ticket is already in a terminal/blocked state - skip execution if so
|
|
1714
|
+
# This prevents re-execution of jobs for already-blocked tickets
|
|
1715
|
+
if ticket.state in [
|
|
1716
|
+
TicketState.BLOCKED.value,
|
|
1717
|
+
TicketState.DONE.value,
|
|
1718
|
+
TicketState.ABANDONED.value,
|
|
1719
|
+
]:
|
|
1720
|
+
import logging
|
|
1721
|
+
|
|
1722
|
+
logging.getLogger(__name__).info(
|
|
1723
|
+
f"Skipping execution for job {job_id}: ticket {ticket_id} is already in {ticket.state} state"
|
|
1724
|
+
)
|
|
1725
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=0)
|
|
1726
|
+
return {
|
|
1727
|
+
"job_id": job_id,
|
|
1728
|
+
"status": "skipped",
|
|
1729
|
+
"reason": f"Ticket already in {ticket.state} state",
|
|
1730
|
+
"ticket_id": ticket_id,
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
# Ensure workspace exists
|
|
1734
|
+
worktree_path, workspace_error = ensure_workspace_for_ticket(ticket_id, goal_id)
|
|
1735
|
+
|
|
1736
|
+
# Get log path (use worktree if available, fallback otherwise)
|
|
1737
|
+
log_path, log_path_relative = get_log_path_for_job(job_id, worktree_path)
|
|
1738
|
+
|
|
1739
|
+
write_log(log_path, "Starting execute task...")
|
|
1740
|
+
|
|
1741
|
+
# Workspace is required for execution
|
|
1742
|
+
if workspace_error or not worktree_path:
|
|
1743
|
+
write_log(
|
|
1744
|
+
log_path,
|
|
1745
|
+
f"ERROR: Could not create workspace: {workspace_error or 'Unknown error'}",
|
|
1746
|
+
)
|
|
1747
|
+
write_log(log_path, "Execution requires a valid git worktree. Failing job.")
|
|
1748
|
+
update_job_started(job_id, log_path_relative)
|
|
1749
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
1750
|
+
transition_ticket_sync(
|
|
1751
|
+
ticket_id,
|
|
1752
|
+
TicketState.BLOCKED,
|
|
1753
|
+
reason=f"Execution failed: workspace creation error - {workspace_error}",
|
|
1754
|
+
actor_id="execute_worker",
|
|
1755
|
+
)
|
|
1756
|
+
return {"job_id": job_id, "status": "failed", "error": workspace_error}
|
|
1757
|
+
|
|
1758
|
+
write_log(log_path, f"Workspace ready at: {worktree_path}")
|
|
1759
|
+
|
|
1760
|
+
# Mark as running
|
|
1761
|
+
if not update_job_started(job_id, log_path_relative):
|
|
1762
|
+
write_log(log_path, "Job was canceled or not found, aborting.")
|
|
1763
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
1764
|
+
|
|
1765
|
+
# Check for cancellation BEFORE transitioning to EXECUTING state
|
|
1766
|
+
# This prevents a race where the ticket transitions to EXECUTING but the job
|
|
1767
|
+
# is immediately cancelled, leaving the ticket stuck in EXECUTING with no runner
|
|
1768
|
+
if check_canceled(job_id):
|
|
1769
|
+
write_log(log_path, "Job canceled before execution started, aborting.")
|
|
1770
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
1771
|
+
|
|
1772
|
+
# Transition ticket to EXECUTING state BEFORE any execution work begins
|
|
1773
|
+
# This is critical - the ticket MUST be in EXECUTING state while running
|
|
1774
|
+
# This handles transitions from PLANNED, DONE (changes requested), or NEEDS_HUMAN
|
|
1775
|
+
from app.state_machine import validate_transition
|
|
1776
|
+
|
|
1777
|
+
with get_sync_db() as db:
|
|
1778
|
+
current_ticket = db.query(Ticket).filter(Ticket.id == ticket_id).first()
|
|
1779
|
+
if not current_ticket:
|
|
1780
|
+
write_log(log_path, "ERROR: Ticket not found in database")
|
|
1781
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
1782
|
+
return {"job_id": job_id, "status": "failed", "error": "Ticket not found"}
|
|
1783
|
+
|
|
1784
|
+
current_state = TicketState(current_ticket.state)
|
|
1785
|
+
write_log(log_path, f"Current ticket state: '{current_state.value}'")
|
|
1786
|
+
|
|
1787
|
+
if current_state == TicketState.EXECUTING:
|
|
1788
|
+
write_log(log_path, "Ticket already in 'executing' state")
|
|
1789
|
+
elif validate_transition(current_state, TicketState.EXECUTING):
|
|
1790
|
+
write_log(
|
|
1791
|
+
log_path,
|
|
1792
|
+
f"Transitioning ticket from '{current_state.value}' to 'executing'",
|
|
1793
|
+
)
|
|
1794
|
+
# Important: Do transition INSIDE the same db context to ensure atomicity
|
|
1795
|
+
current_ticket.state = TicketState.EXECUTING.value
|
|
1796
|
+
|
|
1797
|
+
# Create transition event
|
|
1798
|
+
event = TicketEvent(
|
|
1799
|
+
ticket_id=ticket_id,
|
|
1800
|
+
event_type=EventType.TRANSITIONED.value,
|
|
1801
|
+
from_state=current_state.value,
|
|
1802
|
+
to_state=TicketState.EXECUTING.value,
|
|
1803
|
+
actor_type=ActorType.EXECUTOR.value,
|
|
1804
|
+
actor_id="execute_worker",
|
|
1805
|
+
reason=f"Execution started (job {job_id})",
|
|
1806
|
+
payload_json=json.dumps({"job_id": job_id}),
|
|
1807
|
+
)
|
|
1808
|
+
db.add(event)
|
|
1809
|
+
db.commit()
|
|
1810
|
+
write_log(log_path, "Successfully transitioned to 'executing' state")
|
|
1811
|
+
else:
|
|
1812
|
+
# Invalid state transition — abort rather than bypass state machine
|
|
1813
|
+
write_log(
|
|
1814
|
+
log_path,
|
|
1815
|
+
f"ABORTING: Cannot transition from '{current_state.value}' to 'executing' (invalid transition)",
|
|
1816
|
+
)
|
|
1817
|
+
with get_sync_db() as db:
|
|
1818
|
+
job = db.query(Job).filter(Job.id == job_id).first()
|
|
1819
|
+
if job:
|
|
1820
|
+
job.status = JobStatus.FAILED.value
|
|
1821
|
+
job.finished_at = datetime.now()
|
|
1822
|
+
db.commit()
|
|
1823
|
+
return
|
|
1824
|
+
|
|
1825
|
+
# Load configuration from DB (board config is the single source of truth)
|
|
1826
|
+
board_config = None
|
|
1827
|
+
board_repo_root = None
|
|
1828
|
+
if ticket.board_id:
|
|
1829
|
+
with get_sync_db() as db:
|
|
1830
|
+
from app.models.board import Board
|
|
1831
|
+
|
|
1832
|
+
board = db.query(Board).filter(Board.id == ticket.board_id).first()
|
|
1833
|
+
if board:
|
|
1834
|
+
if board.config:
|
|
1835
|
+
board_config = board.config
|
|
1836
|
+
if board.repo_root:
|
|
1837
|
+
board_repo_root = board.repo_root
|
|
1838
|
+
|
|
1839
|
+
config = DraftConfig.from_board_config(board_config)
|
|
1840
|
+
execute_config = config.execute_config
|
|
1841
|
+
planner_config = config.planner_config
|
|
1842
|
+
|
|
1843
|
+
# Use board's repo_root as the authoritative main repo path.
|
|
1844
|
+
# Falls back to GIT_REPO_PATH env var / default only when no board repo_root.
|
|
1845
|
+
if board_repo_root:
|
|
1846
|
+
main_repo_path = Path(board_repo_root)
|
|
1847
|
+
else:
|
|
1848
|
+
main_repo_path = WorkspaceService.get_repo_path()
|
|
1849
|
+
|
|
1850
|
+
# =========================================================================
|
|
1851
|
+
# WORKTREE SAFETY VALIDATION (enforced, not assumed)
|
|
1852
|
+
# =========================================================================
|
|
1853
|
+
write_log(log_path, "Validating worktree safety...")
|
|
1854
|
+
worktree_validator = WorktreeValidator(main_repo_path)
|
|
1855
|
+
validation_result = worktree_validator.validate(worktree_path)
|
|
1856
|
+
|
|
1857
|
+
if not validation_result.valid:
|
|
1858
|
+
write_log(log_path, f"SAFETY CHECK FAILED: {validation_result.error}")
|
|
1859
|
+
write_log(log_path, f"Reason: {validation_result.message}")
|
|
1860
|
+
write_log(log_path, "Refusing to execute in unsafe location.")
|
|
1861
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
1862
|
+
transition_ticket_sync(
|
|
1863
|
+
ticket_id,
|
|
1864
|
+
TicketState.BLOCKED,
|
|
1865
|
+
reason=f"Safety check failed: {validation_result.message}",
|
|
1866
|
+
payload={
|
|
1867
|
+
"validation_error": validation_result.error,
|
|
1868
|
+
"worktree_path": str(worktree_path),
|
|
1869
|
+
"main_repo_path": str(main_repo_path),
|
|
1870
|
+
},
|
|
1871
|
+
actor_id="execute_worker",
|
|
1872
|
+
)
|
|
1873
|
+
return {
|
|
1874
|
+
"job_id": job_id,
|
|
1875
|
+
"status": "failed",
|
|
1876
|
+
"error": f"Safety check failed: {validation_result.message}",
|
|
1877
|
+
"validation_error": validation_result.error,
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
write_log(log_path, f"Worktree validated: branch={validation_result.branch}")
|
|
1881
|
+
|
|
1882
|
+
# =========================================================================
|
|
1883
|
+
# YOLO MODE CHECK (refuse if enabled but allowlist empty)
|
|
1884
|
+
# =========================================================================
|
|
1885
|
+
yolo_status = execute_config.check_yolo_status(
|
|
1886
|
+
str(worktree_path.resolve()),
|
|
1887
|
+
repo_root=str(main_repo_path),
|
|
1888
|
+
)
|
|
1889
|
+
model_info = (
|
|
1890
|
+
f", model={execute_config.executor_model}"
|
|
1891
|
+
if execute_config.executor_model
|
|
1892
|
+
else ""
|
|
1893
|
+
)
|
|
1894
|
+
write_log(
|
|
1895
|
+
log_path,
|
|
1896
|
+
f"Execute config: timeout={execute_config.timeout}s, preferred_executor={execute_config.preferred_executor}{model_info}",
|
|
1897
|
+
)
|
|
1898
|
+
|
|
1899
|
+
if yolo_status == YoloStatus.REFUSED:
|
|
1900
|
+
refusal_reason = execute_config.get_yolo_refusal_reason(
|
|
1901
|
+
repo_root=str(main_repo_path)
|
|
1902
|
+
)
|
|
1903
|
+
write_log(log_path, f"YOLO MODE REFUSED: {refusal_reason}")
|
|
1904
|
+
write_log(log_path, "Transitioning to needs_human for manual approval.")
|
|
1905
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
1906
|
+
transition_ticket_sync(
|
|
1907
|
+
ticket_id,
|
|
1908
|
+
TicketState.NEEDS_HUMAN,
|
|
1909
|
+
reason=f"YOLO mode refused: {refusal_reason}",
|
|
1910
|
+
payload={
|
|
1911
|
+
"yolo_refused": True,
|
|
1912
|
+
"refusal_reason": refusal_reason,
|
|
1913
|
+
"worktree": str(worktree_path),
|
|
1914
|
+
},
|
|
1915
|
+
actor_id="execute_worker",
|
|
1916
|
+
)
|
|
1917
|
+
return {
|
|
1918
|
+
"job_id": job_id,
|
|
1919
|
+
"status": "yolo_refused",
|
|
1920
|
+
"worktree": str(worktree_path),
|
|
1921
|
+
"reason": refusal_reason,
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
yolo_enabled = yolo_status == YoloStatus.ALLOWED
|
|
1925
|
+
write_log(log_path, f"YOLO mode: {yolo_status.value}")
|
|
1926
|
+
|
|
1927
|
+
# Detect available executor CLI
|
|
1928
|
+
try:
|
|
1929
|
+
executor_info = ExecutorService.detect_executor(
|
|
1930
|
+
preferred=execute_config.preferred_executor,
|
|
1931
|
+
agent_path=planner_config.agent_path,
|
|
1932
|
+
)
|
|
1933
|
+
write_log(
|
|
1934
|
+
log_path,
|
|
1935
|
+
f"Found executor: {executor_info.executor_type.value} ({executor_info.mode.value}) at {executor_info.path}",
|
|
1936
|
+
)
|
|
1937
|
+
except ExecutorNotFoundError as e:
|
|
1938
|
+
write_log(log_path, f"ERROR: {e.message}")
|
|
1939
|
+
write_log(
|
|
1940
|
+
log_path,
|
|
1941
|
+
"No code executor CLI found. Please install Claude Code CLI or Cursor CLI.",
|
|
1942
|
+
)
|
|
1943
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
1944
|
+
transition_ticket_sync(
|
|
1945
|
+
ticket_id,
|
|
1946
|
+
TicketState.BLOCKED,
|
|
1947
|
+
reason=f"Execution failed: {e.message}",
|
|
1948
|
+
actor_id="execute_worker",
|
|
1949
|
+
)
|
|
1950
|
+
return {"job_id": job_id, "status": "failed", "error": e.message}
|
|
1951
|
+
|
|
1952
|
+
# Check for feedback from previous revision (if this is a re-run after changes requested)
|
|
1953
|
+
feedback_bundle = get_feedback_bundle_for_ticket(ticket_id)
|
|
1954
|
+
if feedback_bundle:
|
|
1955
|
+
write_log(
|
|
1956
|
+
log_path,
|
|
1957
|
+
f"Found feedback from revision #{feedback_bundle.get('revision_number', '?')}",
|
|
1958
|
+
)
|
|
1959
|
+
write_log(
|
|
1960
|
+
log_path, f" - Summary: {feedback_bundle.get('summary', '')[:100]}..."
|
|
1961
|
+
)
|
|
1962
|
+
write_log(
|
|
1963
|
+
log_path,
|
|
1964
|
+
f" - Comments to address: {len(feedback_bundle.get('comments', []))}",
|
|
1965
|
+
)
|
|
1966
|
+
else:
|
|
1967
|
+
write_log(log_path, "No previous revision feedback found (fresh execution)")
|
|
1968
|
+
|
|
1969
|
+
# Check for queued follow-up prompt (from instant follow-up queue)
|
|
1970
|
+
from app.services.queued_message_service import queued_message_service
|
|
1971
|
+
|
|
1972
|
+
followup_prompt = queued_message_service.get_followup_prompt(ticket_id)
|
|
1973
|
+
additional_context = None
|
|
1974
|
+
if followup_prompt:
|
|
1975
|
+
write_log(log_path, f"Found queued follow-up: {followup_prompt[:100]}...")
|
|
1976
|
+
additional_context = f"\n\n--- FOLLOW-UP REQUEST ---\n{followup_prompt}\n\nPlease address the above follow-up request while continuing work on this ticket."
|
|
1977
|
+
|
|
1978
|
+
# Get related tickets context for better prompt
|
|
1979
|
+
related_tickets_context = _get_related_tickets_context_sync(ticket_id)
|
|
1980
|
+
if related_tickets_context:
|
|
1981
|
+
write_log(
|
|
1982
|
+
log_path,
|
|
1983
|
+
f"Found context: {len(related_tickets_context.get('completed_tickets', []))} completed tickets, {len(related_tickets_context.get('dependencies', []))} dependencies",
|
|
1984
|
+
)
|
|
1985
|
+
|
|
1986
|
+
# Build prompt bundle
|
|
1987
|
+
write_log(log_path, "Building prompt bundle...")
|
|
1988
|
+
prompt_builder = PromptBundleBuilder(
|
|
1989
|
+
worktree_path, job_id, repo_root=main_repo_path
|
|
1990
|
+
)
|
|
1991
|
+
verify_commands = (
|
|
1992
|
+
config.verify_config.commands if config.verify_config.commands else None
|
|
1993
|
+
)
|
|
1994
|
+
prompt_file = prompt_builder.build_prompt(
|
|
1995
|
+
ticket_title=ticket.title,
|
|
1996
|
+
ticket_description=ticket.description,
|
|
1997
|
+
feedback_bundle=feedback_bundle,
|
|
1998
|
+
additional_context=additional_context,
|
|
1999
|
+
related_tickets_context=related_tickets_context,
|
|
2000
|
+
verify_commands=verify_commands,
|
|
2001
|
+
)
|
|
2002
|
+
write_log(log_path, f"Prompt bundle created at: {prompt_file}")
|
|
2003
|
+
|
|
2004
|
+
# Get evidence directory
|
|
2005
|
+
evidence_dir = prompt_builder.get_evidence_dir()
|
|
2006
|
+
evidence_records: list[str] = []
|
|
2007
|
+
|
|
2008
|
+
# Check for cancellation before execution
|
|
2009
|
+
if check_canceled(job_id):
|
|
2010
|
+
write_log(log_path, "Job canceled, stopping execution.")
|
|
2011
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
2012
|
+
|
|
2013
|
+
# =========================================================================
|
|
2014
|
+
# INTERACTIVE EXECUTOR (Cursor) - Hand off to user immediately
|
|
2015
|
+
# =========================================================================
|
|
2016
|
+
if executor_info.is_interactive():
|
|
2017
|
+
write_log(
|
|
2018
|
+
log_path, f"Executor {executor_info.executor_type.value} is INTERACTIVE."
|
|
2019
|
+
)
|
|
2020
|
+
write_log(log_path, "Workspace and prompt bundle are ready.")
|
|
2021
|
+
write_log(log_path, "Transitioning to 'needs_human' for manual completion.")
|
|
2022
|
+
write_log(log_path, "")
|
|
2023
|
+
write_log(log_path, "=== INSTRUCTIONS FOR HUMAN ===")
|
|
2024
|
+
write_log(log_path, f"1. Open the worktree in your editor: {worktree_path}")
|
|
2025
|
+
write_log(log_path, f"2. Read the prompt: {prompt_file}")
|
|
2026
|
+
write_log(log_path, "3. Implement the requested changes")
|
|
2027
|
+
write_log(log_path, "4. Commit your changes")
|
|
2028
|
+
write_log(log_path, "5. Mark the ticket as ready for verification")
|
|
2029
|
+
write_log(log_path, "==============================")
|
|
2030
|
+
|
|
2031
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
2032
|
+
transition_ticket_sync(
|
|
2033
|
+
ticket_id,
|
|
2034
|
+
TicketState.NEEDS_HUMAN,
|
|
2035
|
+
reason=f"Interactive executor ({executor_info.executor_type.value}): workspace ready, awaiting human completion",
|
|
2036
|
+
payload={
|
|
2037
|
+
"executor": executor_info.executor_type.value,
|
|
2038
|
+
"mode": executor_info.mode.value,
|
|
2039
|
+
"worktree": str(worktree_path),
|
|
2040
|
+
"prompt_file": str(prompt_file),
|
|
2041
|
+
},
|
|
2042
|
+
actor_id="execute_worker",
|
|
2043
|
+
)
|
|
2044
|
+
return {
|
|
2045
|
+
"job_id": job_id,
|
|
2046
|
+
"status": "needs_human",
|
|
2047
|
+
"worktree": str(worktree_path),
|
|
2048
|
+
"executor": executor_info.executor_type.value,
|
|
2049
|
+
"mode": executor_info.mode.value,
|
|
2050
|
+
"prompt_file": str(prompt_file),
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
# =========================================================================
|
|
2054
|
+
# HEADLESS EXECUTOR (Claude) - Run automatically
|
|
2055
|
+
# =========================================================================
|
|
2056
|
+
write_log(
|
|
2057
|
+
log_path, f"Running headless executor: {executor_info.executor_type.value}..."
|
|
2058
|
+
)
|
|
2059
|
+
executor_evidence_id = str(uuid.uuid4())
|
|
2060
|
+
|
|
2061
|
+
# Check for existing session to continue (session continuity)
|
|
2062
|
+
from app.services.agent_session_service import get_session_service
|
|
2063
|
+
|
|
2064
|
+
session_service = get_session_service(worktree_path)
|
|
2065
|
+
existing_session = session_service.get_session(ticket_id)
|
|
2066
|
+
session_flag = None
|
|
2067
|
+
if existing_session:
|
|
2068
|
+
session_flag = session_service.get_continue_flag(
|
|
2069
|
+
ticket_id, executor_info.executor_type.value
|
|
2070
|
+
)
|
|
2071
|
+
if session_flag:
|
|
2072
|
+
write_log(
|
|
2073
|
+
log_path,
|
|
2074
|
+
f"Continuing from session: {existing_session.session_id} (execution #{existing_session.execution_count + 1})",
|
|
2075
|
+
)
|
|
2076
|
+
|
|
2077
|
+
# Get the command with YOLO mode and model selection
|
|
2078
|
+
# Returns (command, stdin_content) tuple - prompt piped via stdin to avoid ARG_MAX
|
|
2079
|
+
executor_command, executor_stdin = executor_info.get_apply_command(
|
|
2080
|
+
prompt_file,
|
|
2081
|
+
worktree_path,
|
|
2082
|
+
yolo_mode=yolo_enabled,
|
|
2083
|
+
model=execute_config.executor_model,
|
|
2084
|
+
)
|
|
2085
|
+
|
|
2086
|
+
# Add session continuation flag if available
|
|
2087
|
+
if session_flag:
|
|
2088
|
+
executor_command = executor_command + session_flag.split()
|
|
2089
|
+
|
|
2090
|
+
# Log command (without full prompt content)
|
|
2091
|
+
if yolo_enabled:
|
|
2092
|
+
write_log(
|
|
2093
|
+
log_path,
|
|
2094
|
+
f"Command: {executor_command[0]} --print --dangerously-skip-permissions <prompt>",
|
|
2095
|
+
)
|
|
2096
|
+
else:
|
|
2097
|
+
write_log(log_path, f"Command: {executor_command[0]} --print <prompt>")
|
|
2098
|
+
write_log(
|
|
2099
|
+
log_path,
|
|
2100
|
+
"NOTE: Running in permissioned mode. Some operations may require approval.",
|
|
2101
|
+
)
|
|
2102
|
+
|
|
2103
|
+
# Track execution timing for metadata
|
|
2104
|
+
executor_start_time = time.time()
|
|
2105
|
+
|
|
2106
|
+
# Enable log normalization for cursor-agent (outputs JSON streaming format)
|
|
2107
|
+
should_normalize = executor_info.executor_type == ExecutorType.CURSOR_AGENT
|
|
2108
|
+
|
|
2109
|
+
# Use retry wrapper for improved reliability (handles transient failures)
|
|
2110
|
+
executor_exit_code, executor_stdout_path, executor_stderr_path = (
|
|
2111
|
+
run_executor_cli_with_retry(
|
|
2112
|
+
command=executor_command,
|
|
2113
|
+
cwd=worktree_path,
|
|
2114
|
+
evidence_dir=evidence_dir,
|
|
2115
|
+
evidence_id=executor_evidence_id,
|
|
2116
|
+
repo_root=main_repo_path,
|
|
2117
|
+
timeout=execute_config.timeout,
|
|
2118
|
+
job_id=job_id, # Enable real-time streaming
|
|
2119
|
+
normalize_logs=should_normalize, # Parse cursor-agent JSON for nice display
|
|
2120
|
+
stdin_content=executor_stdin, # Pipe prompt via stdin (ARG_MAX safety)
|
|
2121
|
+
log_path=log_path, # Enable retry logging
|
|
2122
|
+
)
|
|
2123
|
+
)
|
|
2124
|
+
|
|
2125
|
+
# Calculate execution duration
|
|
2126
|
+
executor_duration_ms = int((time.time() - executor_start_time) * 1000)
|
|
2127
|
+
|
|
2128
|
+
# Create EXECUTOR_META evidence with structured metadata
|
|
2129
|
+
executor_meta_id = str(uuid.uuid4())
|
|
2130
|
+
executor_meta = {
|
|
2131
|
+
"exit_code": executor_exit_code,
|
|
2132
|
+
"duration_ms": executor_duration_ms,
|
|
2133
|
+
"executor_type": executor_info.executor_type.value,
|
|
2134
|
+
"mode": executor_info.mode.value,
|
|
2135
|
+
"command": f"{executor_command[0]} --print {'--dangerously-skip-permissions ' if yolo_enabled else ''}<prompt>",
|
|
2136
|
+
"yolo_enabled": yolo_enabled,
|
|
2137
|
+
"timeout_configured": execute_config.timeout,
|
|
2138
|
+
}
|
|
2139
|
+
executor_meta_path = evidence_dir / f"{executor_meta_id}.meta.json"
|
|
2140
|
+
executor_meta_path.write_text(json.dumps(executor_meta, indent=2))
|
|
2141
|
+
# Store relative path for secure DB storage
|
|
2142
|
+
executor_meta_relpath = str(executor_meta_path)
|
|
2143
|
+
create_evidence_record(
|
|
2144
|
+
ticket_id=ticket_id,
|
|
2145
|
+
job_id=job_id,
|
|
2146
|
+
command="executor_metadata",
|
|
2147
|
+
exit_code=executor_exit_code,
|
|
2148
|
+
stdout_path=executor_meta_relpath,
|
|
2149
|
+
stderr_path="",
|
|
2150
|
+
evidence_id=executor_meta_id,
|
|
2151
|
+
kind=EvidenceKind.EXECUTOR_META,
|
|
2152
|
+
)
|
|
2153
|
+
evidence_records.append(executor_meta_id)
|
|
2154
|
+
|
|
2155
|
+
# Create evidence record for executor output (typed)
|
|
2156
|
+
create_evidence_record(
|
|
2157
|
+
ticket_id=ticket_id,
|
|
2158
|
+
job_id=job_id,
|
|
2159
|
+
command=f"{executor_command[0]} --print {'--dangerously-skip-permissions ' if yolo_enabled else ''}<prompt>",
|
|
2160
|
+
exit_code=executor_exit_code,
|
|
2161
|
+
stdout_path=executor_stdout_path,
|
|
2162
|
+
stderr_path=executor_stderr_path,
|
|
2163
|
+
evidence_id=executor_evidence_id,
|
|
2164
|
+
kind=EvidenceKind.EXECUTOR_STDOUT,
|
|
2165
|
+
)
|
|
2166
|
+
evidence_records.append(executor_evidence_id)
|
|
2167
|
+
|
|
2168
|
+
# Extract and save session ID for continuity
|
|
2169
|
+
try:
|
|
2170
|
+
stdout_content = (main_repo_path / executor_stdout_path).read_text()
|
|
2171
|
+
new_session_id = session_service.extract_session_id_from_output(stdout_content)
|
|
2172
|
+
if new_session_id:
|
|
2173
|
+
session_service.save_session(
|
|
2174
|
+
session_id=new_session_id,
|
|
2175
|
+
ticket_id=ticket_id,
|
|
2176
|
+
agent_type=executor_info.executor_type.value,
|
|
2177
|
+
)
|
|
2178
|
+
write_log(
|
|
2179
|
+
log_path,
|
|
2180
|
+
f"Saved session ID for future continuity: {new_session_id[:16]}...",
|
|
2181
|
+
)
|
|
2182
|
+
except Exception as e:
|
|
2183
|
+
logger.debug(f"Could not extract session ID: {e}")
|
|
2184
|
+
|
|
2185
|
+
write_log(log_path, f"Executor completed in {executor_duration_ms}ms")
|
|
2186
|
+
if executor_exit_code == 0:
|
|
2187
|
+
write_log(log_path, "Executor CLI completed successfully (exit code: 0)")
|
|
2188
|
+
else:
|
|
2189
|
+
write_log(log_path, f"Executor CLI FAILED (exit code: {executor_exit_code})")
|
|
2190
|
+
# Read stderr for more details
|
|
2191
|
+
try:
|
|
2192
|
+
stderr_content = Path(executor_stderr_path).read_text()[:500]
|
|
2193
|
+
if stderr_content:
|
|
2194
|
+
write_log(log_path, f"Executor stderr: {stderr_content}")
|
|
2195
|
+
except Exception:
|
|
2196
|
+
pass
|
|
2197
|
+
|
|
2198
|
+
# Capture git diff regardless of exit code (to see what changes were made)
|
|
2199
|
+
write_log(log_path, "Capturing git diff...")
|
|
2200
|
+
diff_stat_evidence_id = str(uuid.uuid4())
|
|
2201
|
+
diff_patch_evidence_id = str(uuid.uuid4())
|
|
2202
|
+
|
|
2203
|
+
diff_exit_code, diff_stat_path, diff_patch_path, diff_stat, has_changes = (
|
|
2204
|
+
capture_git_diff(
|
|
2205
|
+
cwd=worktree_path,
|
|
2206
|
+
evidence_dir=evidence_dir,
|
|
2207
|
+
evidence_id=diff_stat_evidence_id, # Used for both files with different extensions
|
|
2208
|
+
repo_root=main_repo_path,
|
|
2209
|
+
)
|
|
2210
|
+
)
|
|
2211
|
+
|
|
2212
|
+
# Create typed evidence records for git diff
|
|
2213
|
+
create_evidence_record(
|
|
2214
|
+
ticket_id=ticket_id,
|
|
2215
|
+
job_id=job_id,
|
|
2216
|
+
command="git diff --stat",
|
|
2217
|
+
exit_code=diff_exit_code,
|
|
2218
|
+
stdout_path=diff_stat_path,
|
|
2219
|
+
stderr_path="", # stderr captured in patch record
|
|
2220
|
+
evidence_id=diff_stat_evidence_id,
|
|
2221
|
+
kind=EvidenceKind.GIT_DIFF_STAT,
|
|
2222
|
+
)
|
|
2223
|
+
evidence_records.append(diff_stat_evidence_id)
|
|
2224
|
+
|
|
2225
|
+
create_evidence_record(
|
|
2226
|
+
ticket_id=ticket_id,
|
|
2227
|
+
job_id=job_id,
|
|
2228
|
+
command="git diff",
|
|
2229
|
+
exit_code=diff_exit_code,
|
|
2230
|
+
stdout_path=diff_patch_path,
|
|
2231
|
+
stderr_path="",
|
|
2232
|
+
evidence_id=diff_patch_evidence_id,
|
|
2233
|
+
kind=EvidenceKind.GIT_DIFF_PATCH,
|
|
2234
|
+
)
|
|
2235
|
+
evidence_records.append(diff_patch_evidence_id)
|
|
2236
|
+
|
|
2237
|
+
write_log(log_path, f"Git diff summary:\n{diff_stat}")
|
|
2238
|
+
write_log(log_path, f"Has changes: {has_changes}")
|
|
2239
|
+
|
|
2240
|
+
# Check for cancellation before state transition
|
|
2241
|
+
if check_canceled(job_id):
|
|
2242
|
+
write_log(log_path, "Job canceled, stopping execution.")
|
|
2243
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
2244
|
+
|
|
2245
|
+
# =========================================================================
|
|
2246
|
+
# STATE TRANSITIONS
|
|
2247
|
+
# =========================================================================
|
|
2248
|
+
|
|
2249
|
+
# Case 1: Executor failed
|
|
2250
|
+
if executor_exit_code != 0:
|
|
2251
|
+
write_log(log_path, f"Execution FAILED with exit code {executor_exit_code}")
|
|
2252
|
+
write_log(log_path, "Transitioning ticket to 'blocked'")
|
|
2253
|
+
transition_ticket_sync(
|
|
2254
|
+
ticket_id,
|
|
2255
|
+
TicketState.BLOCKED,
|
|
2256
|
+
reason=f"Execution failed: {executor_info.executor_type.value} CLI exited with code {executor_exit_code}",
|
|
2257
|
+
payload={
|
|
2258
|
+
"executor": executor_info.executor_type.value,
|
|
2259
|
+
"exit_code": executor_exit_code,
|
|
2260
|
+
"evidence_ids": evidence_records,
|
|
2261
|
+
"diff_summary": diff_stat,
|
|
2262
|
+
"yolo_mode": yolo_enabled,
|
|
2263
|
+
},
|
|
2264
|
+
actor_id="execute_worker",
|
|
2265
|
+
)
|
|
2266
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=executor_exit_code)
|
|
2267
|
+
return {
|
|
2268
|
+
"job_id": job_id,
|
|
2269
|
+
"status": "failed",
|
|
2270
|
+
"worktree": str(worktree_path),
|
|
2271
|
+
"executor": executor_info.executor_type.value,
|
|
2272
|
+
"exit_code": executor_exit_code,
|
|
2273
|
+
"evidence_ids": evidence_records,
|
|
2274
|
+
"diff_summary": diff_stat,
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2277
|
+
# Case 2: Executor succeeded but NO CHANGES produced
|
|
2278
|
+
if not has_changes:
|
|
2279
|
+
write_log(log_path, "Execution completed but NO CHANGES were produced.")
|
|
2280
|
+
write_log(log_path, "Analyzing why no code changes were made...")
|
|
2281
|
+
|
|
2282
|
+
# Read executor stdout for analysis
|
|
2283
|
+
try:
|
|
2284
|
+
executor_stdout_content = (
|
|
2285
|
+
main_repo_path / executor_stdout_path
|
|
2286
|
+
).read_text()
|
|
2287
|
+
except Exception as e:
|
|
2288
|
+
write_log(log_path, f"Warning: Could not read executor output: {e}")
|
|
2289
|
+
executor_stdout_content = ""
|
|
2290
|
+
|
|
2291
|
+
# Analyze why no changes were produced using LLM
|
|
2292
|
+
analysis = analyze_no_changes_reason(
|
|
2293
|
+
ticket_title=ticket.title,
|
|
2294
|
+
ticket_description=ticket.description,
|
|
2295
|
+
executor_stdout=executor_stdout_content,
|
|
2296
|
+
planner_config=planner_config,
|
|
2297
|
+
)
|
|
2298
|
+
|
|
2299
|
+
write_log(log_path, f"Analysis result: {analysis.reason}")
|
|
2300
|
+
write_log(log_path, f" - Needs code changes: {analysis.needs_code_changes}")
|
|
2301
|
+
write_log(
|
|
2302
|
+
log_path, f" - Requires manual work: {analysis.requires_manual_work}"
|
|
2303
|
+
)
|
|
2304
|
+
|
|
2305
|
+
followup_ticket_id = None
|
|
2306
|
+
|
|
2307
|
+
# Handle based on analysis result
|
|
2308
|
+
if analysis.requires_manual_work and analysis.manual_work_description:
|
|
2309
|
+
# Create a [Manual Work] follow-up ticket
|
|
2310
|
+
write_log(log_path, "Creating [Manual Work] follow-up ticket...")
|
|
2311
|
+
followup_ticket_id = create_manual_work_followup_sync(
|
|
2312
|
+
parent_ticket_id=ticket_id,
|
|
2313
|
+
parent_ticket_title=ticket.title,
|
|
2314
|
+
manual_work_description=analysis.manual_work_description,
|
|
2315
|
+
goal_id=goal_id,
|
|
2316
|
+
board_id=ticket.board_id,
|
|
2317
|
+
)
|
|
2318
|
+
if followup_ticket_id:
|
|
2319
|
+
write_log(log_path, f"Created follow-up ticket: {followup_ticket_id}")
|
|
2320
|
+
else:
|
|
2321
|
+
write_log(log_path, "Warning: Failed to create follow-up ticket")
|
|
2322
|
+
|
|
2323
|
+
# Block the original ticket with reference to manual work
|
|
2324
|
+
reason = f"Requires manual work: {analysis.reason}"
|
|
2325
|
+
payload = {
|
|
2326
|
+
"executor": executor_info.executor_type.value,
|
|
2327
|
+
"evidence_ids": evidence_records,
|
|
2328
|
+
"diff_summary": diff_stat,
|
|
2329
|
+
"no_changes": True,
|
|
2330
|
+
"yolo_mode": yolo_enabled,
|
|
2331
|
+
"requires_manual_work": True,
|
|
2332
|
+
"manual_work_followup_id": followup_ticket_id,
|
|
2333
|
+
"analysis_reason": analysis.reason,
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
elif not analysis.needs_code_changes:
|
|
2337
|
+
# No changes needed - mark as blocked with skip_followup flag
|
|
2338
|
+
write_log(log_path, "No code changes needed. Blocking without follow-up.")
|
|
2339
|
+
reason = f"No changes required: {analysis.reason}"
|
|
2340
|
+
payload = {
|
|
2341
|
+
"executor": executor_info.executor_type.value,
|
|
2342
|
+
"evidence_ids": evidence_records,
|
|
2343
|
+
"diff_summary": diff_stat,
|
|
2344
|
+
"no_changes": True,
|
|
2345
|
+
"yolo_mode": yolo_enabled,
|
|
2346
|
+
"no_changes_needed": True,
|
|
2347
|
+
"skip_followup": True, # Signal to planner to not create follow-ups
|
|
2348
|
+
"analysis_reason": analysis.reason,
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
else:
|
|
2352
|
+
# Needs investigation - use original behavior (planner may create follow-up)
|
|
2353
|
+
write_log(log_path, "Needs investigation. Blocking for review.")
|
|
2354
|
+
reason = f"Execution completed but no code changes were produced: {analysis.reason}"
|
|
2355
|
+
payload = {
|
|
2356
|
+
"executor": executor_info.executor_type.value,
|
|
2357
|
+
"evidence_ids": evidence_records,
|
|
2358
|
+
"diff_summary": diff_stat,
|
|
2359
|
+
"no_changes": True,
|
|
2360
|
+
"yolo_mode": yolo_enabled,
|
|
2361
|
+
"needs_investigation": True,
|
|
2362
|
+
"analysis_reason": analysis.reason,
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
write_log(
|
|
2366
|
+
log_path, f"Transitioning ticket to 'blocked' (reason: {reason[:100]}...)"
|
|
2367
|
+
)
|
|
2368
|
+
transition_ticket_sync(
|
|
2369
|
+
ticket_id,
|
|
2370
|
+
TicketState.BLOCKED,
|
|
2371
|
+
reason=reason,
|
|
2372
|
+
payload=payload,
|
|
2373
|
+
actor_id="execute_worker",
|
|
2374
|
+
)
|
|
2375
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
2376
|
+
|
|
2377
|
+
result_payload = {
|
|
2378
|
+
"job_id": job_id,
|
|
2379
|
+
"status": "no_changes",
|
|
2380
|
+
"worktree": str(worktree_path),
|
|
2381
|
+
"executor": executor_info.executor_type.value,
|
|
2382
|
+
"evidence_ids": evidence_records,
|
|
2383
|
+
"diff_summary": diff_stat,
|
|
2384
|
+
"analysis": {
|
|
2385
|
+
"reason": analysis.reason,
|
|
2386
|
+
"needs_code_changes": analysis.needs_code_changes,
|
|
2387
|
+
"requires_manual_work": analysis.requires_manual_work,
|
|
2388
|
+
},
|
|
2389
|
+
}
|
|
2390
|
+
if followup_ticket_id:
|
|
2391
|
+
result_payload["manual_work_followup_id"] = followup_ticket_id
|
|
2392
|
+
|
|
2393
|
+
return result_payload
|
|
2394
|
+
|
|
2395
|
+
# Case 3: Executor succeeded with changes → verifying
|
|
2396
|
+
write_log(log_path, "Execution completed successfully with changes!")
|
|
2397
|
+
|
|
2398
|
+
# Create revision for this execution
|
|
2399
|
+
revision_id = create_revision_for_job(
|
|
2400
|
+
ticket_id=ticket_id,
|
|
2401
|
+
job_id=job_id,
|
|
2402
|
+
diff_stat_evidence_id=diff_stat_evidence_id,
|
|
2403
|
+
diff_patch_evidence_id=diff_patch_evidence_id,
|
|
2404
|
+
)
|
|
2405
|
+
if revision_id:
|
|
2406
|
+
write_log(log_path, f"Created revision {revision_id}")
|
|
2407
|
+
else:
|
|
2408
|
+
write_log(log_path, "WARNING: Failed to create revision record")
|
|
2409
|
+
|
|
2410
|
+
write_log(log_path, "Transitioning ticket to 'verifying'")
|
|
2411
|
+
transition_ticket_sync(
|
|
2412
|
+
ticket_id,
|
|
2413
|
+
TicketState.VERIFYING,
|
|
2414
|
+
reason=f"Execution completed by {executor_info.executor_type.value} CLI with changes",
|
|
2415
|
+
payload={
|
|
2416
|
+
"executor": executor_info.executor_type.value,
|
|
2417
|
+
"evidence_ids": evidence_records,
|
|
2418
|
+
"diff_summary": diff_stat,
|
|
2419
|
+
"has_changes": True,
|
|
2420
|
+
"yolo_mode": yolo_enabled,
|
|
2421
|
+
"revision_id": revision_id,
|
|
2422
|
+
},
|
|
2423
|
+
actor_id="execute_worker",
|
|
2424
|
+
)
|
|
2425
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
2426
|
+
return {
|
|
2427
|
+
"job_id": job_id,
|
|
2428
|
+
"status": "succeeded",
|
|
2429
|
+
"worktree": str(worktree_path),
|
|
2430
|
+
"executor": executor_info.executor_type.value,
|
|
2431
|
+
"evidence_ids": evidence_records,
|
|
2432
|
+
"diff_summary": diff_stat,
|
|
2433
|
+
"has_changes": True,
|
|
2434
|
+
"revision_id": revision_id,
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
|
|
2438
|
+
def verify_ticket_task(job_id: str) -> dict:
|
|
2439
|
+
"""Verify task wrapper for Celery."""
|
|
2440
|
+
try:
|
|
2441
|
+
return _verify_ticket_task_impl(job_id)
|
|
2442
|
+
except Exception as e:
|
|
2443
|
+
import logging
|
|
2444
|
+
|
|
2445
|
+
logging.getLogger(__name__).error(
|
|
2446
|
+
f"verify_ticket_task crashed for job {job_id}: {e}",
|
|
2447
|
+
exc_info=True,
|
|
2448
|
+
)
|
|
2449
|
+
try:
|
|
2450
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
2451
|
+
except Exception:
|
|
2452
|
+
pass
|
|
2453
|
+
try:
|
|
2454
|
+
result = get_job_with_ticket(job_id)
|
|
2455
|
+
if result:
|
|
2456
|
+
_, ticket = result
|
|
2457
|
+
transition_ticket_sync(
|
|
2458
|
+
ticket.id,
|
|
2459
|
+
TicketState.BLOCKED,
|
|
2460
|
+
reason=f"Verification crashed: {e}",
|
|
2461
|
+
actor_id="verify_worker",
|
|
2462
|
+
)
|
|
2463
|
+
except Exception:
|
|
2464
|
+
pass
|
|
2465
|
+
return {"job_id": job_id, "status": "failed", "error": str(e)}
|
|
2466
|
+
|
|
2467
|
+
|
|
2468
|
+
def _verify_ticket_task_impl(job_id: str) -> dict:
|
|
2469
|
+
"""
|
|
2470
|
+
Verify task for a ticket.
|
|
2471
|
+
|
|
2472
|
+
This task:
|
|
2473
|
+
1. Ensures a worktree exists for the ticket
|
|
2474
|
+
2. Loads verification commands from draft.yaml
|
|
2475
|
+
3. Runs each command in the isolated worktree directory
|
|
2476
|
+
4. Creates Evidence records with captured stdout/stderr
|
|
2477
|
+
5. Transitions ticket based on verification outcome
|
|
2478
|
+
"""
|
|
2479
|
+
# Get job and ticket info
|
|
2480
|
+
result = get_job_with_ticket(job_id)
|
|
2481
|
+
if not result:
|
|
2482
|
+
return {
|
|
2483
|
+
"job_id": job_id,
|
|
2484
|
+
"status": "failed",
|
|
2485
|
+
"error": "Job or ticket not found",
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
job, ticket = result
|
|
2489
|
+
goal_id = ticket.goal_id
|
|
2490
|
+
ticket_id = ticket.id
|
|
2491
|
+
|
|
2492
|
+
# Check if ticket is already in a terminal/blocked state - skip verification if so
|
|
2493
|
+
if ticket.state in [
|
|
2494
|
+
TicketState.BLOCKED.value,
|
|
2495
|
+
TicketState.DONE.value,
|
|
2496
|
+
TicketState.ABANDONED.value,
|
|
2497
|
+
]:
|
|
2498
|
+
import logging
|
|
2499
|
+
|
|
2500
|
+
logging.getLogger(__name__).info(
|
|
2501
|
+
f"Skipping verification for job {job_id}: ticket {ticket_id} is already in {ticket.state} state"
|
|
2502
|
+
)
|
|
2503
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=0)
|
|
2504
|
+
return {
|
|
2505
|
+
"job_id": job_id,
|
|
2506
|
+
"status": "skipped",
|
|
2507
|
+
"reason": f"Ticket already in {ticket.state} state",
|
|
2508
|
+
"ticket_id": ticket_id,
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
# Ensure workspace exists
|
|
2512
|
+
worktree_path, workspace_error = ensure_workspace_for_ticket(ticket_id, goal_id)
|
|
2513
|
+
|
|
2514
|
+
# Get log path (use worktree if available, fallback otherwise)
|
|
2515
|
+
log_path, log_path_relative = get_log_path_for_job(job_id, worktree_path)
|
|
2516
|
+
|
|
2517
|
+
write_log(log_path, "Starting verify task...")
|
|
2518
|
+
|
|
2519
|
+
if workspace_error:
|
|
2520
|
+
write_log(log_path, f"WARNING: Could not create workspace: {workspace_error}")
|
|
2521
|
+
write_log(log_path, "Continuing with fallback execution...")
|
|
2522
|
+
else:
|
|
2523
|
+
write_log(log_path, f"Workspace ready at: {worktree_path}")
|
|
2524
|
+
|
|
2525
|
+
# Mark as running
|
|
2526
|
+
if not update_job_started(job_id, log_path_relative):
|
|
2527
|
+
write_log(log_path, "Job was canceled or not found, aborting.")
|
|
2528
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
2529
|
+
|
|
2530
|
+
# Check for cancellation
|
|
2531
|
+
if check_canceled(job_id):
|
|
2532
|
+
write_log(log_path, "Job canceled, stopping execution.")
|
|
2533
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
2534
|
+
|
|
2535
|
+
# Load configuration from DB (board config is the single source of truth)
|
|
2536
|
+
board_config = None
|
|
2537
|
+
if ticket.board_id:
|
|
2538
|
+
with get_sync_db() as db:
|
|
2539
|
+
from app.models.board import Board
|
|
2540
|
+
|
|
2541
|
+
board = db.query(Board).filter(Board.id == ticket.board_id).first()
|
|
2542
|
+
if board and board.config:
|
|
2543
|
+
board_config = board.config
|
|
2544
|
+
|
|
2545
|
+
config = DraftConfig.from_board_config(board_config)
|
|
2546
|
+
verify_config = config.verify_config
|
|
2547
|
+
verify_commands = verify_config.commands
|
|
2548
|
+
|
|
2549
|
+
# Get repo root for relative path computation
|
|
2550
|
+
repo_root = WorkspaceService.get_repo_path()
|
|
2551
|
+
|
|
2552
|
+
write_log(log_path, f"Loaded {len(verify_commands)} verification command(s)")
|
|
2553
|
+
write_log(
|
|
2554
|
+
log_path,
|
|
2555
|
+
"On success: transition to 'needs_human' (requires user approval to move to done)",
|
|
2556
|
+
)
|
|
2557
|
+
|
|
2558
|
+
if not verify_commands:
|
|
2559
|
+
write_log(
|
|
2560
|
+
log_path, "No verification commands configured, skipping verification."
|
|
2561
|
+
)
|
|
2562
|
+
# No commands = success, always transition to needs_human for review
|
|
2563
|
+
write_log(log_path, "Transitioning ticket to 'needs_human' for review")
|
|
2564
|
+
transition_ticket_sync(
|
|
2565
|
+
ticket_id,
|
|
2566
|
+
TicketState.NEEDS_HUMAN,
|
|
2567
|
+
reason="Verification passed (no commands configured), awaiting human approval",
|
|
2568
|
+
)
|
|
2569
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
2570
|
+
return {
|
|
2571
|
+
"job_id": job_id,
|
|
2572
|
+
"status": "succeeded",
|
|
2573
|
+
"worktree": str(worktree_path) if worktree_path else None,
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2576
|
+
# Get evidence directory (with validation against repo_root)
|
|
2577
|
+
evidence_dir = get_evidence_dir(worktree_path, job_id, repo_root=repo_root)
|
|
2578
|
+
|
|
2579
|
+
# Run verification commands with timing
|
|
2580
|
+
verify_start_time = time.time()
|
|
2581
|
+
all_succeeded = True
|
|
2582
|
+
failed_commands: list[dict] = []
|
|
2583
|
+
command_results: list[dict] = []
|
|
2584
|
+
evidence_records: list[str] = []
|
|
2585
|
+
|
|
2586
|
+
for i, command in enumerate(verify_commands):
|
|
2587
|
+
# Check for cancellation before each command
|
|
2588
|
+
if check_canceled(job_id):
|
|
2589
|
+
write_log(log_path, "Job canceled, stopping execution.")
|
|
2590
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
2591
|
+
|
|
2592
|
+
write_log(
|
|
2593
|
+
log_path, f"Running command {i + 1}/{len(verify_commands)}: {command}"
|
|
2594
|
+
)
|
|
2595
|
+
|
|
2596
|
+
# Generate evidence ID
|
|
2597
|
+
evidence_id = str(uuid.uuid4())
|
|
2598
|
+
cmd_start_time = time.time()
|
|
2599
|
+
|
|
2600
|
+
# Run the command (returns relative paths for secure DB storage)
|
|
2601
|
+
exit_code, stdout_path, stderr_path = run_verification_command(
|
|
2602
|
+
command=command,
|
|
2603
|
+
cwd=worktree_path,
|
|
2604
|
+
evidence_dir=evidence_dir,
|
|
2605
|
+
evidence_id=evidence_id,
|
|
2606
|
+
repo_root=repo_root,
|
|
2607
|
+
timeout=300,
|
|
2608
|
+
extra_allowed_commands=verify_config.extra_allowed_commands,
|
|
2609
|
+
)
|
|
2610
|
+
|
|
2611
|
+
cmd_duration_ms = int((time.time() - cmd_start_time) * 1000)
|
|
2612
|
+
|
|
2613
|
+
# Track command result for metadata
|
|
2614
|
+
command_results.append(
|
|
2615
|
+
{
|
|
2616
|
+
"command": command,
|
|
2617
|
+
"exit_code": exit_code,
|
|
2618
|
+
"duration_ms": cmd_duration_ms,
|
|
2619
|
+
"evidence_id": evidence_id,
|
|
2620
|
+
}
|
|
2621
|
+
)
|
|
2622
|
+
|
|
2623
|
+
# Create evidence record (typed as verification output)
|
|
2624
|
+
create_evidence_record(
|
|
2625
|
+
ticket_id=ticket_id,
|
|
2626
|
+
job_id=job_id,
|
|
2627
|
+
command=command,
|
|
2628
|
+
exit_code=exit_code,
|
|
2629
|
+
stdout_path=stdout_path,
|
|
2630
|
+
stderr_path=stderr_path,
|
|
2631
|
+
evidence_id=evidence_id,
|
|
2632
|
+
kind=EvidenceKind.VERIFY_STDOUT,
|
|
2633
|
+
)
|
|
2634
|
+
evidence_records.append(evidence_id)
|
|
2635
|
+
|
|
2636
|
+
if exit_code == 0:
|
|
2637
|
+
write_log(
|
|
2638
|
+
log_path, f"Command succeeded (exit code: 0, {cmd_duration_ms}ms)"
|
|
2639
|
+
)
|
|
2640
|
+
else:
|
|
2641
|
+
write_log(
|
|
2642
|
+
log_path,
|
|
2643
|
+
f"Command FAILED (exit code: {exit_code}, {cmd_duration_ms}ms)",
|
|
2644
|
+
)
|
|
2645
|
+
all_succeeded = False
|
|
2646
|
+
failed_commands.append(
|
|
2647
|
+
{
|
|
2648
|
+
"command": command,
|
|
2649
|
+
"exit_code": exit_code,
|
|
2650
|
+
"evidence_id": evidence_id,
|
|
2651
|
+
}
|
|
2652
|
+
)
|
|
2653
|
+
# Stop on first failure
|
|
2654
|
+
write_log(log_path, "Stopping verification due to failure.")
|
|
2655
|
+
break
|
|
2656
|
+
|
|
2657
|
+
# Calculate total verification duration
|
|
2658
|
+
verify_duration_ms = int((time.time() - verify_start_time) * 1000)
|
|
2659
|
+
|
|
2660
|
+
# Create VERIFY_META evidence with structured metadata
|
|
2661
|
+
verify_meta_id = str(uuid.uuid4())
|
|
2662
|
+
verify_meta = {
|
|
2663
|
+
"total_duration_ms": verify_duration_ms,
|
|
2664
|
+
"commands_configured": verify_commands,
|
|
2665
|
+
"commands_run": len(command_results),
|
|
2666
|
+
"all_succeeded": all_succeeded,
|
|
2667
|
+
"results": command_results,
|
|
2668
|
+
}
|
|
2669
|
+
verify_meta_path = evidence_dir / f"{verify_meta_id}.meta.json"
|
|
2670
|
+
verify_meta_path.write_text(json.dumps(verify_meta, indent=2))
|
|
2671
|
+
# Store relative path for secure DB storage
|
|
2672
|
+
verify_meta_relpath = str(verify_meta_path)
|
|
2673
|
+
create_evidence_record(
|
|
2674
|
+
ticket_id=ticket_id,
|
|
2675
|
+
job_id=job_id,
|
|
2676
|
+
command="verify_metadata",
|
|
2677
|
+
exit_code=0 if all_succeeded else 1,
|
|
2678
|
+
stdout_path=verify_meta_relpath,
|
|
2679
|
+
stderr_path="",
|
|
2680
|
+
evidence_id=verify_meta_id,
|
|
2681
|
+
kind=EvidenceKind.VERIFY_META,
|
|
2682
|
+
)
|
|
2683
|
+
evidence_records.append(verify_meta_id)
|
|
2684
|
+
|
|
2685
|
+
write_log(log_path, f"Total verification time: {verify_duration_ms}ms")
|
|
2686
|
+
|
|
2687
|
+
# Transition ticket based on outcome
|
|
2688
|
+
if all_succeeded:
|
|
2689
|
+
write_log(log_path, "All verification commands passed!")
|
|
2690
|
+
|
|
2691
|
+
# Check if autonomy mode allows auto-approval (skip NEEDS_HUMAN)
|
|
2692
|
+
auto_approved = False
|
|
2693
|
+
try:
|
|
2694
|
+
from app.services.autonomy_service import AutonomyService
|
|
2695
|
+
|
|
2696
|
+
with get_sync_db() as autonomy_db:
|
|
2697
|
+
ticket_for_check = (
|
|
2698
|
+
autonomy_db.query(Ticket).filter(Ticket.id == ticket_id).first()
|
|
2699
|
+
)
|
|
2700
|
+
if ticket_for_check:
|
|
2701
|
+
autonomy_svc = AutonomyService()
|
|
2702
|
+
check = autonomy_svc.can_auto_approve_revision_sync(
|
|
2703
|
+
autonomy_db, ticket_for_check
|
|
2704
|
+
)
|
|
2705
|
+
if check.approved:
|
|
2706
|
+
write_log(
|
|
2707
|
+
log_path,
|
|
2708
|
+
f"Autonomy: auto-approving revision ({check.reason})",
|
|
2709
|
+
)
|
|
2710
|
+
# Transition directly to DONE, skipping NEEDS_HUMAN
|
|
2711
|
+
transition_ticket_sync(
|
|
2712
|
+
ticket_id,
|
|
2713
|
+
TicketState.DONE,
|
|
2714
|
+
reason=f"Auto-approved: verification passed ({len(verify_commands)} command(s)), {check.reason}",
|
|
2715
|
+
payload={
|
|
2716
|
+
"evidence_ids": evidence_records,
|
|
2717
|
+
"duration_ms": verify_duration_ms,
|
|
2718
|
+
"auto_approved": True,
|
|
2719
|
+
},
|
|
2720
|
+
actor_id="autonomy_service",
|
|
2721
|
+
)
|
|
2722
|
+
# Record audit event
|
|
2723
|
+
autonomy_svc.record_auto_action_sync(
|
|
2724
|
+
autonomy_db,
|
|
2725
|
+
ticket_for_check,
|
|
2726
|
+
action_type="approve_revision",
|
|
2727
|
+
details={
|
|
2728
|
+
"reason": check.reason,
|
|
2729
|
+
"evidence_ids": evidence_records,
|
|
2730
|
+
},
|
|
2731
|
+
from_state=TicketState.VERIFYING.value,
|
|
2732
|
+
to_state=TicketState.DONE.value,
|
|
2733
|
+
)
|
|
2734
|
+
autonomy_db.commit()
|
|
2735
|
+
auto_approved = True
|
|
2736
|
+
else:
|
|
2737
|
+
write_log(
|
|
2738
|
+
log_path, f"Autonomy: not auto-approving ({check.reason})"
|
|
2739
|
+
)
|
|
2740
|
+
except Exception as e:
|
|
2741
|
+
write_log(
|
|
2742
|
+
log_path, f"Autonomy check failed (falling back to NEEDS_HUMAN): {e}"
|
|
2743
|
+
)
|
|
2744
|
+
logger.warning(f"Autonomy check failed for ticket {ticket_id}: {e}")
|
|
2745
|
+
|
|
2746
|
+
if not auto_approved:
|
|
2747
|
+
# Default flow: transition to needs_human for review
|
|
2748
|
+
write_log(log_path, "Transitioning ticket to 'needs_human' for review")
|
|
2749
|
+
transition_ticket_sync(
|
|
2750
|
+
ticket_id,
|
|
2751
|
+
TicketState.NEEDS_HUMAN,
|
|
2752
|
+
reason=f"Verification passed: {len(verify_commands)} command(s) succeeded, awaiting human approval",
|
|
2753
|
+
payload={
|
|
2754
|
+
"evidence_ids": evidence_records,
|
|
2755
|
+
"duration_ms": verify_duration_ms,
|
|
2756
|
+
},
|
|
2757
|
+
)
|
|
2758
|
+
|
|
2759
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
2760
|
+
return {
|
|
2761
|
+
"job_id": job_id,
|
|
2762
|
+
"status": "succeeded",
|
|
2763
|
+
"worktree": str(worktree_path) if worktree_path else None,
|
|
2764
|
+
"evidence_ids": evidence_records,
|
|
2765
|
+
"auto_approved": auto_approved,
|
|
2766
|
+
}
|
|
2767
|
+
else:
|
|
2768
|
+
# Verification failed
|
|
2769
|
+
failure_summary = "; ".join(
|
|
2770
|
+
[
|
|
2771
|
+
f"'{fc['command']}' failed with exit code {fc['exit_code']}"
|
|
2772
|
+
for fc in failed_commands
|
|
2773
|
+
]
|
|
2774
|
+
)
|
|
2775
|
+
write_log(log_path, f"Verification FAILED: {failure_summary}")
|
|
2776
|
+
write_log(log_path, "Transitioning ticket to 'blocked'")
|
|
2777
|
+
|
|
2778
|
+
transition_ticket_sync(
|
|
2779
|
+
ticket_id,
|
|
2780
|
+
TicketState.BLOCKED,
|
|
2781
|
+
reason=f"Verification failed: {failure_summary}",
|
|
2782
|
+
payload={
|
|
2783
|
+
"evidence_ids": evidence_records,
|
|
2784
|
+
"failed_commands": failed_commands,
|
|
2785
|
+
},
|
|
2786
|
+
)
|
|
2787
|
+
|
|
2788
|
+
# Use the exit code of the first failed command
|
|
2789
|
+
final_exit_code = failed_commands[0]["exit_code"] if failed_commands else 1
|
|
2790
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=final_exit_code)
|
|
2791
|
+
|
|
2792
|
+
return {
|
|
2793
|
+
"job_id": job_id,
|
|
2794
|
+
"status": "failed",
|
|
2795
|
+
"worktree": str(worktree_path) if worktree_path else None,
|
|
2796
|
+
"evidence_ids": evidence_records,
|
|
2797
|
+
"failed_commands": failed_commands,
|
|
2798
|
+
}
|
|
2799
|
+
|
|
2800
|
+
|
|
2801
|
+
def job_watchdog_task() -> dict:
|
|
2802
|
+
"""
|
|
2803
|
+
Periodic task to monitor and recover stuck jobs.
|
|
2804
|
+
|
|
2805
|
+
This task runs every minute via Celery Beat and checks for:
|
|
2806
|
+
1. RUNNING jobs with stale heartbeat (no update in 2 minutes)
|
|
2807
|
+
2. RUNNING jobs that exceeded their timeout
|
|
2808
|
+
3. QUEUED jobs stuck in queue for over 10 minutes
|
|
2809
|
+
|
|
2810
|
+
For each stuck job, it marks the job as FAILED and transitions
|
|
2811
|
+
the associated ticket to BLOCKED.
|
|
2812
|
+
"""
|
|
2813
|
+
from app.services.job_watchdog_service import run_job_watchdog
|
|
2814
|
+
|
|
2815
|
+
result = run_job_watchdog()
|
|
2816
|
+
|
|
2817
|
+
return {
|
|
2818
|
+
"stale_jobs_recovered": result.stale_jobs_recovered,
|
|
2819
|
+
"timed_out_jobs_recovered": result.timed_out_jobs_recovered,
|
|
2820
|
+
"stuck_queued_jobs_failed": result.stuck_queued_jobs_failed,
|
|
2821
|
+
"tickets_blocked": result.tickets_blocked,
|
|
2822
|
+
"details": result.details,
|
|
2823
|
+
}
|
|
2824
|
+
|
|
2825
|
+
|
|
2826
|
+
def planner_tick_task() -> dict:
|
|
2827
|
+
"""
|
|
2828
|
+
Periodic task to run a planner tick and pick up next tickets.
|
|
2829
|
+
|
|
2830
|
+
This task runs every 5 seconds via Celery Beat and:
|
|
2831
|
+
1. Checks if any PLANNED tickets are ready to execute
|
|
2832
|
+
2. If no ticket is currently EXECUTING/VERIFYING, queues the next one
|
|
2833
|
+
3. Handles follow-ups for BLOCKED tickets (if LLM configured)
|
|
2834
|
+
|
|
2835
|
+
This ensures tickets automatically flow through the queue without
|
|
2836
|
+
requiring the /planner/start HTTP request to stay connected.
|
|
2837
|
+
"""
|
|
2838
|
+
from app.services.planner_tick_sync import PlannerLockError, run_planner_tick_sync
|
|
2839
|
+
|
|
2840
|
+
try:
|
|
2841
|
+
result = run_planner_tick_sync()
|
|
2842
|
+
return {
|
|
2843
|
+
"status": "success",
|
|
2844
|
+
"executed": result.get("executed", 0),
|
|
2845
|
+
"followups_created": result.get("followups_created", 0),
|
|
2846
|
+
"reflections_added": result.get("reflections_added", 0),
|
|
2847
|
+
}
|
|
2848
|
+
except PlannerLockError:
|
|
2849
|
+
# Lock conflict - another tick is running, this is fine
|
|
2850
|
+
return {"status": "skipped", "reason": "Another tick in progress"}
|
|
2851
|
+
except Exception as e:
|
|
2852
|
+
import logging
|
|
2853
|
+
|
|
2854
|
+
logging.getLogger(__name__).error(f"Planner tick failed: {e}")
|
|
2855
|
+
return {"status": "error", "error": str(e)[:200]}
|
|
2856
|
+
|
|
2857
|
+
|
|
2858
|
+
def resume_ticket_task(job_id: str) -> dict:
|
|
2859
|
+
"""Resume task wrapper for Celery."""
|
|
2860
|
+
return _resume_ticket_task_impl(job_id)
|
|
2861
|
+
|
|
2862
|
+
|
|
2863
|
+
def _resume_ticket_task_impl(job_id: str) -> dict:
|
|
2864
|
+
"""
|
|
2865
|
+
Resume a ticket after human completion (interactive executor flow).
|
|
2866
|
+
|
|
2867
|
+
This task is used when a ticket was transitioned to 'needs_human' by an
|
|
2868
|
+
interactive executor (like Cursor). The human has made their changes, and
|
|
2869
|
+
now wants to continue the workflow.
|
|
2870
|
+
|
|
2871
|
+
This task:
|
|
2872
|
+
1. Validates the ticket is in 'needs_human' state
|
|
2873
|
+
2. Captures the git diff as evidence
|
|
2874
|
+
3. Checks if there are any changes
|
|
2875
|
+
4. Transitions to 'verifying' if changes exist, or 'blocked' if no changes
|
|
2876
|
+
"""
|
|
2877
|
+
# Get job and ticket info
|
|
2878
|
+
result = get_job_with_ticket(job_id)
|
|
2879
|
+
if not result:
|
|
2880
|
+
return {
|
|
2881
|
+
"job_id": job_id,
|
|
2882
|
+
"status": "failed",
|
|
2883
|
+
"error": "Job or ticket not found",
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2886
|
+
job, ticket = result
|
|
2887
|
+
goal_id = ticket.goal_id
|
|
2888
|
+
ticket_id = ticket.id
|
|
2889
|
+
|
|
2890
|
+
# Ensure workspace exists
|
|
2891
|
+
worktree_path, workspace_error = ensure_workspace_for_ticket(ticket_id, goal_id)
|
|
2892
|
+
|
|
2893
|
+
# Get log path (use worktree if available, fallback otherwise)
|
|
2894
|
+
log_path, log_path_relative = get_log_path_for_job(job_id, worktree_path)
|
|
2895
|
+
|
|
2896
|
+
write_log(log_path, "Starting resume task...")
|
|
2897
|
+
|
|
2898
|
+
# Workspace is required
|
|
2899
|
+
if workspace_error or not worktree_path:
|
|
2900
|
+
write_log(
|
|
2901
|
+
log_path,
|
|
2902
|
+
f"ERROR: Could not find workspace: {workspace_error or 'Unknown error'}",
|
|
2903
|
+
)
|
|
2904
|
+
update_job_started(job_id, log_path_relative)
|
|
2905
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
2906
|
+
return {"job_id": job_id, "status": "failed", "error": workspace_error}
|
|
2907
|
+
|
|
2908
|
+
write_log(log_path, f"Workspace found at: {worktree_path}")
|
|
2909
|
+
|
|
2910
|
+
# Mark as running
|
|
2911
|
+
if not update_job_started(job_id, log_path_relative):
|
|
2912
|
+
write_log(log_path, "Job was canceled or not found, aborting.")
|
|
2913
|
+
return {"job_id": job_id, "status": "canceled"}
|
|
2914
|
+
|
|
2915
|
+
# Validate ticket is in needs_human state
|
|
2916
|
+
if ticket.state != TicketState.NEEDS_HUMAN.value:
|
|
2917
|
+
write_log(
|
|
2918
|
+
log_path,
|
|
2919
|
+
f"ERROR: Ticket is in '{ticket.state}' state, expected 'needs_human'",
|
|
2920
|
+
)
|
|
2921
|
+
write_log(
|
|
2922
|
+
log_path, "Resume can only be called on tickets in 'needs_human' state."
|
|
2923
|
+
)
|
|
2924
|
+
update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
|
|
2925
|
+
return {
|
|
2926
|
+
"job_id": job_id,
|
|
2927
|
+
"status": "failed",
|
|
2928
|
+
"error": f"Ticket must be in 'needs_human' state to resume, got '{ticket.state}'",
|
|
2929
|
+
}
|
|
2930
|
+
|
|
2931
|
+
# Get repo root for evidence storage and relative path computation
|
|
2932
|
+
repo_root = WorkspaceService.get_repo_path()
|
|
2933
|
+
|
|
2934
|
+
# Get evidence directory (central location)
|
|
2935
|
+
evidence_dir = central_evidence_dir(job_id)
|
|
2936
|
+
evidence_records: list[str] = []
|
|
2937
|
+
|
|
2938
|
+
# Capture git diff
|
|
2939
|
+
write_log(log_path, "Capturing git diff...")
|
|
2940
|
+
diff_stat_evidence_id = str(uuid.uuid4())
|
|
2941
|
+
diff_patch_evidence_id = str(uuid.uuid4())
|
|
2942
|
+
|
|
2943
|
+
diff_exit_code, diff_stat_path, diff_patch_path, diff_stat, has_changes = (
|
|
2944
|
+
capture_git_diff(
|
|
2945
|
+
cwd=worktree_path,
|
|
2946
|
+
evidence_dir=evidence_dir,
|
|
2947
|
+
evidence_id=diff_stat_evidence_id,
|
|
2948
|
+
repo_root=repo_root,
|
|
2949
|
+
)
|
|
2950
|
+
)
|
|
2951
|
+
|
|
2952
|
+
# Create typed evidence records for git diff
|
|
2953
|
+
create_evidence_record(
|
|
2954
|
+
ticket_id=ticket_id,
|
|
2955
|
+
job_id=job_id,
|
|
2956
|
+
command="git diff --stat",
|
|
2957
|
+
exit_code=diff_exit_code,
|
|
2958
|
+
stdout_path=diff_stat_path,
|
|
2959
|
+
stderr_path="",
|
|
2960
|
+
evidence_id=diff_stat_evidence_id,
|
|
2961
|
+
kind=EvidenceKind.GIT_DIFF_STAT,
|
|
2962
|
+
)
|
|
2963
|
+
evidence_records.append(diff_stat_evidence_id)
|
|
2964
|
+
|
|
2965
|
+
create_evidence_record(
|
|
2966
|
+
ticket_id=ticket_id,
|
|
2967
|
+
job_id=job_id,
|
|
2968
|
+
command="git diff",
|
|
2969
|
+
exit_code=diff_exit_code,
|
|
2970
|
+
stdout_path=diff_patch_path,
|
|
2971
|
+
stderr_path="",
|
|
2972
|
+
evidence_id=diff_patch_evidence_id,
|
|
2973
|
+
kind=EvidenceKind.GIT_DIFF_PATCH,
|
|
2974
|
+
)
|
|
2975
|
+
evidence_records.append(diff_patch_evidence_id)
|
|
2976
|
+
|
|
2977
|
+
write_log(log_path, f"Git diff summary:\n{diff_stat}")
|
|
2978
|
+
write_log(log_path, f"Has changes: {has_changes}")
|
|
2979
|
+
|
|
2980
|
+
# Determine outcome
|
|
2981
|
+
if not has_changes:
|
|
2982
|
+
write_log(log_path, "No changes detected in worktree.")
|
|
2983
|
+
write_log(log_path, "Transitioning to 'blocked' (reason: no changes)")
|
|
2984
|
+
transition_ticket_sync(
|
|
2985
|
+
ticket_id,
|
|
2986
|
+
TicketState.BLOCKED,
|
|
2987
|
+
reason="Resume completed but no code changes were found in worktree",
|
|
2988
|
+
payload={
|
|
2989
|
+
"evidence_ids": evidence_records,
|
|
2990
|
+
"diff_summary": diff_stat,
|
|
2991
|
+
"no_changes": True,
|
|
2992
|
+
},
|
|
2993
|
+
actor_id="resume_worker",
|
|
2994
|
+
)
|
|
2995
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
2996
|
+
return {
|
|
2997
|
+
"job_id": job_id,
|
|
2998
|
+
"status": "no_changes",
|
|
2999
|
+
"worktree": str(worktree_path),
|
|
3000
|
+
"evidence_ids": evidence_records,
|
|
3001
|
+
}
|
|
3002
|
+
|
|
3003
|
+
# Changes exist - transition to verifying
|
|
3004
|
+
write_log(log_path, "Changes detected! Transitioning to 'verifying'")
|
|
3005
|
+
transition_ticket_sync(
|
|
3006
|
+
ticket_id,
|
|
3007
|
+
TicketState.VERIFYING,
|
|
3008
|
+
reason="Human completed changes, ready for verification",
|
|
3009
|
+
payload={
|
|
3010
|
+
"evidence_ids": evidence_records,
|
|
3011
|
+
"diff_summary": diff_stat,
|
|
3012
|
+
"resumed_from_interactive": True,
|
|
3013
|
+
},
|
|
3014
|
+
actor_id="resume_worker",
|
|
3015
|
+
)
|
|
3016
|
+
update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
|
|
3017
|
+
|
|
3018
|
+
write_log(log_path, "Resume completed successfully!")
|
|
3019
|
+
return {
|
|
3020
|
+
"job_id": job_id,
|
|
3021
|
+
"status": "succeeded",
|
|
3022
|
+
"worktree": str(worktree_path),
|
|
3023
|
+
"evidence_ids": evidence_records,
|
|
3024
|
+
"diff_summary": diff_stat,
|
|
3025
|
+
}
|
|
3026
|
+
|
|
3027
|
+
|
|
3028
|
+
def poll_pr_statuses():
|
|
3029
|
+
"""
|
|
3030
|
+
Periodic task to poll GitHub PR statuses for tickets.
|
|
3031
|
+
|
|
3032
|
+
This task runs every 5 minutes and:
|
|
3033
|
+
1. Finds tickets with open PRs
|
|
3034
|
+
2. Checks PR status on GitHub
|
|
3035
|
+
3. Auto-transitions tickets if PR is merged
|
|
3036
|
+
"""
|
|
3037
|
+
import subprocess
|
|
3038
|
+
from datetime import datetime
|
|
3039
|
+
from pathlib import Path
|
|
3040
|
+
|
|
3041
|
+
from app.models.workspace import Workspace
|
|
3042
|
+
from app.services.git_host import get_git_host_provider
|
|
3043
|
+
from app.state_machine import TicketState
|
|
3044
|
+
|
|
3045
|
+
# First, collect ticket info without holding DB connection during network calls
|
|
3046
|
+
tickets_to_check = []
|
|
3047
|
+
|
|
3048
|
+
with get_sync_db() as db:
|
|
3049
|
+
# Find tickets with open PRs
|
|
3050
|
+
tickets_with_prs = (
|
|
3051
|
+
db.query(Ticket)
|
|
3052
|
+
.filter(
|
|
3053
|
+
Ticket.pr_number.isnot(None),
|
|
3054
|
+
Ticket.pr_state.in_(["OPEN", "CLOSED"]),
|
|
3055
|
+
)
|
|
3056
|
+
.all()
|
|
3057
|
+
)
|
|
3058
|
+
|
|
3059
|
+
if not tickets_with_prs:
|
|
3060
|
+
return {"message": "No PRs to poll", "checked": 0}
|
|
3061
|
+
|
|
3062
|
+
for ticket in tickets_with_prs:
|
|
3063
|
+
# Get workspace for repo path
|
|
3064
|
+
workspace = (
|
|
3065
|
+
db.query(Workspace).filter(Workspace.ticket_id == ticket.id).first()
|
|
3066
|
+
)
|
|
3067
|
+
|
|
3068
|
+
if workspace and workspace.worktree_path:
|
|
3069
|
+
repo_path = Path(workspace.worktree_path)
|
|
3070
|
+
if repo_path.exists():
|
|
3071
|
+
tickets_to_check.append(
|
|
3072
|
+
{
|
|
3073
|
+
"ticket_id": ticket.id,
|
|
3074
|
+
"pr_number": ticket.pr_number,
|
|
3075
|
+
"pr_state": ticket.pr_state,
|
|
3076
|
+
"pr_merged_at": ticket.pr_merged_at,
|
|
3077
|
+
"repo_path": str(repo_path),
|
|
3078
|
+
}
|
|
3079
|
+
)
|
|
3080
|
+
|
|
3081
|
+
if not tickets_to_check:
|
|
3082
|
+
return {"message": "No valid tickets to poll", "checked": 0}
|
|
3083
|
+
|
|
3084
|
+
git_host = get_git_host_provider()
|
|
3085
|
+
|
|
3086
|
+
# Check if git host CLI is available
|
|
3087
|
+
if not git_host.is_available():
|
|
3088
|
+
return {
|
|
3089
|
+
"message": f"{git_host.name} CLI not available, skipping poll",
|
|
3090
|
+
"checked": 0,
|
|
3091
|
+
}
|
|
3092
|
+
|
|
3093
|
+
updated_count = 0
|
|
3094
|
+
merged_count = 0
|
|
3095
|
+
errors = []
|
|
3096
|
+
|
|
3097
|
+
# Poll GitHub outside of DB session to avoid blocking
|
|
3098
|
+
for ticket_info in tickets_to_check:
|
|
3099
|
+
try:
|
|
3100
|
+
repo_path = Path(ticket_info["repo_path"])
|
|
3101
|
+
pr_number = ticket_info["pr_number"]
|
|
3102
|
+
|
|
3103
|
+
# Use synchronous subprocess call with timeout instead of asyncio.run()
|
|
3104
|
+
result = subprocess.run(
|
|
3105
|
+
["gh", "pr", "view", str(pr_number), "--json", "state,merged"],
|
|
3106
|
+
cwd=repo_path,
|
|
3107
|
+
capture_output=True,
|
|
3108
|
+
text=True,
|
|
3109
|
+
timeout=30, # 30 second timeout per PR check
|
|
3110
|
+
)
|
|
3111
|
+
|
|
3112
|
+
if result.returncode != 0:
|
|
3113
|
+
continue
|
|
3114
|
+
|
|
3115
|
+
pr_details = json.loads(result.stdout)
|
|
3116
|
+
ticket_info["new_state"] = pr_details.get("state", "OPEN")
|
|
3117
|
+
ticket_info["merged"] = pr_details.get("merged", False)
|
|
3118
|
+
|
|
3119
|
+
except subprocess.TimeoutExpired:
|
|
3120
|
+
errors.append(f"Timeout checking PR for ticket {ticket_info['ticket_id']}")
|
|
3121
|
+
continue
|
|
3122
|
+
except Exception as e:
|
|
3123
|
+
errors.append(
|
|
3124
|
+
f"Error polling PR for ticket {ticket_info['ticket_id']}: {e}"
|
|
3125
|
+
)
|
|
3126
|
+
continue
|
|
3127
|
+
|
|
3128
|
+
# Now update DB with results (quick operation)
|
|
3129
|
+
with get_sync_db() as db:
|
|
3130
|
+
for ticket_info in tickets_to_check:
|
|
3131
|
+
if "new_state" not in ticket_info:
|
|
3132
|
+
continue # Skip if we didn't get PR details
|
|
3133
|
+
|
|
3134
|
+
try:
|
|
3135
|
+
ticket = (
|
|
3136
|
+
db.query(Ticket)
|
|
3137
|
+
.filter(Ticket.id == ticket_info["ticket_id"])
|
|
3138
|
+
.first()
|
|
3139
|
+
)
|
|
3140
|
+
if not ticket:
|
|
3141
|
+
continue
|
|
3142
|
+
|
|
3143
|
+
old_state = ticket.pr_state
|
|
3144
|
+
ticket.pr_state = ticket_info["new_state"]
|
|
3145
|
+
|
|
3146
|
+
if ticket_info.get("merged") and not ticket.pr_merged_at:
|
|
3147
|
+
ticket.pr_merged_at = datetime.now()
|
|
3148
|
+
merged_count += 1
|
|
3149
|
+
|
|
3150
|
+
# Auto-transition ticket if PR was merged
|
|
3151
|
+
if ticket_info.get("merged") and old_state != "MERGED":
|
|
3152
|
+
from app.state_machine import validate_transition
|
|
3153
|
+
|
|
3154
|
+
ticket.pr_state = "MERGED"
|
|
3155
|
+
|
|
3156
|
+
if validate_transition(ticket.state, TicketState.DONE.value):
|
|
3157
|
+
ticket.state = TicketState.DONE.value
|
|
3158
|
+
|
|
3159
|
+
# Create event
|
|
3160
|
+
event = TicketEvent(
|
|
3161
|
+
ticket_id=ticket.id,
|
|
3162
|
+
event_type=EventType.TRANSITIONED.value,
|
|
3163
|
+
from_state=old_state or "REVIEW",
|
|
3164
|
+
to_state=TicketState.DONE.value,
|
|
3165
|
+
actor_type=ActorType.SYSTEM.value,
|
|
3166
|
+
actor_id="poll_pr_statuses",
|
|
3167
|
+
reason=f"PR #{ticket.pr_number} was merged",
|
|
3168
|
+
)
|
|
3169
|
+
db.add(event)
|
|
3170
|
+
else:
|
|
3171
|
+
import logging
|
|
3172
|
+
|
|
3173
|
+
logging.getLogger(__name__).warning(
|
|
3174
|
+
f"Cannot transition ticket {ticket.id} from "
|
|
3175
|
+
f"{ticket.state} to DONE on PR merge"
|
|
3176
|
+
)
|
|
3177
|
+
|
|
3178
|
+
updated_count += 1
|
|
3179
|
+
|
|
3180
|
+
except Exception as e:
|
|
3181
|
+
errors.append(f"Error updating ticket {ticket_info['ticket_id']}: {e}")
|
|
3182
|
+
continue
|
|
3183
|
+
|
|
3184
|
+
return {
|
|
3185
|
+
"message": "PR polling completed",
|
|
3186
|
+
"checked": len(tickets_to_check),
|
|
3187
|
+
"updated": updated_count,
|
|
3188
|
+
"merged": merged_count,
|
|
3189
|
+
"errors": errors[:10] if errors else [], # Limit error list
|
|
3190
|
+
}
|