okstra 0.207.1 → 0.208.0

package/runtime/python/okstra_ctl/business_flow/engine.py

@@ -0,0 +1,518 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, replace
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, cast
+from uuid import uuid4
+
+from ..agent.invocation import (
+    AgentInvocationError,
+    materialize_standalone_result,
+    publish_standalone_completion,
+    verify_standalone_completion,
+)
+from ..json_boundary import JsonBoundaryError, load_owned_object, serialize_owned_object
+from .contracts import (
+    OPERATION,
+    BusinessFact,
+    BusinessFlowError,
+    BusinessResult,
+    FlowExecution,
+    FlowRequest,
+    FlowRequestPayload,
+    SourceSnapshot,
+    artifact_path,
+    now_iso,
+    read_artifact,
+    schema_definition,
+    write_artifact,
+)
+from .invocation import FlowExecutor, PreparedCall, ProviderFlowExecutor
+from .source import (
+    candidate_projects,
+    capture_projects,
+    changed_projects,
+    source_changes,
+    validate_evidence,
+)
+from .store import (
+    contribute_knowledge,
+    query_knowledge,
+    read_shared_knowledge,
+    record_task_execution,
+    resolve_conflicts,
+    validate_resolutions,
+)
+
+
+def explain_flow(
+    request: FlowRequest, executor: FlowExecutor | None = None
+) -> FlowExecution:
+    request.validate()
+    request = replace(
+        request,
+        project_root=request.project_root.resolve(),
+        source_root=request.source_root.resolve() if request.source_root else None,
+        source_artifacts=tuple(
+            str(Path(path).resolve()) for path in request.source_artifacts
+        ),
+    )
+    execution_id = "flow-" + uuid4().hex
+    path = artifact_path(
+        request.project_root, "executions", execution_id, "execution.json"
+    )
+    execution: FlowExecution = {
+        "schemaVersion": 1,
+        "id": execution_id,
+        "createdAt": now_iso(),
+        "request": _request_payload(request),
+        "projects": [],
+        "status": "prepared",
+        "readOnlyAudit": {
+            "policy": "source-readonly",
+            "boundary": "none",
+            "coverage": "partial",
+            "method": "before/after Git HEAD, index and source file fingerprints",
+            "unobserved": [
+                "ignored files",
+                "external services",
+                "writes outside candidate source roots",
+                "transient writes restored before the audit",
+            ],
+        },
+    }
+    write_artifact(path, execution, "execution")
+    try:
+        candidates = candidate_projects(request)
+        execution["projects"] = capture_projects(candidates)
+        execution["sourceArtifactDigests"] = {
+            name: hashlib.sha256(Path(name).read_bytes()).hexdigest()
+            for name in request.source_artifacts
+        }
+        _investigate(request, execution, path, executor or ProviderFlowExecutor())
+    except (OSError, ValueError, AgentInvocationError, JsonBoundaryError) as exc:
+        execution.update(status="failed", error=str(exc))
+        write_artifact(path, execution, "execution")
+    return execution
+
+
+def _request_payload(request: FlowRequest) -> FlowRequestPayload:
+    payload = asdict(request)
+    payload["project_root"] = str(request.project_root)
+    payload["source_root"] = str(request.source_root) if request.source_root else None
+    payload["source_artifacts"] = list(request.source_artifacts)
+    return cast(FlowRequestPayload, payload)
+
+
+def _investigate(
+    request: FlowRequest, execution: FlowExecution, path: Path, executor: FlowExecutor
+) -> None:
+    baseline = (
+        load_execution(request.project_root, request.baseline)
+        if request.baseline
+        else None
+    )
+    execution["changes"] = source_changes(
+        baseline["projects"] if baseline else [], execution["projects"]
+    )
+    call = executor.prepare(
+        request, execution["id"], _instructions(request, execution, baseline)
+    )
+    execution.update(status="running", metadataPath=str(call.metadata_path))
+    write_artifact(path, execution, "execution")
+    returned = executor.invoke(call)
+    verified = _accept_return(call, returned)
+    result = _parse_result(verified, path)
+    _validate_investigation_result(request, execution, result)
+    execution["result"] = result
+    execution["status"] = "partial" if result["gaps"] else "complete"
+    producer = {
+        "executionId": execution["id"],
+        "taskKey": request.task_key,
+        "mode": request.mode,
+        "metadataPath": str(call.metadata_path),
+    }
+    execution["contribution"] = contribute_knowledge(
+        request.project_root, result["facts"], execution["projects"], producer
+    )
+    if result["resolutions"]:
+        resolve_conflicts(request.project_root, result["resolutions"], producer)
+    write_artifact(path, execution, "execution")
+    _reinvestigate_conflicts(request, execution, path, executor)
+    if request.mode != "contribute":
+        from .report import render_explanation
+
+        execution["reports"] = render_explanation(request.project_root, execution)
+    write_artifact(path, execution, "execution")
+
+
+def _validate_investigation_result(
+    request: FlowRequest, execution: FlowExecution, result: BusinessResult
+) -> None:
+    changed = changed_projects(
+        execution["projects"], capture_projects(candidate_projects(request))
+    )
+    if changed:
+        raise BusinessFlowError(
+            "source changed during read-only investigation: " + ", ".join(changed)
+        )
+    if any(
+        hashlib.sha256(Path(name).read_bytes()).hexdigest() != digest
+        for name, digest in execution["sourceArtifactDigests"].items()
+    ):
+        raise BusinessFlowError("supplied source artifact changed during investigation")
+    _validate_result(result, execution["projects"], request)
+    unavailable = {
+        row["projectId"] for row in execution["changes"] if not row["baselineAvailable"]
+    }
+    if request.mode == "post" and any(
+        step["before"] and unavailable.intersection(step["projectIds"])
+        for step in result["steps"]
+    ):
+        raise BusinessFlowError(
+            "before-state assertion references a project without a captured baseline"
+        )
+    if result["resolutions"]:
+        _validate_reconciliation(result, request, execution["projects"])
+
+
+def _accept_return(call: PreparedCall, returned: bytes) -> str:
+    materialize_standalone_result(
+        project_root=call.project_root,
+        purpose=OPERATION,
+        metadata_path=call.metadata_path,
+        returned_body=returned,
+    )
+    completion = publish_standalone_completion(
+        project_root=call.project_root,
+        purpose=OPERATION,
+        metadata_path=call.metadata_path,
+        completed_at=datetime.now(timezone.utc),
+    )
+    return verify_standalone_completion(
+        completion, project_root=call.project_root, expected_purpose=OPERATION
+    ).returned_body
+
+
+def _parse_result(body: str, path: Path) -> BusinessResult:
+    text = body.strip()
+    if text.startswith("```json\n") and text.endswith("\n```"):
+        text = text[8:-4]
+    try:
+        result = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise BusinessFlowError(
+            f"business investigator returned invalid JSON: {exc}"
+        ) from exc
+    serialize_owned_object(
+        path,
+        result,
+        artifact="business investigator response",
+        schema=schema_definition("result"),
+    )
+    return cast(BusinessResult, result)
+
+
+def _validate_result(
+    result: BusinessResult, projects: list[SourceSnapshot], request: FlowRequest
+) -> None:
+    ids = {row["projectId"] for row in projects}
+    if not set(result["inspectedProjects"]).issubset(ids):
+        raise BusinessFlowError("inspection references an unregistered candidate")
+    result["gaps"].extend(
+        f"Candidate not inspected: {key}"
+        for key in sorted(ids - set(result["inspectedProjects"]))
+    )
+    result["gaps"].extend(
+        f"{row['projectId']}: {gap}" for row in projects for gap in row["gaps"]
+    )
+    if request.mode == "post" and not request.baseline:
+        result["gaps"].append(
+            "No pre-implementation baseline: before-state cannot be established"
+        )
+        if any(step["before"] for step in result["steps"]):
+            raise BusinessFlowError(
+                "before-state assertions require a captured baseline"
+            )
+    keys = {row["key"] for row in result["facts"]}
+    if len(keys) != len(result["facts"]) or len(
+        {step["id"] for step in result["steps"]}
+    ) != len(result["steps"]):
+        raise BusinessFlowError(
+            "business claim keys and step identifiers must be unique"
+        )
+    _validate_fact_evidence(result, projects, request)
+    for step in result["steps"]:
+        if not set(step["projectIds"]).issubset(ids) or not set(
+            step["factKeys"]
+        ).issubset(keys):
+            raise BusinessFlowError(
+                "business step references an unknown project or claim"
+            )
+    _validate_relationships(result, ids, keys)
+    if not result["steps"] and request.mode not in {"contribute", "reconcile"}:
+        result["gaps"].append(
+            "Business steps could not be established from the available sources"
+        )
+
+
+def _validate_relationships(
+    result: BusinessResult, ids: set[str], keys: set[str]
+) -> None:
+    for relationship in result["relationships"]:
+        if not {relationship["from"], relationship["to"]}.issubset(ids) or not set(
+            relationship["factKeys"]
+        ).issubset(keys):
+            raise BusinessFlowError(
+                "business relationship references an unknown project or claim"
+            )
+        evidence_projects = {
+            e["projectId"]
+            for fact in result["facts"]
+            if fact["key"] in relationship["factKeys"]
+            for e in fact["evidence"]
+        }
+        if not {relationship["from"], relationship["to"]}.issubset(evidence_projects):
+            result["gaps"].append(
+                f"Relationship endpoint evidence incomplete: {relationship['from']} / {relationship['to']}"
+            )
+
+
+def _validate_fact_evidence(
+    result: BusinessResult, projects: list[SourceSnapshot], request: FlowRequest
+) -> None:
+    for fact in result["facts"]:
+        if fact["level"] in {"static", "execution-verified"} and not fact["evidence"]:
+            raise BusinessFlowError(
+                "confirmed business claims require inspected code evidence"
+            )
+        for evidence in fact["evidence"]:
+            validate_evidence(evidence, projects)
+            if evidence["projectId"] not in result["inspectedProjects"]:
+                raise BusinessFlowError(
+                    "claim evidence references an uninspected candidate"
+                )
+        if fact["level"] == "execution-verified":
+            _validate_execution_receipts(fact, request, projects)
+
+
+def _validate_execution_receipts(
+    fact: BusinessFact, request: FlowRequest, projects: list[SourceSnapshot]
+) -> None:
+    receipts = fact.get("executionEvidence", [])
+    if not receipts or not set(receipts).issubset(request.source_artifacts):
+        raise BusinessFlowError(
+            "execution verification must cite a supplied existing verification record"
+        )
+    states = {row["projectId"]: row["digest"] for row in projects}
+    for receipt in receipts:
+        record = load_owned_object(
+            Path(receipt),
+            artifact="existing business verification receipt",
+            schema=schema_definition("verificationReceipt"),
+        )
+        start, end = (
+            datetime.fromisoformat(record["startedAt"]),
+            datetime.fromisoformat(record["endedAt"]),
+        )
+        if (
+            start.tzinfo is None
+            or end.tzinfo is None
+            or end < start
+            or end > datetime.now(timezone.utc)
+        ):
+            raise BusinessFlowError(
+                "verification receipt has invalid execution timestamps"
+            )
+        needed = {row["projectId"] for row in fact["evidence"]}
+        if not needed.issubset(record["sourceStates"]) or any(
+            states.get(key) != digest for key, digest in record["sourceStates"].items()
+        ):
+            raise BusinessFlowError(
+                "verification receipt does not match the captured source version"
+            )
+
+
+def _validate_reconciliation(
+    result: BusinessResult, request: FlowRequest, projects: list[SourceSnapshot]
+) -> None:
+    if request.mode != "reconcile":
+        raise BusinessFlowError(
+            "conflict resolutions require a dedicated reinvestigation"
+        )
+    knowledge = read_shared_knowledge(request.project_root)
+    validate_resolutions(knowledge, result["resolutions"])
+    claims = {row["id"]: row for row in knowledge["claims"]}
+    states = {row["projectId"]: row["digest"] for row in projects}
+    for resolution in result["resolutions"]:
+        claim = claims[resolution["acceptedClaimId"]]
+        if any(
+            states.get(key) != value["digest"]
+            for key, value in claim["sourceStates"].items()
+        ):
+            raise BusinessFlowError(
+                "conflict source changed; preserve historical conflict"
+            )
+        if not any(
+            fact["key"] == claim["key"]
+            and fact["value"] == claim["value"]
+            and fact["level"] in {"static", "execution-verified"}
+            for fact in result["facts"]
+        ):
+            raise BusinessFlowError(
+                "accepted conflict claim lacks reinspected confirming evidence"
+            )
+
+
+def _instructions(
+    request: FlowRequest, execution: FlowExecution, baseline: FlowExecution | None
+) -> str:
+    context = _investigation_context(request, execution, baseline)
+    schema = schema_definition("result")
+    return "\n".join(
+        [
+            "# Business Flow Investigation",
+            "",
+            "Return only one JSON object matching the schema below.",
+            "Read the listed source roots and source artifacts only. Do not edit files, execute tests, start services, deploy or migrate.",
+            "Inspect inbound AND outbound dependencies across all candidates, including indirect call/event/queue/shared-data/file/batch relationships. Name every uninspected candidate or missing branch in gaps.",
+            "Explain the product business from start to finish for a new developer: purpose, rules, input/output, states, failure, retry and recovery. Link each step and relationship to factKeys.",
+            "Use stable keys in business/subject/attribute form. Facts are source-backed assertions; expected changes remain expected. Cite exact inspected excerpts with one-based line/endLine. Tests are static evidence unless a supplied existing verification record supports the execution claim.",
+            "For post mode, use the supplied project-specific baseline. Never invent missing before-state. Describe the actual delta and distinguish confirmed side effects from risks.",
+            "For contribute mode, extract newly observed business facts from the supplied validated task record and inspect source to confirm them. Empty facts are valid. Do not copy the report narrative as established truth.",
+            "For reconcile mode, reinspect every conflicting source and condition. Return explicit resolutions only with evidence-backed reasoning and original claimIds/acceptedClaimId. Preserve genuinely unresolved policy decisions in questions. Other modes return resolutions=[].",
+            "Keep fact keys, business identifiers and fact values canonical in English; preserve original business terms in aliases. Do not create conflicts by translating the same fact. Read source evidence before accepting stored knowledge, including other projects' claims.",
+            "execution-verified requires a supplied JSON verificationReceipt with command, startedAt, endedAt, exitCode=0, output and matching projectId-to-digest sourceStates. A final report alone is not an execution receipt.",
+            f"Write report titles, summaries, terms, steps, relationship data, gaps and questions in {request.report_language}; preserve paths, identifiers and enum values.",
+            "",
+            "## Source Material and Captured Baseline",
+            json.dumps(context, ensure_ascii=False, indent=2),
+            "",
+            "## Result Schema",
+            json.dumps(schema, ensure_ascii=False, indent=2),
+            "",
+        ]
+    )
+
+
+def _investigation_context(
+    request: FlowRequest, execution: FlowExecution, baseline: FlowExecution | None
+) -> dict[str, Any]:
+    compact = [
+        {
+            "projectId": row["projectId"],
+            "root": row["root"],
+            "head": row["head"],
+            "digest": row["digest"],
+            "sourceFileCount": len(row["files"]),
+            "dirtyPaths": list(row["dirty"]),
+            "gaps": row["gaps"],
+        }
+        for row in execution["projects"]
+    ]
+    return {
+        "request": execution["request"],
+        "projects": compact,
+        "snapshotPath": str(
+            artifact_path(
+                request.project_root, "executions", execution["id"], "execution.json"
+            )
+        ),
+        "changes": execution["changes"],
+        "baseline": baseline.get("result") if baseline else None,
+        "baselinePath": str(
+            artifact_path(
+                request.project_root, "executions", request.baseline, "execution.json"
+            )
+        )
+        if baseline
+        else "",
+        "knowledge": query_knowledge(
+            request.project_root, request.question, request.source_root
+        ),
+    }
+
+
+def _reinvestigate_conflicts(
+    request: FlowRequest, execution: FlowExecution, path: Path, executor: FlowExecutor
+) -> None:
+    if request.mode == "reconcile":
+        return
+    conflict_keys = {
+        row["key"]
+        for row in query_knowledge(
+            request.project_root, request.question, request.source_root
+        )["claims"]
+        if row["status"] == "conflict"
+    }
+    if not conflict_keys:
+        return
+    child = explain_flow(
+        replace(
+            request,
+            mode="reconcile",
+            question="\n".join(sorted(conflict_keys)),
+            baseline="",
+        ),
+        executor,
+    )
+    execution["reconciliation"] = {
+        "executionId": child["id"],
+        "status": child["status"],
+        "error": child.get("error", ""),
+        "questions": child.get("result", {}).get("questions", []),
+    }
+    unresolved = [
+        row["key"]
+        for row in query_knowledge(
+            request.project_root, request.question, request.source_root
+        )["claims"]
+        if row["status"] == "conflict"
+    ]
+    if unresolved:
+        execution["result"]["questions"].append(
+            "Unresolved shared business conflicts: "
+            + ", ".join(sorted(set(unresolved)))
+        )
+        execution["status"] = "partial"
+    write_artifact(path, execution, "execution")
+
+
+def load_execution(project_root: Path, execution_id: str) -> FlowExecution:
+    if not execution_id or any(
+        char not in "abcdefghijklmnopqrstuvwxyz0123456789-" for char in execution_id
+    ):
+        raise BusinessFlowError("invalid business explanation execution identifier")
+    return cast(
+        FlowExecution,
+        read_artifact(
+            artifact_path(project_root, "executions", execution_id, "execution.json"),
+            "execution",
+        ),
+    )
+
+
+def rerun_explanation(
+    project_root: Path,
+    execution_id: str,
+    executor: FlowExecutor | None = None,
+    *,
+    source_root: Path | None = None,
+) -> FlowExecution:
+    old = load_execution(project_root, execution_id)
+    payload = dict(old["request"])
+    payload["project_root"] = project_root
+    payload["source_root"] = (
+        Path(payload["source_root"]) if payload["source_root"] else None
+    )
+    payload["source_artifacts"] = tuple(payload["source_artifacts"])
+    if source_root is not None:
+        payload["source_root"] = source_root
+    request = FlowRequest(**payload)
+    execution = explain_flow(request, executor)
+    if request.task_key:
+        record_task_execution(request, execution, execution["id"])
+    return execution