okstra 0.231.0 → 0.232.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -3
- package/dist/cli-registry.mjs +6 -0
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/memory/memory.mjs +9 -55
- package/dist/commands/memory/memory.mjs.map +1 -1
- package/dist/lib/memory-auto-register.d.mts +1 -0
- package/dist/lib/memory-auto-register.mjs +113 -0
- package/dist/lib/memory-auto-register.mjs.map +1 -0
- package/dist/lib/memory-entry-contract.d.mts +12 -0
- package/dist/lib/memory-entry-contract.mjs +46 -0
- package/dist/lib/memory-entry-contract.mjs.map +1 -0
- package/dist/lib/python-helper.d.mts +1 -1
- package/dist/lib/python-helper.mjs +3 -2
- package/dist/lib/python-helper.mjs.map +1 -1
- package/dist/lib/record-secrets.d.mts +1 -0
- package/dist/lib/record-secrets.mjs +28 -0
- package/dist/lib/record-secrets.mjs.map +1 -0
- package/dist/lib/types.d.mts +2 -1
- package/docs/architecture/storage-model.md +129 -2
- package/docs/architecture.md +11 -2
- package/docs/cli.md +234 -0
- package/docs/project-structure-overview.md +2 -1
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/agents/common.json +8 -8
- package/runtime/agents/roles/verifier.json +2 -4
- package/runtime/bin/okstra-provider-exec.py +10 -4
- package/runtime/prompts/coding-preflight/architectures/hexagonal.md +23 -30
- package/runtime/prompts/coding-preflight/authoring-checks.md +17 -0
- package/runtime/prompts/coding-preflight/clean-code.md +85 -281
- package/runtime/prompts/coding-preflight/frameworks/node-server.md +2 -3
- package/runtime/prompts/coding-preflight/languages/java.md +3 -5
- package/runtime/prompts/coding-preflight/languages/javascript-typescript.md +5 -12
- package/runtime/prompts/coding-preflight/languages/kotlin.md +2 -4
- package/runtime/prompts/coding-preflight/languages/python.md +2 -7
- package/runtime/prompts/coding-preflight/languages/rust.md +2 -8
- package/runtime/prompts/coding-preflight/languages/sql.md +2 -4
- package/runtime/prompts/coding-preflight/overview.md +25 -38
- package/runtime/prompts/coding-preflight/review-rules.json +30 -0
- package/runtime/prompts/duties/code-reviewer.json +15 -18
- package/runtime/prompts/duties/implementation-verifier.json +1 -0
- package/runtime/prompts/lead/convergence.md +3 -3
- package/runtime/prompts/lead/okstra-lead-contract.md +2 -2
- package/runtime/prompts/lead/report-writer.md +1 -1
- package/runtime/prompts/lead/team-contract.md +1 -1
- package/runtime/prompts/profiles/_clarification-recommendation.md +3 -2
- package/runtime/prompts/profiles/_coding-conventions-preflight.md +41 -6
- package/runtime/python/okstra_ctl/agent/invocation.py +121 -36
- package/runtime/python/okstra_ctl/agent/prompt_cli/cli.py +6 -0
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +30 -4
- package/runtime/python/okstra_ctl/agent/standalone.py +3 -0
- package/runtime/python/okstra_ctl/analysis_packet.py +2 -19
- package/runtime/python/okstra_ctl/approval_decisions.py +16 -0
- package/runtime/python/okstra_ctl/attempt_evidence.py +10 -0
- package/runtime/python/okstra_ctl/business_flow/invocation.py +35 -3
- package/runtime/python/okstra_ctl/code_review_census.py +71 -7
- package/runtime/python/okstra_ctl/code_review_coverage.py +4 -274
- package/runtime/python/okstra_ctl/code_review_integrity.py +154 -0
- package/runtime/python/okstra_ctl/code_review_invocation.py +212 -0
- package/runtime/python/okstra_ctl/code_review_scope.py +36 -9
- package/runtime/python/okstra_ctl/code_review_target.py +85 -11
- package/runtime/python/okstra_ctl/consumers.py +12 -9
- package/runtime/python/okstra_ctl/convergence.py +112 -20
- package/runtime/python/okstra_ctl/convergence_engine.py +1 -0
- package/runtime/python/okstra_ctl/convergence_provenance.py +27 -11
- package/runtime/python/okstra_ctl/correction_round.py +6 -1
- package/runtime/python/okstra_ctl/design_review.py +207 -0
- package/runtime/python/okstra_ctl/design_review_evidence.py +297 -0
- package/runtime/python/okstra_ctl/dispatch_checkpoints.py +28 -5
- package/runtime/python/okstra_ctl/dispatch_core.py +3 -11
- package/runtime/python/okstra_ctl/doctor.py +8 -25
- package/runtime/python/okstra_ctl/doyouknow/__init__.py +9 -0
- package/runtime/python/okstra_ctl/doyouknow/cli.py +91 -0
- package/runtime/python/okstra_ctl/doyouknow/config.py +58 -0
- package/runtime/python/okstra_ctl/doyouknow/contracts.py +184 -0
- package/runtime/python/okstra_ctl/doyouknow/identity.py +22 -0
- package/runtime/python/okstra_ctl/doyouknow/paths.py +22 -0
- package/runtime/python/okstra_ctl/doyouknow/pending.py +72 -0
- package/runtime/python/okstra_ctl/doyouknow/registration.py +260 -0
- package/runtime/python/okstra_ctl/doyouknow/source_contracts.py +128 -0
- package/runtime/python/okstra_ctl/doyouknow/source_store.py +172 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/io.py +36 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/manager.py +261 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/memory.py +179 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/task.py +95 -0
- package/runtime/python/okstra_ctl/doyouknow/store.py +260 -0
- package/runtime/python/okstra_ctl/doyouknow/sync.py +58 -0
- package/runtime/python/okstra_ctl/execution_mutation_audit.py +18 -2
- package/runtime/python/okstra_ctl/file_size_cap.py +3 -4
- package/runtime/python/okstra_ctl/json_boundary.py +21 -2
- package/runtime/python/okstra_ctl/jsonl.py +21 -0
- package/runtime/python/okstra_ctl/manager_cli.py +26 -0
- package/runtime/python/okstra_ctl/manager_registration.py +29 -0
- package/runtime/python/okstra_ctl/manager_sync.py +3 -22
- package/runtime/python/okstra_ctl/model_io/renderers.py +13 -2
- package/runtime/python/okstra_ctl/model_io_cli.py +7 -1
- package/runtime/python/okstra_ctl/phases/final_verification/validation.py +20 -0
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +12 -2
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-diff-review.md +19 -6
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-self-check.md +5 -3
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +39 -11
- package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation.template.html +13 -0
- package/runtime/python/okstra_ctl/phases/implementation/spec.md +26 -1
- package/runtime/python/okstra_ctl/phases/implementation/validation.py +86 -2
- package/runtime/python/okstra_ctl/phases/implementation_planning/authoring.py +4 -0
- package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +1 -1
- package/runtime/python/okstra_ctl/report_assembly.py +15 -0
- package/runtime/python/okstra_ctl/report_narrative.py +3 -0
- package/runtime/python/okstra_ctl/report_prose_keys.py +7 -0
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +19 -2
- package/runtime/python/okstra_ctl/review_policy.py +359 -0
- package/runtime/python/okstra_ctl/review_verdicts.py +291 -0
- package/runtime/python/okstra_ctl/run.py +24 -1
- package/runtime/python/okstra_ctl/stage_done.py +11 -0
- package/runtime/python/okstra_ctl/standalone_review_audit.py +28 -0
- package/runtime/python/okstra_ctl/team.py +3 -1
- package/runtime/python/okstra_ctl/verification_generation.py +23 -1
- package/runtime/python/okstra_ctl/verification_input_types.py +15 -1
- package/runtime/python/okstra_ctl/verification_inputs.py +28 -6
- package/runtime/python/okstra_ctl/verification_invocation.py +7 -1
- package/runtime/python/okstra_ctl/verification_target.py +2 -0
- package/runtime/python/okstra_ctl/worker_dispatch.py +3 -1
- package/runtime/python/okstra_ctl/worker_prompt_headers.py +24 -9
- package/runtime/python/okstra_ctl/worker_runner.py +66 -10
- package/runtime/python/okstra_ctl/wrapper_status.py +16 -0
- package/runtime/python/okstra_ctl/write_policy.py +34 -1
- package/runtime/python/okstra_project/__init__.py +2 -0
- package/runtime/python/okstra_project/dirs.py +18 -0
- package/runtime/python/okstra_project/state.py +25 -20
- package/runtime/schemas/doyouknow-config-v1.schema.json +20 -0
- package/runtime/schemas/doyouknow-v1.schema.json +65 -0
- package/runtime/schemas/final-report-v2.0.schema.json +157 -0
- package/runtime/schemas/final-report-v3.0.schema.json +157 -0
- package/runtime/schemas/memory-entry-contract-v1.json +11 -0
- package/runtime/schemas/record-secret-patterns.json +41 -0
- package/runtime/skills/okstra-code-review/SKILL.md +71 -123
- package/runtime/skills/okstra-code-review/references/census-rules.md +29 -105
- package/runtime/skills/okstra-code-review/references/review-calibration.md +34 -54
- package/runtime/skills/okstra-manager/SKILL.md +7 -0
- package/runtime/skills/okstra-memory/SKILL.md +7 -0
- package/runtime/templates/implementation-worker-preamble.md +4 -9
- package/runtime/templates/worker-prompt-preamble.md +4 -12
- package/runtime/validators/validate-run.py +17 -8
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import asdict, dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
from ..json_boundary import serialize_owned_object
|
|
9
|
+
from .contracts import (DoyouknowError, MAX_INPUT_BYTES, Node, Relation, _check_input_depth,
|
|
10
|
+
_decoded_strings, asset_object, canonical_json, record_secret_policy_digest, reject_secrets)
|
|
11
|
+
from .identity import stable_id
|
|
12
|
+
|
|
13
|
+
SourceKind = Literal["memory", "manager", "task"]
|
|
14
|
+
Lifecycle = Literal["active", "archived", "unlinked", "missing"]
|
|
15
|
+
JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SourceRef:
|
|
20
|
+
kind: SourceKind
|
|
21
|
+
owner_id: str
|
|
22
|
+
native_id: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class SourceDomain:
|
|
27
|
+
kind: SourceKind
|
|
28
|
+
owner_id: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class SourceSnapshot:
|
|
33
|
+
ref: SourceRef
|
|
34
|
+
primary_node: Node
|
|
35
|
+
locator: str
|
|
36
|
+
metadata: dict[str, JsonValue] = field(default_factory=dict)
|
|
37
|
+
nodes: tuple[Node, ...] = ()
|
|
38
|
+
relations: tuple[Relation, ...] = ()
|
|
39
|
+
lifecycle: Lifecycle = "active"
|
|
40
|
+
source_updated_at: str | None = None
|
|
41
|
+
scope: str | None = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class SourceScan:
|
|
46
|
+
domain: SourceDomain
|
|
47
|
+
snapshots: tuple[SourceSnapshot, ...]
|
|
48
|
+
observed_refs: tuple[SourceRef, ...]
|
|
49
|
+
status: Literal["complete", "partial", "unavailable"]
|
|
50
|
+
errors: tuple[str, ...] = ()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class PreparedSource:
|
|
55
|
+
ref: SourceRef
|
|
56
|
+
fingerprint: str | None
|
|
57
|
+
snapshot: SourceSnapshot | None = field(default=None, repr=False)
|
|
58
|
+
outcome: Literal["ready", "excluded", "unavailable"] = "ready"
|
|
59
|
+
secret_texts: tuple[str, ...] = field(default=(), repr=False)
|
|
60
|
+
scan: SourceScan | None = field(default=None, repr=False)
|
|
61
|
+
error_code: str | None = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def prepared_snapshot(snapshot: SourceSnapshot) -> PreparedSource:
|
|
65
|
+
return PreparedSource(snapshot.ref, stable_id("source-content", canonical_json(asdict(snapshot)), record_secret_policy_digest()), snapshot)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def validate_prepared_snapshot(prepared: PreparedSource) -> None:
|
|
69
|
+
if prepared.outcome != "ready" or prepared.snapshot is None:
|
|
70
|
+
raise DoyouknowError("source snapshot unavailable")
|
|
71
|
+
for text in prepared.secret_texts:
|
|
72
|
+
reject_secrets(text)
|
|
73
|
+
snapshot_payload(prepared.snapshot)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def validate_source_value(value: object) -> str:
|
|
77
|
+
_check_input_depth(value)
|
|
78
|
+
try:
|
|
79
|
+
encoded = canonical_json(value)
|
|
80
|
+
except (ValueError, TypeError) as exc:
|
|
81
|
+
raise DoyouknowError("source input is not a JSON value") from exc
|
|
82
|
+
if len(encoded.encode("utf-8")) > MAX_INPUT_BYTES:
|
|
83
|
+
raise DoyouknowError("source input exceeds 2000000 bytes")
|
|
84
|
+
reject_secrets("\n".join(_decoded_strings(value)))
|
|
85
|
+
return encoded
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def snapshot_payload(snapshot: SourceSnapshot) -> dict:
|
|
89
|
+
payload = json.loads(validate_source_value({
|
|
90
|
+
"ref": {"kind": snapshot.ref.kind, "owner_id": snapshot.ref.owner_id, "native_id": snapshot.ref.native_id},
|
|
91
|
+
"primary_node": snapshot.primary_node, "locator": snapshot.locator,
|
|
92
|
+
"metadata": snapshot.metadata, "nodes": [snapshot.primary_node, *snapshot.nodes],
|
|
93
|
+
"relations": list(snapshot.relations), "lifecycle": snapshot.lifecycle,
|
|
94
|
+
"source_updated_at": snapshot.source_updated_at, "scope": snapshot.scope,
|
|
95
|
+
}))
|
|
96
|
+
_require_snapshot_fields(snapshot)
|
|
97
|
+
_require_snapshot_graph(payload)
|
|
98
|
+
return payload
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _require_snapshot_fields(snapshot: SourceSnapshot) -> None:
|
|
102
|
+
if snapshot.ref.kind not in ("memory", "manager", "task") or snapshot.lifecycle not in ("active", "archived", "unlinked", "missing"):
|
|
103
|
+
raise DoyouknowError("invalid source kind or lifecycle")
|
|
104
|
+
for value in (snapshot.ref.owner_id, snapshot.ref.native_id, snapshot.locator):
|
|
105
|
+
if not isinstance(value, str) or not value.strip():
|
|
106
|
+
raise DoyouknowError("source identity and locator must be nonempty strings")
|
|
107
|
+
if snapshot.scope is not None and (not isinstance(snapshot.scope, str) or not snapshot.scope.strip()):
|
|
108
|
+
raise DoyouknowError("source scope must be a nonempty string or null")
|
|
109
|
+
if snapshot.source_updated_at is not None and not isinstance(snapshot.source_updated_at, str):
|
|
110
|
+
raise DoyouknowError("source update time must be a string or null")
|
|
111
|
+
if not isinstance(snapshot.metadata, dict):
|
|
112
|
+
raise DoyouknowError("source metadata must be an object")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _require_snapshot_graph(payload: dict) -> None:
|
|
116
|
+
schema = asset_object("doyouknow-v1.schema.json")
|
|
117
|
+
if len(payload["nodes"]) > 1000 or len(payload["relations"]) > 4000:
|
|
118
|
+
raise DoyouknowError("source graph exceeds input limits")
|
|
119
|
+
for node in payload["nodes"]:
|
|
120
|
+
serialize_owned_object(Path("source-node.json"), node, artifact="source node", schema={**schema, **schema["$defs"]["node"]})
|
|
121
|
+
refs = [node.get("ref", node["key"]) for node in payload["nodes"]]
|
|
122
|
+
identities = [(node["kind"], node["scope"], node["key"]) for node in payload["nodes"]]
|
|
123
|
+
if "$source" in refs or "$record" in refs or len(refs) != len(set(refs)) or len(identities) != len(set(identities)):
|
|
124
|
+
raise DoyouknowError("source node references and identities must be unique")
|
|
125
|
+
for relation in payload["relations"]:
|
|
126
|
+
serialize_owned_object(Path("source-relation.json"), relation, artifact="source relation", schema={**schema, **schema["$defs"]["relation"]})
|
|
127
|
+
if relation["from"] not in {"$source", *refs} or relation["to"] not in {"$source", *refs}:
|
|
128
|
+
raise DoyouknowError("source relation endpoint is not in this snapshot")
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from contextlib import closing
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .contracts import DoyouknowError, MAX_INPUT_BYTES, canonical_json
|
|
10
|
+
from .identity import node_id, source_id, stable_id
|
|
11
|
+
from .source_contracts import SourceRef, SourceScan, SourceSnapshot, snapshot_payload, validate_source_value
|
|
12
|
+
from .store import SCHEMA_VERSION, SQLITE_TIMEOUT_SECONDS, _digest, _insert_node, _require_database_location, database_path, write_connection
|
|
13
|
+
|
|
14
|
+
SOURCE_SCHEMA_STATEMENTS = (
|
|
15
|
+
"""create table if not exists source (
|
|
16
|
+
id text primary key, kind text not null, owner_id text not null,
|
|
17
|
+
native_id text not null, node_id text not null references node(id),
|
|
18
|
+
latest_snapshot text not null, scope text,
|
|
19
|
+
lifecycle text not null check(lifecycle in ('active', 'archived', 'unlinked', 'missing')),
|
|
20
|
+
checked_at text not null, attempted_at text not null,
|
|
21
|
+
scan_status text not null check(scan_status in ('complete', 'partial', 'unavailable')),
|
|
22
|
+
unique(kind, owner_id, native_id)
|
|
23
|
+
)""",
|
|
24
|
+
"""create table if not exists source_snapshot (
|
|
25
|
+
id text primary key, source_id text not null references source(id),
|
|
26
|
+
previous_snapshot text references source_snapshot(id),
|
|
27
|
+
content_digest text not null, payload text not null, created_at text not null
|
|
28
|
+
)""",
|
|
29
|
+
"""create table if not exists source_relation (
|
|
30
|
+
id text primary key, snapshot_id text not null references source_snapshot(id),
|
|
31
|
+
from_node text not null references node(id), to_node text not null references node(id),
|
|
32
|
+
kind text not null, reason text not null, evidence text not null,
|
|
33
|
+
standing text not null check(standing in ('observed', 'user-stated', 'candidate'))
|
|
34
|
+
)""",
|
|
35
|
+
"create index if not exists source_relation_snapshot on source_relation(snapshot_id)",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def store_source_snapshot(snapshot: SourceSnapshot, *, home: Path | None = None) -> dict:
|
|
40
|
+
payload = snapshot_payload(snapshot)
|
|
41
|
+
with write_connection(database_path(home)) as connection:
|
|
42
|
+
_write_source(connection, source_id(snapshot.ref), payload, datetime.now(timezone.utc).isoformat())
|
|
43
|
+
return {"ok": True, "sourceId": source_id(snapshot.ref)}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def apply_source_scan(scan: SourceScan, *, home: Path | None = None) -> dict:
|
|
47
|
+
payloads, refs = _validated_scan(scan)
|
|
48
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
49
|
+
with write_connection(database_path(home)) as connection:
|
|
50
|
+
for snapshot, payload in zip(scan.snapshots, payloads):
|
|
51
|
+
_write_source(connection, source_id(snapshot.ref), payload, now)
|
|
52
|
+
_mark_unrefreshed_sources(connection, scan, refs, now)
|
|
53
|
+
return {"ok": True, "status": scan.status, "registered": len(payloads), "errors": list(scan.errors)}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _validated_scan(scan: SourceScan) -> tuple[list[dict], list[SourceRef]]:
|
|
57
|
+
payloads = [snapshot_payload(snapshot) for snapshot in scan.snapshots]
|
|
58
|
+
refs = [*scan.observed_refs, *(snapshot.ref for snapshot in scan.snapshots)]
|
|
59
|
+
validate_source_value({"domain": [scan.domain.kind, scan.domain.owner_id], "refs": [[ref.kind, ref.owner_id, ref.native_id] for ref in refs], "errors": list(scan.errors)})
|
|
60
|
+
if scan.status not in ("complete", "partial", "unavailable"):
|
|
61
|
+
raise DoyouknowError("invalid source scan status")
|
|
62
|
+
if scan.status == "complete" and scan.errors:
|
|
63
|
+
raise DoyouknowError("a complete source scan cannot contain errors")
|
|
64
|
+
if scan.domain.kind not in ("memory", "manager", "task") or not scan.domain.owner_id.strip():
|
|
65
|
+
raise DoyouknowError("invalid source scan domain")
|
|
66
|
+
if any((ref.kind, ref.owner_id) != (scan.domain.kind, scan.domain.owner_id) for ref in refs):
|
|
67
|
+
raise DoyouknowError("source scan contains a different domain")
|
|
68
|
+
if any(not isinstance(ref.native_id, str) or not ref.native_id.strip() for ref in refs):
|
|
69
|
+
raise DoyouknowError("source identity must be a nonempty string")
|
|
70
|
+
if len({source_id(snapshot.ref) for snapshot in scan.snapshots}) != len(scan.snapshots):
|
|
71
|
+
raise DoyouknowError("source scan contains duplicate snapshots")
|
|
72
|
+
return payloads, refs
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _mark_unrefreshed_sources(connection: sqlite3.Connection, scan: SourceScan, refs: list[SourceRef], now: str) -> None:
|
|
76
|
+
seen = {source_id(ref) for ref in refs}
|
|
77
|
+
updated = {source_id(snapshot.ref) for snapshot in scan.snapshots}
|
|
78
|
+
rows = connection.execute("select id from source where kind = ? and owner_id = ?", (scan.domain.kind, scan.domain.owner_id)).fetchall()
|
|
79
|
+
for (identity,) in rows:
|
|
80
|
+
if identity in updated:
|
|
81
|
+
continue
|
|
82
|
+
if scan.status == "complete" and identity not in seen:
|
|
83
|
+
connection.execute("update source set lifecycle = 'missing', attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
|
|
84
|
+
else:
|
|
85
|
+
connection.execute("update source set attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _write_source(connection: sqlite3.Connection, identity: str, payload: dict, now: str) -> None:
|
|
89
|
+
previous = connection.execute("select source.latest_snapshot, source_snapshot.content_digest from source inner join source_snapshot on source.latest_snapshot = source_snapshot.id where source.id = ?", (identity,)).fetchone()
|
|
90
|
+
digest = _digest(payload)
|
|
91
|
+
if previous and previous[1] == digest:
|
|
92
|
+
connection.execute("update source set checked_at = ?, attempted_at = ?, scan_status = 'complete', lifecycle = ? where id = ?", (now, now, payload["lifecycle"], identity))
|
|
93
|
+
return
|
|
94
|
+
nodes = payload["nodes"]
|
|
95
|
+
for node in nodes:
|
|
96
|
+
_insert_node(connection, node_id(node), node["kind"], node["scope"], node["key"], node["title"])
|
|
97
|
+
snapshot_id = stable_id("source-snapshot", identity, previous[0] if previous else "", digest)
|
|
98
|
+
ref = payload["ref"]
|
|
99
|
+
connection.execute("insert into source (id, kind, owner_id, native_id, node_id, latest_snapshot, scope, lifecycle, checked_at, attempted_at, scan_status) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'complete') on conflict(id) do update set node_id=excluded.node_id, latest_snapshot=excluded.latest_snapshot, scope=excluded.scope, lifecycle=excluded.lifecycle, checked_at=excluded.checked_at, attempted_at=excluded.attempted_at, scan_status=excluded.scan_status", (identity, ref["kind"], ref["owner_id"], ref["native_id"], node_id(payload["primary_node"]), snapshot_id, payload["scope"], payload["lifecycle"], now, now))
|
|
100
|
+
connection.execute("insert into source_snapshot (id, source_id, previous_snapshot, content_digest, payload, created_at) values (?, ?, ?, ?, ?, ?)", (snapshot_id, identity, previous[0] if previous else None, digest, canonical_json(payload), now))
|
|
101
|
+
identities = {node.get("ref", node["key"]): node_id(node) for node in nodes}
|
|
102
|
+
identities["$source"] = node_id(payload["primary_node"])
|
|
103
|
+
for index, relation in enumerate(payload["relations"]):
|
|
104
|
+
connection.execute("insert into source_relation (id, snapshot_id, from_node, to_node, kind, reason, evidence, standing) values (?, ?, ?, ?, ?, ?, ?, ?)", (stable_id("source-relation", snapshot_id, str(index)), snapshot_id, identities[relation["from"]], identities[relation["to"]], relation["kind"], relation["reason"], canonical_json(relation["evidence"]), relation["standing"]))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def list_sources(*, home: Path | None = None, scope: str | None = None, kind: str | None = None, limit: int = 100, after: str | None = None) -> dict:
|
|
108
|
+
if not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= 1000:
|
|
109
|
+
raise DoyouknowError("source limit must be between 1 and 1000")
|
|
110
|
+
if after is not None and (not isinstance(after, str) or not after.strip()):
|
|
111
|
+
raise DoyouknowError("source cursor must be a nonempty string")
|
|
112
|
+
path = database_path(home)
|
|
113
|
+
_require_database_location(path)
|
|
114
|
+
if not path.is_file():
|
|
115
|
+
return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
|
|
116
|
+
try:
|
|
117
|
+
with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=SQLITE_TIMEOUT_SECONDS)) as connection:
|
|
118
|
+
connection.execute("begin")
|
|
119
|
+
version = connection.execute("pragma user_version").fetchone()[0]
|
|
120
|
+
if version == 1:
|
|
121
|
+
return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
|
|
122
|
+
if version != SCHEMA_VERSION:
|
|
123
|
+
raise DoyouknowError(f"unsupported doyouknow database version: {version}")
|
|
124
|
+
rows = connection.execute(
|
|
125
|
+
"""select source.id, source.node_id, source.latest_snapshot,
|
|
126
|
+
source.lifecycle, source.checked_at, source.attempted_at,
|
|
127
|
+
source.scan_status, source_snapshot.payload
|
|
128
|
+
from source inner join source_snapshot
|
|
129
|
+
on source.latest_snapshot = source_snapshot.id
|
|
130
|
+
where source.id > ? and (? is null or source.kind = ?)
|
|
131
|
+
and (? is null or source.scope = ?)
|
|
132
|
+
order by source.id limit ?""",
|
|
133
|
+
(after or "", kind, kind, scope, scope, limit + 1),
|
|
134
|
+
)
|
|
135
|
+
return _source_page(connection, rows, limit)
|
|
136
|
+
except sqlite3.Error as exc:
|
|
137
|
+
raise DoyouknowError(f"doyouknow source lookup failed: {exc}") from exc
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _source_page(connection: sqlite3.Connection, rows: sqlite3.Cursor, limit: int) -> dict:
|
|
141
|
+
result = {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
|
|
142
|
+
item_bytes = 0
|
|
143
|
+
for row in rows:
|
|
144
|
+
if len(result["sources"]) >= limit:
|
|
145
|
+
result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
|
|
146
|
+
break
|
|
147
|
+
view = _source_view(connection, row)
|
|
148
|
+
view_bytes = len(canonical_json(view).encode("utf-8"))
|
|
149
|
+
header = {"ok": True, "sources": [], "truncated": True, "nextCursor": view["sourceId"]}
|
|
150
|
+
header_bytes = len(canonical_json(header).encode("utf-8"))
|
|
151
|
+
if header_bytes + view_bytes > MAX_INPUT_BYTES:
|
|
152
|
+
view = {key: view[key] for key in ("sourceId", "nodeId", "snapshotId", "lifecycle", "checkedAt", "attemptedAt", "scanStatus")}
|
|
153
|
+
view["detailsOmitted"] = True
|
|
154
|
+
view_bytes = len(canonical_json(view).encode("utf-8"))
|
|
155
|
+
required = header_bytes + item_bytes + view_bytes
|
|
156
|
+
if required > MAX_INPUT_BYTES:
|
|
157
|
+
result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
|
|
158
|
+
break
|
|
159
|
+
result["sources"].append(view)
|
|
160
|
+
item_bytes += view_bytes + 2
|
|
161
|
+
return result
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _source_view(connection: sqlite3.Connection, row: tuple) -> dict:
|
|
165
|
+
payload = json.loads(row[7])
|
|
166
|
+
relations = connection.execute("select from_node, to_node, kind, reason, evidence, standing from source_relation where snapshot_id = ? order by rowid", (row[2],)).fetchall()
|
|
167
|
+
return {"sourceId": row[0], "nodeId": row[1], "snapshotId": row[2],
|
|
168
|
+
"lifecycle": row[3], "checkedAt": row[4], "attemptedAt": row[5], "scanStatus": row[6],
|
|
169
|
+
"ref": payload["ref"], "scope": payload["scope"], "locator": payload["locator"],
|
|
170
|
+
"sourceUpdatedAt": payload["source_updated_at"], "metadata": payload["metadata"],
|
|
171
|
+
"nodes": [{"id": node_id(node), **node} for node in payload["nodes"]],
|
|
172
|
+
"relations": [{"fromNodeId": relation[0], "toNodeId": relation[1], "kind": relation[2], "reason": relation[3], "evidence": json.loads(relation[4]), "standing": relation[5]} for relation in relations]}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from okstra_project.dirs import contained_relative_path
|
|
6
|
+
|
|
7
|
+
from ...json_boundary import JsonBoundaryError, load_owned_object, read_bounded_bytes
|
|
8
|
+
from ..contracts import DoyouknowError, MAX_INPUT_BYTES
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def safe_source_path(root: Path, relative: str | Path) -> Path:
|
|
12
|
+
base = root.resolve()
|
|
13
|
+
contained = contained_relative_path(str(relative), base)
|
|
14
|
+
if contained is None:
|
|
15
|
+
raise DoyouknowError("source path is outside its registered root")
|
|
16
|
+
return base / contained
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def read_source_text(root: Path, relative: str | Path) -> str:
|
|
20
|
+
path = safe_source_path(root, relative)
|
|
21
|
+
if not path.is_file():
|
|
22
|
+
raise DoyouknowError("source text is not a regular file")
|
|
23
|
+
try:
|
|
24
|
+
return read_bounded_bytes(path, max_bytes=MAX_INPUT_BYTES, artifact="source text").decode("utf-8")
|
|
25
|
+
except (OSError, UnicodeError, JsonBoundaryError) as exc:
|
|
26
|
+
raise DoyouknowError("source text is unreadable or exceeds the input limit") from exc
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def read_source_object(root: Path, relative: str | Path) -> dict:
|
|
30
|
+
path = safe_source_path(root, relative)
|
|
31
|
+
if not path.is_file():
|
|
32
|
+
raise DoyouknowError("source object is not a regular file")
|
|
33
|
+
try:
|
|
34
|
+
return load_owned_object(path, artifact="source object", max_bytes=MAX_INPUT_BYTES)
|
|
35
|
+
except (OSError, JsonBoundaryError, RecursionError) as exc:
|
|
36
|
+
raise DoyouknowError("source object is unreadable or exceeds the input limit") from exc
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from dataclasses import replace
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from stat import S_ISDIR
|
|
9
|
+
|
|
10
|
+
from okstra_project import StateError, parse_task_key
|
|
11
|
+
|
|
12
|
+
from ...json_boundary import JsonBoundaryError
|
|
13
|
+
from ...jsonl import parse_jsonl_snapshot
|
|
14
|
+
from ...manager_paths import (
|
|
15
|
+
ManagerPathError, canonical_manager_id, children_json_path,
|
|
16
|
+
directives_jsonl_path, manager_json_path, projects_json_path,
|
|
17
|
+
manager_root, snapshots_json_path, task_groups_root, task_manifest_path,
|
|
18
|
+
)
|
|
19
|
+
from ..config import SourceConfig, includes_project
|
|
20
|
+
from ..contracts import DoyouknowError, Node, Relation, SecretRefusal, canonical_json, record_secret_policy_digest
|
|
21
|
+
from ..source_contracts import PreparedSource, SourceDomain, SourceRef, SourceScan, SourceSnapshot, prepared_snapshot, validate_prepared_snapshot
|
|
22
|
+
from .io import read_source_object, read_source_text, safe_source_path
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _optional_object(home: Path, path: Path, default: dict) -> dict:
|
|
26
|
+
target = safe_source_path(home, path)
|
|
27
|
+
return read_source_object(home, target) if target.exists() else default
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _manager_manifest_candidates(home: Path, root: Path) -> tuple[Path, ...]:
|
|
31
|
+
groups = safe_source_path(home, task_groups_root(home, root.name))
|
|
32
|
+
try:
|
|
33
|
+
groups.stat()
|
|
34
|
+
except FileNotFoundError:
|
|
35
|
+
return ()
|
|
36
|
+
candidates = []
|
|
37
|
+
for group in sorted(groups.iterdir()):
|
|
38
|
+
if not S_ISDIR(safe_source_path(home, group).stat().st_mode):
|
|
39
|
+
continue
|
|
40
|
+
for task in sorted(group.iterdir()):
|
|
41
|
+
if S_ISDIR(safe_source_path(home, task).stat().st_mode):
|
|
42
|
+
candidates.append(task / "manifest.json")
|
|
43
|
+
return tuple(candidates)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _project_rows(rows: object, settings: SourceConfig) -> list[dict]:
|
|
47
|
+
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
|
48
|
+
raise DoyouknowError("manager project rows must be objects")
|
|
49
|
+
return [row for row in rows if not row.get("projectId") or includes_project(settings, row["projectId"])]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _manager_projects(home: Path, root: Path, settings: SourceConfig) -> list[dict]:
|
|
53
|
+
payload = _optional_object(home, projects_json_path(home, root.name), {"projects": []})
|
|
54
|
+
rows = _project_rows(payload.get("projects"), settings)
|
|
55
|
+
for row in rows:
|
|
56
|
+
if not isinstance(row.get("projectId"), str) or not row["projectId"] or not isinstance(row.get("projectRoot"), str) or not row["projectRoot"]:
|
|
57
|
+
raise DoyouknowError("manager project registration is invalid")
|
|
58
|
+
return rows
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _root_snapshot(home: Path, manager: dict, projects: list[dict], settings: SourceConfig) -> SourceSnapshot:
|
|
62
|
+
manager_id = manager["managerId"]
|
|
63
|
+
nodes = tuple({"kind": "project", "scope": "okstra-project", "key": row["projectId"], "ref": f"project-{index}", "title": row["projectId"]} for index, row in enumerate(projects))
|
|
64
|
+
relations = tuple({"from": "$source", "to": node["ref"], "kind": "includes-project", "reason": "The manager registers this project.", "evidence": [str(projects_json_path(home, manager_id))], "standing": "observed"} for node in nodes)
|
|
65
|
+
snapshot = SourceSnapshot(
|
|
66
|
+
SourceRef("manager", manager_id, "$manager"),
|
|
67
|
+
{"kind": "entity", "scope": "okstra-manager", "key": manager_id, "ref": "primary", "title": manager_id},
|
|
68
|
+
str(manager_json_path(home, manager_id)), {"manager": manager, "projects": projects}, nodes, relations,
|
|
69
|
+
source_updated_at=manager.get("updatedAt"), scope=settings["scope"],
|
|
70
|
+
)
|
|
71
|
+
return snapshot
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _task_records(home: Path, manager_id: str, group: str, task_id: str, settings: SourceConfig) -> tuple[list[dict], list[dict], dict]:
|
|
75
|
+
children = _project_rows(read_source_object(home, children_json_path(home, manager_id, group, task_id)).get("children"), settings)
|
|
76
|
+
saved = _optional_object(home, snapshots_json_path(home, manager_id, group, task_id), {})
|
|
77
|
+
saved = {**saved, "children": _project_rows(saved.get("children", []), settings)}
|
|
78
|
+
directives_path = safe_source_path(home, directives_jsonl_path(home, manager_id, group, task_id))
|
|
79
|
+
directives, damaged = parse_jsonl_snapshot(read_source_text(home, directives_path)) if directives_path.exists() else ([], [])
|
|
80
|
+
if damaged:
|
|
81
|
+
raise DoyouknowError("manager notes contain damaged rows")
|
|
82
|
+
return children, _project_rows(directives, settings), saved
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _child_graph(children: list[dict], linked: set[str], evidence: Path) -> tuple[list[Node], list[Relation], list[dict]]:
|
|
86
|
+
nodes: list[Node] = []
|
|
87
|
+
relations: list[Relation] = []
|
|
88
|
+
kept_children = []
|
|
89
|
+
for index, child in enumerate(children):
|
|
90
|
+
key, project_id = child.get("taskKey"), child.get("projectId")
|
|
91
|
+
if not isinstance(key, str) or not isinstance(project_id, str) or parse_task_key(key) != (project_id, child.get("taskGroup"), child.get("taskId")):
|
|
92
|
+
raise DoyouknowError("manager child identity is invalid")
|
|
93
|
+
ref = f"child-{index}"
|
|
94
|
+
nodes.append({"kind": "task", "scope": project_id, "key": key, "ref": ref, "title": child["taskId"]})
|
|
95
|
+
relations.append({"from": "$source", "to": ref, "kind": "assigns-task" if project_id in linked else "references-unlinked-task", "reason": "The manager lists this child task.", "evidence": [str(evidence)], "standing": "observed"})
|
|
96
|
+
kept_children.append({**child, "projectLinked": project_id in linked})
|
|
97
|
+
return nodes, relations, kept_children
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _task_snapshot(home: Path, path: Path, manifest: dict, ref: SourceRef, projects: list[dict], settings: SourceConfig) -> SourceSnapshot:
|
|
101
|
+
group, task_id = manifest["taskGroup"], manifest["taskId"]
|
|
102
|
+
manager_id = ref.owner_id
|
|
103
|
+
if manifest.get("managerId") != manager_id or path.resolve() != task_manifest_path(home, manager_id, group, task_id).resolve():
|
|
104
|
+
raise DoyouknowError("manager task identity does not match its source")
|
|
105
|
+
children, directives, saved = _task_records(home, manager_id, group, task_id, settings)
|
|
106
|
+
child_nodes, child_relations, kept_children = _child_graph(children, {row["projectId"] for row in projects}, children_json_path(home, manager_id, group, task_id))
|
|
107
|
+
nodes = ({"kind": "entity", "scope": "okstra-manager", "key": manager_id, "ref": "manager", "title": manager_id}, *child_nodes)
|
|
108
|
+
relations = ({"from": "$source", "to": "manager", "kind": "belongs-to", "reason": "The manager owns this task.", "evidence": [str(path)], "standing": "observed"}, *child_relations)
|
|
109
|
+
snapshot = SourceSnapshot(
|
|
110
|
+
ref,
|
|
111
|
+
{"kind": "task", "scope": "manager:" + manager_id, "key": ref.native_id, "ref": "primary", "title": task_id},
|
|
112
|
+
str(path), {"manifest": manifest, "children": kept_children, "notes": directives, "snapshot": saved},
|
|
113
|
+
nodes, relations, source_updated_at=manifest.get("updatedAt"), scope=settings["scope"],
|
|
114
|
+
)
|
|
115
|
+
return snapshot
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _manager_manifest_data(home: Path, path: Path) -> dict:
|
|
119
|
+
manifest = read_source_object(home, path)
|
|
120
|
+
group, task_id = manifest.get("taskGroup"), manifest.get("taskId")
|
|
121
|
+
if not isinstance(group, str) or not group or not isinstance(task_id, str) or not task_id:
|
|
122
|
+
raise DoyouknowError("manager task identity is missing")
|
|
123
|
+
return manifest
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _validate_manager_snapshot(snapshot: SourceSnapshot, gate: Callable[[PreparedSource], bool] | None) -> None:
|
|
127
|
+
prepared = prepared_snapshot(snapshot)
|
|
128
|
+
if gate is not None:
|
|
129
|
+
if not gate(prepared):
|
|
130
|
+
raise SecretRefusal("manager source refused")
|
|
131
|
+
else:
|
|
132
|
+
validate_prepared_snapshot(prepared)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _manager_task_source(home: Path, path: Path, manager_id: str, projects: list[dict], settings: SourceConfig) -> tuple[SourceRef | None, SourceSnapshot | None, str | None]:
|
|
136
|
+
ref = None
|
|
137
|
+
try:
|
|
138
|
+
manifest = _manager_manifest_data(home, path)
|
|
139
|
+
ref = SourceRef("manager", manager_id, "task:" + canonical_json([manifest["taskGroup"], manifest["taskId"]]))
|
|
140
|
+
snapshot = _task_snapshot(home, path, manifest, ref, projects, settings)
|
|
141
|
+
return ref, snapshot, None
|
|
142
|
+
except (DoyouknowError, JsonBoundaryError, OSError, StateError, TypeError, RecursionError):
|
|
143
|
+
return ref, None, "manager-task-unavailable"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _scan_manager(home: Path, root: Path, settings: SourceConfig, *, validate: bool = True, gate: Callable[[PreparedSource], bool] | None = None) -> tuple[SourceScan, list[dict]]:
|
|
147
|
+
manager_id = root.name
|
|
148
|
+
domain = SourceDomain("manager", manager_id)
|
|
149
|
+
snapshots, observed, errors = [], [], []
|
|
150
|
+
try:
|
|
151
|
+
canonical_manager_id(manager_id)
|
|
152
|
+
manager = read_source_object(home, manager_json_path(home, manager_id))
|
|
153
|
+
if manager.get("managerId") != manager_id:
|
|
154
|
+
raise DoyouknowError("manager identity does not match its source")
|
|
155
|
+
projects = _manager_projects(home, root, settings)
|
|
156
|
+
root_snapshot = _root_snapshot(home, manager, projects, settings)
|
|
157
|
+
snapshots.append(root_snapshot)
|
|
158
|
+
observed.append(SourceRef("manager", manager_id, "$manager"))
|
|
159
|
+
for path in _manager_manifest_candidates(home, root):
|
|
160
|
+
ref, snapshot, error = _manager_task_source(home, path, manager_id, projects, settings)
|
|
161
|
+
if ref is not None:
|
|
162
|
+
observed.append(ref)
|
|
163
|
+
if snapshot is not None:
|
|
164
|
+
snapshots.append(snapshot)
|
|
165
|
+
else:
|
|
166
|
+
errors.append(error)
|
|
167
|
+
except (DoyouknowError, JsonBoundaryError, ManagerPathError, OSError, StateError, TypeError, RecursionError):
|
|
168
|
+
return SourceScan(domain, (), (), "unavailable", ("manager-source-unavailable",)), []
|
|
169
|
+
scan = SourceScan(domain, tuple(snapshots), tuple(observed), "partial" if errors else "complete", tuple(errors))
|
|
170
|
+
checked = gate_manager_scan(scan, gate=gate) if validate else scan
|
|
171
|
+
return checked, [] if checked.status == "unavailable" else projects
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def gate_manager_scan(scan: SourceScan, *, gate: Callable[[PreparedSource], bool] | None = None) -> SourceScan:
|
|
175
|
+
if scan.status == "unavailable":
|
|
176
|
+
return scan
|
|
177
|
+
root = next((snapshot for snapshot in scan.snapshots if snapshot.ref.native_id == "$manager"), None)
|
|
178
|
+
try:
|
|
179
|
+
if root is None:
|
|
180
|
+
raise DoyouknowError("manager root is unavailable")
|
|
181
|
+
_validate_manager_snapshot(root, gate)
|
|
182
|
+
except SecretRefusal:
|
|
183
|
+
return replace(scan, snapshots=(), status="unavailable", errors=("manager-secret-refused",))
|
|
184
|
+
except (DoyouknowError, JsonBoundaryError, OSError, StateError, TypeError, RecursionError):
|
|
185
|
+
return replace(scan, snapshots=(), status="unavailable", errors=("manager-source-unavailable",))
|
|
186
|
+
snapshots = [root]
|
|
187
|
+
errors = list(scan.errors)
|
|
188
|
+
for snapshot in scan.snapshots:
|
|
189
|
+
if snapshot is root:
|
|
190
|
+
continue
|
|
191
|
+
try:
|
|
192
|
+
_validate_manager_snapshot(snapshot, gate)
|
|
193
|
+
snapshots.append(snapshot)
|
|
194
|
+
except SecretRefusal:
|
|
195
|
+
errors.append("manager-secret-refused")
|
|
196
|
+
except (DoyouknowError, JsonBoundaryError, OSError, StateError, TypeError, RecursionError):
|
|
197
|
+
errors.append("manager-task-unavailable")
|
|
198
|
+
return replace(scan, snapshots=tuple(snapshots), status="partial" if errors else scan.status, errors=tuple(errors))
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _manager_source_paths(home: Path, root: Path) -> list[tuple[Path, bool]]:
|
|
202
|
+
manager_id = root.name
|
|
203
|
+
paths = [(manager_json_path(home, manager_id), True), (projects_json_path(home, manager_id), False)]
|
|
204
|
+
for path in _manager_manifest_candidates(home, root):
|
|
205
|
+
manifest = _manager_manifest_data(home, path)
|
|
206
|
+
group, task_id = manifest["taskGroup"], manifest["taskId"]
|
|
207
|
+
paths.extend((
|
|
208
|
+
(path, True),
|
|
209
|
+
(children_json_path(home, manager_id, group, task_id), True),
|
|
210
|
+
(snapshots_json_path(home, manager_id, group, task_id), False),
|
|
211
|
+
(directives_jsonl_path(home, manager_id, group, task_id), False),
|
|
212
|
+
))
|
|
213
|
+
return sorted(paths, key=lambda item: str(item[0]))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _manager_source_fingerprint(home: Path, root: Path, settings: SourceConfig) -> str:
|
|
217
|
+
projection = {key: settings[key] for key in ("scope", "includeProjects", "excludeProjects")}
|
|
218
|
+
digest = hashlib.sha256(canonical_json([projection, record_secret_policy_digest()]).encode("utf-8"))
|
|
219
|
+
for path, required in _manager_source_paths(home, root):
|
|
220
|
+
target = safe_source_path(home, path)
|
|
221
|
+
body_digest = None
|
|
222
|
+
if required or target.exists():
|
|
223
|
+
body_digest = hashlib.sha256(read_source_text(home, target).encode("utf-8")).hexdigest()
|
|
224
|
+
digest.update(canonical_json([str(path.relative_to(root)), body_digest]).encode("utf-8"))
|
|
225
|
+
return digest.hexdigest()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def prepare_manager_source(home: Path, manager_id: str, settings: SourceConfig) -> PreparedSource:
|
|
229
|
+
ref = SourceRef("manager", manager_id, "$manager")
|
|
230
|
+
if "manager" not in settings["sources"]:
|
|
231
|
+
return PreparedSource(ref, None, outcome="excluded")
|
|
232
|
+
try:
|
|
233
|
+
root = manager_root(home, manager_id)
|
|
234
|
+
before = _manager_source_fingerprint(home, root, settings)
|
|
235
|
+
scan, _ = _scan_manager(home, root, settings, validate=False)
|
|
236
|
+
after = _manager_source_fingerprint(home, root, settings)
|
|
237
|
+
if scan.status == "unavailable" or before != after:
|
|
238
|
+
return PreparedSource(ref, None, outcome="unavailable", error_code="manager-source-unavailable")
|
|
239
|
+
return PreparedSource(ref, before, scan=scan)
|
|
240
|
+
except (DoyouknowError, JsonBoundaryError, ManagerPathError, OSError, StateError, TypeError, RecursionError):
|
|
241
|
+
return PreparedSource(ref, None, outcome="unavailable", error_code="manager-source-unavailable")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def scan_managers(home: Path, settings: SourceConfig, *, gate: Callable[[PreparedSource], bool] | None = None) -> tuple[tuple[SourceScan, ...], dict[str, tuple[Path, ...]]]:
|
|
245
|
+
scans, projects = [], defaultdict(set)
|
|
246
|
+
if not {"manager", "task"}.intersection(settings["sources"]):
|
|
247
|
+
return (), {}
|
|
248
|
+
try:
|
|
249
|
+
base = safe_source_path(home, "managers")
|
|
250
|
+
roots = sorted(path for path in base.iterdir() if path.is_dir()) if base.exists() else []
|
|
251
|
+
for root in roots:
|
|
252
|
+
if "manager" in settings["sources"]:
|
|
253
|
+
scan, entries = _scan_manager(home, root, settings, gate=gate)
|
|
254
|
+
scans.append(scan)
|
|
255
|
+
else:
|
|
256
|
+
entries = _manager_projects(home, root, settings)
|
|
257
|
+
for entry in entries:
|
|
258
|
+
projects[entry["projectId"]].add(Path(entry["projectRoot"]).resolve())
|
|
259
|
+
except (DoyouknowError, JsonBoundaryError, OSError, TypeError, RecursionError):
|
|
260
|
+
scans.append(SourceScan(SourceDomain("manager", "$discovery"), (), (), "unavailable", ("manager-discovery-unavailable",)))
|
|
261
|
+
return tuple(scans), {key: tuple(sorted(paths)) for key, paths in projects.items()}
|