okstra 0.231.1 → 0.232.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/README.md +12 -0
  2. package/dist/cli-registry.mjs +6 -0
  3. package/dist/cli-registry.mjs.map +1 -1
  4. package/dist/commands/memory/memory.mjs +9 -55
  5. package/dist/commands/memory/memory.mjs.map +1 -1
  6. package/dist/lib/memory-auto-register.d.mts +1 -0
  7. package/dist/lib/memory-auto-register.mjs +113 -0
  8. package/dist/lib/memory-auto-register.mjs.map +1 -0
  9. package/dist/lib/memory-entry-contract.d.mts +12 -0
  10. package/dist/lib/memory-entry-contract.mjs +46 -0
  11. package/dist/lib/memory-entry-contract.mjs.map +1 -0
  12. package/dist/lib/python-helper.d.mts +1 -1
  13. package/dist/lib/python-helper.mjs +3 -2
  14. package/dist/lib/python-helper.mjs.map +1 -1
  15. package/dist/lib/record-secrets.d.mts +1 -0
  16. package/dist/lib/record-secrets.mjs +28 -0
  17. package/dist/lib/record-secrets.mjs.map +1 -0
  18. package/dist/lib/types.d.mts +2 -1
  19. package/docs/architecture/storage-model.md +127 -0
  20. package/docs/architecture.md +7 -0
  21. package/docs/cli.md +234 -0
  22. package/docs/project-structure-overview.md +2 -1
  23. package/package.json +1 -1
  24. package/runtime/BUILD.json +2 -2
  25. package/runtime/agents/common.json +8 -8
  26. package/runtime/agents/roles/verifier.json +2 -4
  27. package/runtime/bin/okstra-provider-exec.py +10 -4
  28. package/runtime/prompts/coding-preflight/architectures/hexagonal.md +23 -30
  29. package/runtime/prompts/coding-preflight/authoring-checks.md +17 -0
  30. package/runtime/prompts/coding-preflight/clean-code.md +85 -281
  31. package/runtime/prompts/coding-preflight/frameworks/node-server.md +2 -3
  32. package/runtime/prompts/coding-preflight/languages/java.md +3 -5
  33. package/runtime/prompts/coding-preflight/languages/javascript-typescript.md +5 -12
  34. package/runtime/prompts/coding-preflight/languages/kotlin.md +2 -4
  35. package/runtime/prompts/coding-preflight/languages/python.md +2 -7
  36. package/runtime/prompts/coding-preflight/languages/rust.md +2 -8
  37. package/runtime/prompts/coding-preflight/languages/sql.md +2 -4
  38. package/runtime/prompts/coding-preflight/overview.md +25 -38
  39. package/runtime/prompts/coding-preflight/review-rules.json +30 -0
  40. package/runtime/prompts/duties/code-reviewer.json +15 -18
  41. package/runtime/prompts/duties/implementation-verifier.json +1 -0
  42. package/runtime/prompts/profiles/_coding-conventions-preflight.md +41 -6
  43. package/runtime/python/okstra_ctl/agent/invocation.py +121 -36
  44. package/runtime/python/okstra_ctl/agent/prompt_cli/cli.py +6 -0
  45. package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +30 -4
  46. package/runtime/python/okstra_ctl/agent/standalone.py +3 -0
  47. package/runtime/python/okstra_ctl/analysis_packet.py +2 -19
  48. package/runtime/python/okstra_ctl/attempt_evidence.py +10 -0
  49. package/runtime/python/okstra_ctl/code_review_census.py +71 -7
  50. package/runtime/python/okstra_ctl/code_review_coverage.py +4 -274
  51. package/runtime/python/okstra_ctl/code_review_integrity.py +154 -0
  52. package/runtime/python/okstra_ctl/code_review_invocation.py +212 -0
  53. package/runtime/python/okstra_ctl/code_review_scope.py +36 -9
  54. package/runtime/python/okstra_ctl/code_review_target.py +85 -11
  55. package/runtime/python/okstra_ctl/consumers.py +12 -9
  56. package/runtime/python/okstra_ctl/design_review.py +207 -0
  57. package/runtime/python/okstra_ctl/design_review_evidence.py +297 -0
  58. package/runtime/python/okstra_ctl/doctor.py +8 -25
  59. package/runtime/python/okstra_ctl/doyouknow/__init__.py +9 -0
  60. package/runtime/python/okstra_ctl/doyouknow/cli.py +91 -0
  61. package/runtime/python/okstra_ctl/doyouknow/config.py +58 -0
  62. package/runtime/python/okstra_ctl/doyouknow/contracts.py +184 -0
  63. package/runtime/python/okstra_ctl/doyouknow/identity.py +22 -0
  64. package/runtime/python/okstra_ctl/doyouknow/paths.py +22 -0
  65. package/runtime/python/okstra_ctl/doyouknow/pending.py +72 -0
  66. package/runtime/python/okstra_ctl/doyouknow/registration.py +260 -0
  67. package/runtime/python/okstra_ctl/doyouknow/source_contracts.py +128 -0
  68. package/runtime/python/okstra_ctl/doyouknow/source_store.py +172 -0
  69. package/runtime/python/okstra_ctl/doyouknow/sources/io.py +36 -0
  70. package/runtime/python/okstra_ctl/doyouknow/sources/manager.py +261 -0
  71. package/runtime/python/okstra_ctl/doyouknow/sources/memory.py +179 -0
  72. package/runtime/python/okstra_ctl/doyouknow/sources/task.py +95 -0
  73. package/runtime/python/okstra_ctl/doyouknow/store.py +260 -0
  74. package/runtime/python/okstra_ctl/doyouknow/sync.py +58 -0
  75. package/runtime/python/okstra_ctl/execution_mutation_audit.py +18 -2
  76. package/runtime/python/okstra_ctl/file_size_cap.py +3 -4
  77. package/runtime/python/okstra_ctl/json_boundary.py +21 -2
  78. package/runtime/python/okstra_ctl/jsonl.py +21 -0
  79. package/runtime/python/okstra_ctl/manager_cli.py +26 -0
  80. package/runtime/python/okstra_ctl/manager_registration.py +29 -0
  81. package/runtime/python/okstra_ctl/manager_sync.py +3 -22
  82. package/runtime/python/okstra_ctl/model_io/renderers.py +13 -2
  83. package/runtime/python/okstra_ctl/model_io_cli.py +7 -1
  84. package/runtime/python/okstra_ctl/phases/final_verification/validation.py +20 -0
  85. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +10 -0
  86. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-diff-review.md +19 -6
  87. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-self-check.md +5 -3
  88. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +38 -10
  89. package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation.template.html +13 -0
  90. package/runtime/python/okstra_ctl/phases/implementation/spec.md +25 -0
  91. package/runtime/python/okstra_ctl/phases/implementation/validation.py +6 -0
  92. package/runtime/python/okstra_ctl/phases/implementation_planning/authoring.py +4 -0
  93. package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +1 -1
  94. package/runtime/python/okstra_ctl/report_assembly.py +15 -0
  95. package/runtime/python/okstra_ctl/report_narrative.py +3 -0
  96. package/runtime/python/okstra_ctl/report_synthesis_packet.py +11 -2
  97. package/runtime/python/okstra_ctl/review_policy.py +359 -0
  98. package/runtime/python/okstra_ctl/review_verdicts.py +291 -0
  99. package/runtime/python/okstra_ctl/run.py +24 -1
  100. package/runtime/python/okstra_ctl/stage_done.py +11 -0
  101. package/runtime/python/okstra_ctl/standalone_review_audit.py +28 -0
  102. package/runtime/python/okstra_ctl/verification_generation.py +23 -1
  103. package/runtime/python/okstra_ctl/verification_input_types.py +15 -1
  104. package/runtime/python/okstra_ctl/verification_inputs.py +28 -6
  105. package/runtime/python/okstra_ctl/verification_invocation.py +7 -1
  106. package/runtime/python/okstra_ctl/verification_target.py +2 -0
  107. package/runtime/python/okstra_ctl/worker_prompt_headers.py +24 -9
  108. package/runtime/python/okstra_ctl/worker_runner.py +66 -10
  109. package/runtime/python/okstra_ctl/write_policy.py +34 -1
  110. package/runtime/python/okstra_project/__init__.py +2 -0
  111. package/runtime/python/okstra_project/dirs.py +18 -0
  112. package/runtime/python/okstra_project/state.py +25 -20
  113. package/runtime/schemas/doyouknow-config-v1.schema.json +20 -0
  114. package/runtime/schemas/doyouknow-v1.schema.json +65 -0
  115. package/runtime/schemas/final-report-v2.0.schema.json +157 -0
  116. package/runtime/schemas/final-report-v3.0.schema.json +157 -0
  117. package/runtime/schemas/memory-entry-contract-v1.json +11 -0
  118. package/runtime/schemas/record-secret-patterns.json +41 -0
  119. package/runtime/skills/okstra-code-review/SKILL.md +71 -123
  120. package/runtime/skills/okstra-code-review/references/census-rules.md +29 -105
  121. package/runtime/skills/okstra-code-review/references/review-calibration.md +34 -54
  122. package/runtime/skills/okstra-manager/SKILL.md +7 -0
  123. package/runtime/skills/okstra-memory/SKILL.md +7 -0
  124. package/runtime/templates/implementation-worker-preamble.md +4 -9
  125. package/runtime/templates/worker-prompt-preamble.md +4 -12
  126. package/runtime/validators/validate-run.py +14 -2
@@ -0,0 +1,128 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from dataclasses import asdict, dataclass, field
5
+ from pathlib import Path
6
+ from typing import Literal
7
+
8
+ from ..json_boundary import serialize_owned_object
9
+ from .contracts import (DoyouknowError, MAX_INPUT_BYTES, Node, Relation, _check_input_depth,
10
+ _decoded_strings, asset_object, canonical_json, record_secret_policy_digest, reject_secrets)
11
+ from .identity import stable_id
12
+
13
+ SourceKind = Literal["memory", "manager", "task"]
14
+ Lifecycle = Literal["active", "archived", "unlinked", "missing"]
15
+ JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class SourceRef:
20
+ kind: SourceKind
21
+ owner_id: str
22
+ native_id: str
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class SourceDomain:
27
+ kind: SourceKind
28
+ owner_id: str
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class SourceSnapshot:
33
+ ref: SourceRef
34
+ primary_node: Node
35
+ locator: str
36
+ metadata: dict[str, JsonValue] = field(default_factory=dict)
37
+ nodes: tuple[Node, ...] = ()
38
+ relations: tuple[Relation, ...] = ()
39
+ lifecycle: Lifecycle = "active"
40
+ source_updated_at: str | None = None
41
+ scope: str | None = None
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class SourceScan:
46
+ domain: SourceDomain
47
+ snapshots: tuple[SourceSnapshot, ...]
48
+ observed_refs: tuple[SourceRef, ...]
49
+ status: Literal["complete", "partial", "unavailable"]
50
+ errors: tuple[str, ...] = ()
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class PreparedSource:
55
+ ref: SourceRef
56
+ fingerprint: str | None
57
+ snapshot: SourceSnapshot | None = field(default=None, repr=False)
58
+ outcome: Literal["ready", "excluded", "unavailable"] = "ready"
59
+ secret_texts: tuple[str, ...] = field(default=(), repr=False)
60
+ scan: SourceScan | None = field(default=None, repr=False)
61
+ error_code: str | None = None
62
+
63
+
64
+ def prepared_snapshot(snapshot: SourceSnapshot) -> PreparedSource:
65
+ return PreparedSource(snapshot.ref, stable_id("source-content", canonical_json(asdict(snapshot)), record_secret_policy_digest()), snapshot)
66
+
67
+
68
+ def validate_prepared_snapshot(prepared: PreparedSource) -> None:
69
+ if prepared.outcome != "ready" or prepared.snapshot is None:
70
+ raise DoyouknowError("source snapshot unavailable")
71
+ for text in prepared.secret_texts:
72
+ reject_secrets(text)
73
+ snapshot_payload(prepared.snapshot)
74
+
75
+
76
+ def validate_source_value(value: object) -> str:
77
+ _check_input_depth(value)
78
+ try:
79
+ encoded = canonical_json(value)
80
+ except (ValueError, TypeError) as exc:
81
+ raise DoyouknowError("source input is not a JSON value") from exc
82
+ if len(encoded.encode("utf-8")) > MAX_INPUT_BYTES:
83
+ raise DoyouknowError("source input exceeds 2000000 bytes")
84
+ reject_secrets("\n".join(_decoded_strings(value)))
85
+ return encoded
86
+
87
+
88
+ def snapshot_payload(snapshot: SourceSnapshot) -> dict:
89
+ payload = json.loads(validate_source_value({
90
+ "ref": {"kind": snapshot.ref.kind, "owner_id": snapshot.ref.owner_id, "native_id": snapshot.ref.native_id},
91
+ "primary_node": snapshot.primary_node, "locator": snapshot.locator,
92
+ "metadata": snapshot.metadata, "nodes": [snapshot.primary_node, *snapshot.nodes],
93
+ "relations": list(snapshot.relations), "lifecycle": snapshot.lifecycle,
94
+ "source_updated_at": snapshot.source_updated_at, "scope": snapshot.scope,
95
+ }))
96
+ _require_snapshot_fields(snapshot)
97
+ _require_snapshot_graph(payload)
98
+ return payload
99
+
100
+
101
+ def _require_snapshot_fields(snapshot: SourceSnapshot) -> None:
102
+ if snapshot.ref.kind not in ("memory", "manager", "task") or snapshot.lifecycle not in ("active", "archived", "unlinked", "missing"):
103
+ raise DoyouknowError("invalid source kind or lifecycle")
104
+ for value in (snapshot.ref.owner_id, snapshot.ref.native_id, snapshot.locator):
105
+ if not isinstance(value, str) or not value.strip():
106
+ raise DoyouknowError("source identity and locator must be nonempty strings")
107
+ if snapshot.scope is not None and (not isinstance(snapshot.scope, str) or not snapshot.scope.strip()):
108
+ raise DoyouknowError("source scope must be a nonempty string or null")
109
+ if snapshot.source_updated_at is not None and not isinstance(snapshot.source_updated_at, str):
110
+ raise DoyouknowError("source update time must be a string or null")
111
+ if not isinstance(snapshot.metadata, dict):
112
+ raise DoyouknowError("source metadata must be an object")
113
+
114
+
115
+ def _require_snapshot_graph(payload: dict) -> None:
116
+ schema = asset_object("doyouknow-v1.schema.json")
117
+ if len(payload["nodes"]) > 1000 or len(payload["relations"]) > 4000:
118
+ raise DoyouknowError("source graph exceeds input limits")
119
+ for node in payload["nodes"]:
120
+ serialize_owned_object(Path("source-node.json"), node, artifact="source node", schema={**schema, **schema["$defs"]["node"]})
121
+ refs = [node.get("ref", node["key"]) for node in payload["nodes"]]
122
+ identities = [(node["kind"], node["scope"], node["key"]) for node in payload["nodes"]]
123
+ if "$source" in refs or "$record" in refs or len(refs) != len(set(refs)) or len(identities) != len(set(identities)):
124
+ raise DoyouknowError("source node references and identities must be unique")
125
+ for relation in payload["relations"]:
126
+ serialize_owned_object(Path("source-relation.json"), relation, artifact="source relation", schema={**schema, **schema["$defs"]["relation"]})
127
+ if relation["from"] not in {"$source", *refs} or relation["to"] not in {"$source", *refs}:
128
+ raise DoyouknowError("source relation endpoint is not in this snapshot")
@@ -0,0 +1,172 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import sqlite3
5
+ from contextlib import closing
6
+ from datetime import datetime, timezone
7
+ from pathlib import Path
8
+
9
+ from .contracts import DoyouknowError, MAX_INPUT_BYTES, canonical_json
10
+ from .identity import node_id, source_id, stable_id
11
+ from .source_contracts import SourceRef, SourceScan, SourceSnapshot, snapshot_payload, validate_source_value
12
+ from .store import SCHEMA_VERSION, SQLITE_TIMEOUT_SECONDS, _digest, _insert_node, _require_database_location, database_path, write_connection
13
+
14
+ SOURCE_SCHEMA_STATEMENTS = (
15
+ """create table if not exists source (
16
+ id text primary key, kind text not null, owner_id text not null,
17
+ native_id text not null, node_id text not null references node(id),
18
+ latest_snapshot text not null, scope text,
19
+ lifecycle text not null check(lifecycle in ('active', 'archived', 'unlinked', 'missing')),
20
+ checked_at text not null, attempted_at text not null,
21
+ scan_status text not null check(scan_status in ('complete', 'partial', 'unavailable')),
22
+ unique(kind, owner_id, native_id)
23
+ )""",
24
+ """create table if not exists source_snapshot (
25
+ id text primary key, source_id text not null references source(id),
26
+ previous_snapshot text references source_snapshot(id),
27
+ content_digest text not null, payload text not null, created_at text not null
28
+ )""",
29
+ """create table if not exists source_relation (
30
+ id text primary key, snapshot_id text not null references source_snapshot(id),
31
+ from_node text not null references node(id), to_node text not null references node(id),
32
+ kind text not null, reason text not null, evidence text not null,
33
+ standing text not null check(standing in ('observed', 'user-stated', 'candidate'))
34
+ )""",
35
+ "create index if not exists source_relation_snapshot on source_relation(snapshot_id)",
36
+ )
37
+
38
+
39
+ def store_source_snapshot(snapshot: SourceSnapshot, *, home: Path | None = None) -> dict:
40
+ payload = snapshot_payload(snapshot)
41
+ with write_connection(database_path(home)) as connection:
42
+ _write_source(connection, source_id(snapshot.ref), payload, datetime.now(timezone.utc).isoformat())
43
+ return {"ok": True, "sourceId": source_id(snapshot.ref)}
44
+
45
+
46
+ def apply_source_scan(scan: SourceScan, *, home: Path | None = None) -> dict:
47
+ payloads, refs = _validated_scan(scan)
48
+ now = datetime.now(timezone.utc).isoformat()
49
+ with write_connection(database_path(home)) as connection:
50
+ for snapshot, payload in zip(scan.snapshots, payloads):
51
+ _write_source(connection, source_id(snapshot.ref), payload, now)
52
+ _mark_unrefreshed_sources(connection, scan, refs, now)
53
+ return {"ok": True, "status": scan.status, "registered": len(payloads), "errors": list(scan.errors)}
54
+
55
+
56
+ def _validated_scan(scan: SourceScan) -> tuple[list[dict], list[SourceRef]]:
57
+ payloads = [snapshot_payload(snapshot) for snapshot in scan.snapshots]
58
+ refs = [*scan.observed_refs, *(snapshot.ref for snapshot in scan.snapshots)]
59
+ validate_source_value({"domain": [scan.domain.kind, scan.domain.owner_id], "refs": [[ref.kind, ref.owner_id, ref.native_id] for ref in refs], "errors": list(scan.errors)})
60
+ if scan.status not in ("complete", "partial", "unavailable"):
61
+ raise DoyouknowError("invalid source scan status")
62
+ if scan.status == "complete" and scan.errors:
63
+ raise DoyouknowError("a complete source scan cannot contain errors")
64
+ if scan.domain.kind not in ("memory", "manager", "task") or not scan.domain.owner_id.strip():
65
+ raise DoyouknowError("invalid source scan domain")
66
+ if any((ref.kind, ref.owner_id) != (scan.domain.kind, scan.domain.owner_id) for ref in refs):
67
+ raise DoyouknowError("source scan contains a different domain")
68
+ if any(not isinstance(ref.native_id, str) or not ref.native_id.strip() for ref in refs):
69
+ raise DoyouknowError("source identity must be a nonempty string")
70
+ if len({source_id(snapshot.ref) for snapshot in scan.snapshots}) != len(scan.snapshots):
71
+ raise DoyouknowError("source scan contains duplicate snapshots")
72
+ return payloads, refs
73
+
74
+
75
+ def _mark_unrefreshed_sources(connection: sqlite3.Connection, scan: SourceScan, refs: list[SourceRef], now: str) -> None:
76
+ seen = {source_id(ref) for ref in refs}
77
+ updated = {source_id(snapshot.ref) for snapshot in scan.snapshots}
78
+ rows = connection.execute("select id from source where kind = ? and owner_id = ?", (scan.domain.kind, scan.domain.owner_id)).fetchall()
79
+ for (identity,) in rows:
80
+ if identity in updated:
81
+ continue
82
+ if scan.status == "complete" and identity not in seen:
83
+ connection.execute("update source set lifecycle = 'missing', attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
84
+ else:
85
+ connection.execute("update source set attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
86
+
87
+
88
+ def _write_source(connection: sqlite3.Connection, identity: str, payload: dict, now: str) -> None:
89
+ previous = connection.execute("select source.latest_snapshot, source_snapshot.content_digest from source inner join source_snapshot on source.latest_snapshot = source_snapshot.id where source.id = ?", (identity,)).fetchone()
90
+ digest = _digest(payload)
91
+ if previous and previous[1] == digest:
92
+ connection.execute("update source set checked_at = ?, attempted_at = ?, scan_status = 'complete', lifecycle = ? where id = ?", (now, now, payload["lifecycle"], identity))
93
+ return
94
+ nodes = payload["nodes"]
95
+ for node in nodes:
96
+ _insert_node(connection, node_id(node), node["kind"], node["scope"], node["key"], node["title"])
97
+ snapshot_id = stable_id("source-snapshot", identity, previous[0] if previous else "", digest)
98
+ ref = payload["ref"]
99
+ connection.execute("insert into source (id, kind, owner_id, native_id, node_id, latest_snapshot, scope, lifecycle, checked_at, attempted_at, scan_status) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'complete') on conflict(id) do update set node_id=excluded.node_id, latest_snapshot=excluded.latest_snapshot, scope=excluded.scope, lifecycle=excluded.lifecycle, checked_at=excluded.checked_at, attempted_at=excluded.attempted_at, scan_status=excluded.scan_status", (identity, ref["kind"], ref["owner_id"], ref["native_id"], node_id(payload["primary_node"]), snapshot_id, payload["scope"], payload["lifecycle"], now, now))
100
+ connection.execute("insert into source_snapshot (id, source_id, previous_snapshot, content_digest, payload, created_at) values (?, ?, ?, ?, ?, ?)", (snapshot_id, identity, previous[0] if previous else None, digest, canonical_json(payload), now))
101
+ identities = {node.get("ref", node["key"]): node_id(node) for node in nodes}
102
+ identities["$source"] = node_id(payload["primary_node"])
103
+ for index, relation in enumerate(payload["relations"]):
104
+ connection.execute("insert into source_relation (id, snapshot_id, from_node, to_node, kind, reason, evidence, standing) values (?, ?, ?, ?, ?, ?, ?, ?)", (stable_id("source-relation", snapshot_id, str(index)), snapshot_id, identities[relation["from"]], identities[relation["to"]], relation["kind"], relation["reason"], canonical_json(relation["evidence"]), relation["standing"]))
105
+
106
+
107
+ def list_sources(*, home: Path | None = None, scope: str | None = None, kind: str | None = None, limit: int = 100, after: str | None = None) -> dict:
108
+ if not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= 1000:
109
+ raise DoyouknowError("source limit must be between 1 and 1000")
110
+ if after is not None and (not isinstance(after, str) or not after.strip()):
111
+ raise DoyouknowError("source cursor must be a nonempty string")
112
+ path = database_path(home)
113
+ _require_database_location(path)
114
+ if not path.is_file():
115
+ return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
116
+ try:
117
+ with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=SQLITE_TIMEOUT_SECONDS)) as connection:
118
+ connection.execute("begin")
119
+ version = connection.execute("pragma user_version").fetchone()[0]
120
+ if version == 1:
121
+ return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
122
+ if version != SCHEMA_VERSION:
123
+ raise DoyouknowError(f"unsupported doyouknow database version: {version}")
124
+ rows = connection.execute(
125
+ """select source.id, source.node_id, source.latest_snapshot,
126
+ source.lifecycle, source.checked_at, source.attempted_at,
127
+ source.scan_status, source_snapshot.payload
128
+ from source inner join source_snapshot
129
+ on source.latest_snapshot = source_snapshot.id
130
+ where source.id > ? and (? is null or source.kind = ?)
131
+ and (? is null or source.scope = ?)
132
+ order by source.id limit ?""",
133
+ (after or "", kind, kind, scope, scope, limit + 1),
134
+ )
135
+ return _source_page(connection, rows, limit)
136
+ except sqlite3.Error as exc:
137
+ raise DoyouknowError(f"doyouknow source lookup failed: {exc}") from exc
138
+
139
+
140
+ def _source_page(connection: sqlite3.Connection, rows: sqlite3.Cursor, limit: int) -> dict:
141
+ result = {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
142
+ item_bytes = 0
143
+ for row in rows:
144
+ if len(result["sources"]) >= limit:
145
+ result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
146
+ break
147
+ view = _source_view(connection, row)
148
+ view_bytes = len(canonical_json(view).encode("utf-8"))
149
+ header = {"ok": True, "sources": [], "truncated": True, "nextCursor": view["sourceId"]}
150
+ header_bytes = len(canonical_json(header).encode("utf-8"))
151
+ if header_bytes + view_bytes > MAX_INPUT_BYTES:
152
+ view = {key: view[key] for key in ("sourceId", "nodeId", "snapshotId", "lifecycle", "checkedAt", "attemptedAt", "scanStatus")}
153
+ view["detailsOmitted"] = True
154
+ view_bytes = len(canonical_json(view).encode("utf-8"))
155
+ required = header_bytes + item_bytes + view_bytes
156
+ if required > MAX_INPUT_BYTES:
157
+ result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
158
+ break
159
+ result["sources"].append(view)
160
+ item_bytes += view_bytes + 2
161
+ return result
162
+
163
+
164
+ def _source_view(connection: sqlite3.Connection, row: tuple) -> dict:
165
+ payload = json.loads(row[7])
166
+ relations = connection.execute("select from_node, to_node, kind, reason, evidence, standing from source_relation where snapshot_id = ? order by rowid", (row[2],)).fetchall()
167
+ return {"sourceId": row[0], "nodeId": row[1], "snapshotId": row[2],
168
+ "lifecycle": row[3], "checkedAt": row[4], "attemptedAt": row[5], "scanStatus": row[6],
169
+ "ref": payload["ref"], "scope": payload["scope"], "locator": payload["locator"],
170
+ "sourceUpdatedAt": payload["source_updated_at"], "metadata": payload["metadata"],
171
+ "nodes": [{"id": node_id(node), **node} for node in payload["nodes"]],
172
+ "relations": [{"fromNodeId": relation[0], "toNodeId": relation[1], "kind": relation[2], "reason": relation[3], "evidence": json.loads(relation[4]), "standing": relation[5]} for relation in relations]}
@@ -0,0 +1,36 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ from okstra_project.dirs import contained_relative_path
6
+
7
+ from ...json_boundary import JsonBoundaryError, load_owned_object, read_bounded_bytes
8
+ from ..contracts import DoyouknowError, MAX_INPUT_BYTES
9
+
10
+
11
+ def safe_source_path(root: Path, relative: str | Path) -> Path:
12
+ base = root.resolve()
13
+ contained = contained_relative_path(str(relative), base)
14
+ if contained is None:
15
+ raise DoyouknowError("source path is outside its registered root")
16
+ return base / contained
17
+
18
+
19
+ def read_source_text(root: Path, relative: str | Path) -> str:
20
+ path = safe_source_path(root, relative)
21
+ if not path.is_file():
22
+ raise DoyouknowError("source text is not a regular file")
23
+ try:
24
+ return read_bounded_bytes(path, max_bytes=MAX_INPUT_BYTES, artifact="source text").decode("utf-8")
25
+ except (OSError, UnicodeError, JsonBoundaryError) as exc:
26
+ raise DoyouknowError("source text is unreadable or exceeds the input limit") from exc
27
+
28
+
29
+ def read_source_object(root: Path, relative: str | Path) -> dict:
30
+ path = safe_source_path(root, relative)
31
+ if not path.is_file():
32
+ raise DoyouknowError("source object is not a regular file")
33
+ try:
34
+ return load_owned_object(path, artifact="source object", max_bytes=MAX_INPUT_BYTES)
35
+ except (OSError, JsonBoundaryError, RecursionError) as exc:
36
+ raise DoyouknowError("source object is unreadable or exceeds the input limit") from exc
@@ -0,0 +1,261 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ from collections.abc import Callable
5
+ from collections import defaultdict
6
+ from dataclasses import replace
7
+ from pathlib import Path
8
+ from stat import S_ISDIR
9
+
10
+ from okstra_project import StateError, parse_task_key
11
+
12
+ from ...json_boundary import JsonBoundaryError
13
+ from ...jsonl import parse_jsonl_snapshot
14
+ from ...manager_paths import (
15
+ ManagerPathError, canonical_manager_id, children_json_path,
16
+ directives_jsonl_path, manager_json_path, projects_json_path,
17
+ manager_root, snapshots_json_path, task_groups_root, task_manifest_path,
18
+ )
19
+ from ..config import SourceConfig, includes_project
20
+ from ..contracts import DoyouknowError, Node, Relation, SecretRefusal, canonical_json, record_secret_policy_digest
21
+ from ..source_contracts import PreparedSource, SourceDomain, SourceRef, SourceScan, SourceSnapshot, prepared_snapshot, validate_prepared_snapshot
22
+ from .io import read_source_object, read_source_text, safe_source_path
23
+
24
+
25
+ def _optional_object(home: Path, path: Path, default: dict) -> dict:
26
+ target = safe_source_path(home, path)
27
+ return read_source_object(home, target) if target.exists() else default
28
+
29
+
30
+ def _manager_manifest_candidates(home: Path, root: Path) -> tuple[Path, ...]:
31
+ groups = safe_source_path(home, task_groups_root(home, root.name))
32
+ try:
33
+ groups.stat()
34
+ except FileNotFoundError:
35
+ return ()
36
+ candidates = []
37
+ for group in sorted(groups.iterdir()):
38
+ if not S_ISDIR(safe_source_path(home, group).stat().st_mode):
39
+ continue
40
+ for task in sorted(group.iterdir()):
41
+ if S_ISDIR(safe_source_path(home, task).stat().st_mode):
42
+ candidates.append(task / "manifest.json")
43
+ return tuple(candidates)
44
+
45
+
46
+ def _project_rows(rows: object, settings: SourceConfig) -> list[dict]:
47
+ if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
48
+ raise DoyouknowError("manager project rows must be objects")
49
+ return [row for row in rows if not row.get("projectId") or includes_project(settings, row["projectId"])]
50
+
51
+
52
+ def _manager_projects(home: Path, root: Path, settings: SourceConfig) -> list[dict]:
53
+ payload = _optional_object(home, projects_json_path(home, root.name), {"projects": []})
54
+ rows = _project_rows(payload.get("projects"), settings)
55
+ for row in rows:
56
+ if not isinstance(row.get("projectId"), str) or not row["projectId"] or not isinstance(row.get("projectRoot"), str) or not row["projectRoot"]:
57
+ raise DoyouknowError("manager project registration is invalid")
58
+ return rows
59
+
60
+
61
+ def _root_snapshot(home: Path, manager: dict, projects: list[dict], settings: SourceConfig) -> SourceSnapshot:
62
+ manager_id = manager["managerId"]
63
+ nodes = tuple({"kind": "project", "scope": "okstra-project", "key": row["projectId"], "ref": f"project-{index}", "title": row["projectId"]} for index, row in enumerate(projects))
64
+ relations = tuple({"from": "$source", "to": node["ref"], "kind": "includes-project", "reason": "The manager registers this project.", "evidence": [str(projects_json_path(home, manager_id))], "standing": "observed"} for node in nodes)
65
+ snapshot = SourceSnapshot(
66
+ SourceRef("manager", manager_id, "$manager"),
67
+ {"kind": "entity", "scope": "okstra-manager", "key": manager_id, "ref": "primary", "title": manager_id},
68
+ str(manager_json_path(home, manager_id)), {"manager": manager, "projects": projects}, nodes, relations,
69
+ source_updated_at=manager.get("updatedAt"), scope=settings["scope"],
70
+ )
71
+ return snapshot
72
+
73
+
74
+ def _task_records(home: Path, manager_id: str, group: str, task_id: str, settings: SourceConfig) -> tuple[list[dict], list[dict], dict]:
75
+ children = _project_rows(read_source_object(home, children_json_path(home, manager_id, group, task_id)).get("children"), settings)
76
+ saved = _optional_object(home, snapshots_json_path(home, manager_id, group, task_id), {})
77
+ saved = {**saved, "children": _project_rows(saved.get("children", []), settings)}
78
+ directives_path = safe_source_path(home, directives_jsonl_path(home, manager_id, group, task_id))
79
+ directives, damaged = parse_jsonl_snapshot(read_source_text(home, directives_path)) if directives_path.exists() else ([], [])
80
+ if damaged:
81
+ raise DoyouknowError("manager notes contain damaged rows")
82
+ return children, _project_rows(directives, settings), saved
83
+
84
+
85
+ def _child_graph(children: list[dict], linked: set[str], evidence: Path) -> tuple[list[Node], list[Relation], list[dict]]:
86
+ nodes: list[Node] = []
87
+ relations: list[Relation] = []
88
+ kept_children = []
89
+ for index, child in enumerate(children):
90
+ key, project_id = child.get("taskKey"), child.get("projectId")
91
+ if not isinstance(key, str) or not isinstance(project_id, str) or parse_task_key(key) != (project_id, child.get("taskGroup"), child.get("taskId")):
92
+ raise DoyouknowError("manager child identity is invalid")
93
+ ref = f"child-{index}"
94
+ nodes.append({"kind": "task", "scope": project_id, "key": key, "ref": ref, "title": child["taskId"]})
95
+ relations.append({"from": "$source", "to": ref, "kind": "assigns-task" if project_id in linked else "references-unlinked-task", "reason": "The manager lists this child task.", "evidence": [str(evidence)], "standing": "observed"})
96
+ kept_children.append({**child, "projectLinked": project_id in linked})
97
+ return nodes, relations, kept_children
98
+
99
+
100
+ def _task_snapshot(home: Path, path: Path, manifest: dict, ref: SourceRef, projects: list[dict], settings: SourceConfig) -> SourceSnapshot:
101
+ group, task_id = manifest["taskGroup"], manifest["taskId"]
102
+ manager_id = ref.owner_id
103
+ if manifest.get("managerId") != manager_id or path.resolve() != task_manifest_path(home, manager_id, group, task_id).resolve():
104
+ raise DoyouknowError("manager task identity does not match its source")
105
+ children, directives, saved = _task_records(home, manager_id, group, task_id, settings)
106
+ child_nodes, child_relations, kept_children = _child_graph(children, {row["projectId"] for row in projects}, children_json_path(home, manager_id, group, task_id))
107
+ nodes = ({"kind": "entity", "scope": "okstra-manager", "key": manager_id, "ref": "manager", "title": manager_id}, *child_nodes)
108
+ relations = ({"from": "$source", "to": "manager", "kind": "belongs-to", "reason": "The manager owns this task.", "evidence": [str(path)], "standing": "observed"}, *child_relations)
109
+ snapshot = SourceSnapshot(
110
+ ref,
111
+ {"kind": "task", "scope": "manager:" + manager_id, "key": ref.native_id, "ref": "primary", "title": task_id},
112
+ str(path), {"manifest": manifest, "children": kept_children, "notes": directives, "snapshot": saved},
113
+ nodes, relations, source_updated_at=manifest.get("updatedAt"), scope=settings["scope"],
114
+ )
115
+ return snapshot
116
+
117
+
118
+ def _manager_manifest_data(home: Path, path: Path) -> dict:
119
+ manifest = read_source_object(home, path)
120
+ group, task_id = manifest.get("taskGroup"), manifest.get("taskId")
121
+ if not isinstance(group, str) or not group or not isinstance(task_id, str) or not task_id:
122
+ raise DoyouknowError("manager task identity is missing")
123
+ return manifest
124
+
125
+
126
+ def _validate_manager_snapshot(snapshot: SourceSnapshot, gate: Callable[[PreparedSource], bool] | None) -> None:
127
+ prepared = prepared_snapshot(snapshot)
128
+ if gate is not None:
129
+ if not gate(prepared):
130
+ raise SecretRefusal("manager source refused")
131
+ else:
132
+ validate_prepared_snapshot(prepared)
133
+
134
+
135
+ def _manager_task_source(home: Path, path: Path, manager_id: str, projects: list[dict], settings: SourceConfig) -> tuple[SourceRef | None, SourceSnapshot | None, str | None]:
136
+ ref = None
137
+ try:
138
+ manifest = _manager_manifest_data(home, path)
139
+ ref = SourceRef("manager", manager_id, "task:" + canonical_json([manifest["taskGroup"], manifest["taskId"]]))
140
+ snapshot = _task_snapshot(home, path, manifest, ref, projects, settings)
141
+ return ref, snapshot, None
142
+ except (DoyouknowError, JsonBoundaryError, OSError, StateError, TypeError, RecursionError):
143
+ return ref, None, "manager-task-unavailable"
144
+
145
+
146
+ def _scan_manager(home: Path, root: Path, settings: SourceConfig, *, validate: bool = True, gate: Callable[[PreparedSource], bool] | None = None) -> tuple[SourceScan, list[dict]]:
147
+ manager_id = root.name
148
+ domain = SourceDomain("manager", manager_id)
149
+ snapshots, observed, errors = [], [], []
150
+ try:
151
+ canonical_manager_id(manager_id)
152
+ manager = read_source_object(home, manager_json_path(home, manager_id))
153
+ if manager.get("managerId") != manager_id:
154
+ raise DoyouknowError("manager identity does not match its source")
155
+ projects = _manager_projects(home, root, settings)
156
+ root_snapshot = _root_snapshot(home, manager, projects, settings)
157
+ snapshots.append(root_snapshot)
158
+ observed.append(SourceRef("manager", manager_id, "$manager"))
159
+ for path in _manager_manifest_candidates(home, root):
160
+ ref, snapshot, error = _manager_task_source(home, path, manager_id, projects, settings)
161
+ if ref is not None:
162
+ observed.append(ref)
163
+ if snapshot is not None:
164
+ snapshots.append(snapshot)
165
+ else:
166
+ errors.append(error)
167
+ except (DoyouknowError, JsonBoundaryError, ManagerPathError, OSError, StateError, TypeError, RecursionError):
168
+ return SourceScan(domain, (), (), "unavailable", ("manager-source-unavailable",)), []
169
+ scan = SourceScan(domain, tuple(snapshots), tuple(observed), "partial" if errors else "complete", tuple(errors))
170
+ checked = gate_manager_scan(scan, gate=gate) if validate else scan
171
+ return checked, [] if checked.status == "unavailable" else projects
172
+
173
+
174
+ def gate_manager_scan(scan: SourceScan, *, gate: Callable[[PreparedSource], bool] | None = None) -> SourceScan:
175
+ if scan.status == "unavailable":
176
+ return scan
177
+ root = next((snapshot for snapshot in scan.snapshots if snapshot.ref.native_id == "$manager"), None)
178
+ try:
179
+ if root is None:
180
+ raise DoyouknowError("manager root is unavailable")
181
+ _validate_manager_snapshot(root, gate)
182
+ except SecretRefusal:
183
+ return replace(scan, snapshots=(), status="unavailable", errors=("manager-secret-refused",))
184
+ except (DoyouknowError, JsonBoundaryError, OSError, StateError, TypeError, RecursionError):
185
+ return replace(scan, snapshots=(), status="unavailable", errors=("manager-source-unavailable",))
186
+ snapshots = [root]
187
+ errors = list(scan.errors)
188
+ for snapshot in scan.snapshots:
189
+ if snapshot is root:
190
+ continue
191
+ try:
192
+ _validate_manager_snapshot(snapshot, gate)
193
+ snapshots.append(snapshot)
194
+ except SecretRefusal:
195
+ errors.append("manager-secret-refused")
196
+ except (DoyouknowError, JsonBoundaryError, OSError, StateError, TypeError, RecursionError):
197
+ errors.append("manager-task-unavailable")
198
+ return replace(scan, snapshots=tuple(snapshots), status="partial" if errors else scan.status, errors=tuple(errors))
199
+
200
+
201
+ def _manager_source_paths(home: Path, root: Path) -> list[tuple[Path, bool]]:
202
+ manager_id = root.name
203
+ paths = [(manager_json_path(home, manager_id), True), (projects_json_path(home, manager_id), False)]
204
+ for path in _manager_manifest_candidates(home, root):
205
+ manifest = _manager_manifest_data(home, path)
206
+ group, task_id = manifest["taskGroup"], manifest["taskId"]
207
+ paths.extend((
208
+ (path, True),
209
+ (children_json_path(home, manager_id, group, task_id), True),
210
+ (snapshots_json_path(home, manager_id, group, task_id), False),
211
+ (directives_jsonl_path(home, manager_id, group, task_id), False),
212
+ ))
213
+ return sorted(paths, key=lambda item: str(item[0]))
214
+
215
+
216
+ def _manager_source_fingerprint(home: Path, root: Path, settings: SourceConfig) -> str:
217
+ projection = {key: settings[key] for key in ("scope", "includeProjects", "excludeProjects")}
218
+ digest = hashlib.sha256(canonical_json([projection, record_secret_policy_digest()]).encode("utf-8"))
219
+ for path, required in _manager_source_paths(home, root):
220
+ target = safe_source_path(home, path)
221
+ body_digest = None
222
+ if required or target.exists():
223
+ body_digest = hashlib.sha256(read_source_text(home, target).encode("utf-8")).hexdigest()
224
+ digest.update(canonical_json([str(path.relative_to(root)), body_digest]).encode("utf-8"))
225
+ return digest.hexdigest()
226
+
227
+
228
+ def prepare_manager_source(home: Path, manager_id: str, settings: SourceConfig) -> PreparedSource:
229
+ ref = SourceRef("manager", manager_id, "$manager")
230
+ if "manager" not in settings["sources"]:
231
+ return PreparedSource(ref, None, outcome="excluded")
232
+ try:
233
+ root = manager_root(home, manager_id)
234
+ before = _manager_source_fingerprint(home, root, settings)
235
+ scan, _ = _scan_manager(home, root, settings, validate=False)
236
+ after = _manager_source_fingerprint(home, root, settings)
237
+ if scan.status == "unavailable" or before != after:
238
+ return PreparedSource(ref, None, outcome="unavailable", error_code="manager-source-unavailable")
239
+ return PreparedSource(ref, before, scan=scan)
240
+ except (DoyouknowError, JsonBoundaryError, ManagerPathError, OSError, StateError, TypeError, RecursionError):
241
+ return PreparedSource(ref, None, outcome="unavailable", error_code="manager-source-unavailable")
242
+
243
+
244
+ def scan_managers(home: Path, settings: SourceConfig, *, gate: Callable[[PreparedSource], bool] | None = None) -> tuple[tuple[SourceScan, ...], dict[str, tuple[Path, ...]]]:
245
+ scans, projects = [], defaultdict(set)
246
+ if not {"manager", "task"}.intersection(settings["sources"]):
247
+ return (), {}
248
+ try:
249
+ base = safe_source_path(home, "managers")
250
+ roots = sorted(path for path in base.iterdir() if path.is_dir()) if base.exists() else []
251
+ for root in roots:
252
+ if "manager" in settings["sources"]:
253
+ scan, entries = _scan_manager(home, root, settings, gate=gate)
254
+ scans.append(scan)
255
+ else:
256
+ entries = _manager_projects(home, root, settings)
257
+ for entry in entries:
258
+ projects[entry["projectId"]].add(Path(entry["projectRoot"]).resolve())
259
+ except (DoyouknowError, JsonBoundaryError, OSError, TypeError, RecursionError):
260
+ scans.append(SourceScan(SourceDomain("manager", "$discovery"), (), (), "unavailable", ("manager-discovery-unavailable",)))
261
+ return tuple(scans), {key: tuple(sorted(paths)) for key, paths in projects.items()}