okstra 0.231.1 → 0.232.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/cli-registry.mjs +6 -0
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/memory/memory.mjs +9 -55
- package/dist/commands/memory/memory.mjs.map +1 -1
- package/dist/lib/memory-auto-register.d.mts +1 -0
- package/dist/lib/memory-auto-register.mjs +113 -0
- package/dist/lib/memory-auto-register.mjs.map +1 -0
- package/dist/lib/memory-entry-contract.d.mts +12 -0
- package/dist/lib/memory-entry-contract.mjs +46 -0
- package/dist/lib/memory-entry-contract.mjs.map +1 -0
- package/dist/lib/python-helper.d.mts +1 -1
- package/dist/lib/python-helper.mjs +3 -2
- package/dist/lib/python-helper.mjs.map +1 -1
- package/dist/lib/record-secrets.d.mts +1 -0
- package/dist/lib/record-secrets.mjs +28 -0
- package/dist/lib/record-secrets.mjs.map +1 -0
- package/dist/lib/types.d.mts +2 -1
- package/docs/architecture/storage-model.md +127 -0
- package/docs/architecture.md +7 -0
- package/docs/cli.md +234 -0
- package/docs/project-structure-overview.md +2 -1
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/agents/common.json +8 -8
- package/runtime/agents/roles/verifier.json +2 -4
- package/runtime/bin/okstra-provider-exec.py +10 -4
- package/runtime/prompts/coding-preflight/architectures/hexagonal.md +23 -30
- package/runtime/prompts/coding-preflight/authoring-checks.md +17 -0
- package/runtime/prompts/coding-preflight/clean-code.md +85 -281
- package/runtime/prompts/coding-preflight/frameworks/node-server.md +2 -3
- package/runtime/prompts/coding-preflight/languages/java.md +3 -5
- package/runtime/prompts/coding-preflight/languages/javascript-typescript.md +5 -12
- package/runtime/prompts/coding-preflight/languages/kotlin.md +2 -4
- package/runtime/prompts/coding-preflight/languages/python.md +2 -7
- package/runtime/prompts/coding-preflight/languages/rust.md +2 -8
- package/runtime/prompts/coding-preflight/languages/sql.md +2 -4
- package/runtime/prompts/coding-preflight/overview.md +25 -38
- package/runtime/prompts/coding-preflight/review-rules.json +30 -0
- package/runtime/prompts/duties/code-reviewer.json +15 -18
- package/runtime/prompts/duties/implementation-verifier.json +1 -0
- package/runtime/prompts/profiles/_coding-conventions-preflight.md +41 -6
- package/runtime/python/okstra_ctl/agent/invocation.py +121 -36
- package/runtime/python/okstra_ctl/agent/prompt_cli/cli.py +6 -0
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +30 -4
- package/runtime/python/okstra_ctl/agent/standalone.py +3 -0
- package/runtime/python/okstra_ctl/analysis_packet.py +2 -19
- package/runtime/python/okstra_ctl/attempt_evidence.py +10 -0
- package/runtime/python/okstra_ctl/code_review_census.py +71 -7
- package/runtime/python/okstra_ctl/code_review_coverage.py +4 -274
- package/runtime/python/okstra_ctl/code_review_integrity.py +154 -0
- package/runtime/python/okstra_ctl/code_review_invocation.py +212 -0
- package/runtime/python/okstra_ctl/code_review_scope.py +36 -9
- package/runtime/python/okstra_ctl/code_review_target.py +85 -11
- package/runtime/python/okstra_ctl/consumers.py +12 -9
- package/runtime/python/okstra_ctl/design_review.py +207 -0
- package/runtime/python/okstra_ctl/design_review_evidence.py +297 -0
- package/runtime/python/okstra_ctl/doctor.py +8 -25
- package/runtime/python/okstra_ctl/doyouknow/__init__.py +9 -0
- package/runtime/python/okstra_ctl/doyouknow/cli.py +91 -0
- package/runtime/python/okstra_ctl/doyouknow/config.py +58 -0
- package/runtime/python/okstra_ctl/doyouknow/contracts.py +184 -0
- package/runtime/python/okstra_ctl/doyouknow/identity.py +22 -0
- package/runtime/python/okstra_ctl/doyouknow/paths.py +22 -0
- package/runtime/python/okstra_ctl/doyouknow/pending.py +72 -0
- package/runtime/python/okstra_ctl/doyouknow/registration.py +260 -0
- package/runtime/python/okstra_ctl/doyouknow/source_contracts.py +128 -0
- package/runtime/python/okstra_ctl/doyouknow/source_store.py +172 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/io.py +36 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/manager.py +261 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/memory.py +179 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/task.py +95 -0
- package/runtime/python/okstra_ctl/doyouknow/store.py +260 -0
- package/runtime/python/okstra_ctl/doyouknow/sync.py +58 -0
- package/runtime/python/okstra_ctl/execution_mutation_audit.py +18 -2
- package/runtime/python/okstra_ctl/file_size_cap.py +3 -4
- package/runtime/python/okstra_ctl/json_boundary.py +21 -2
- package/runtime/python/okstra_ctl/jsonl.py +21 -0
- package/runtime/python/okstra_ctl/manager_cli.py +26 -0
- package/runtime/python/okstra_ctl/manager_registration.py +29 -0
- package/runtime/python/okstra_ctl/manager_sync.py +3 -22
- package/runtime/python/okstra_ctl/model_io/renderers.py +13 -2
- package/runtime/python/okstra_ctl/model_io_cli.py +7 -1
- package/runtime/python/okstra_ctl/phases/final_verification/validation.py +20 -0
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +10 -0
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-diff-review.md +19 -6
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-self-check.md +5 -3
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +38 -10
- package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation.template.html +13 -0
- package/runtime/python/okstra_ctl/phases/implementation/spec.md +25 -0
- package/runtime/python/okstra_ctl/phases/implementation/validation.py +6 -0
- package/runtime/python/okstra_ctl/phases/implementation_planning/authoring.py +4 -0
- package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +1 -1
- package/runtime/python/okstra_ctl/report_assembly.py +15 -0
- package/runtime/python/okstra_ctl/report_narrative.py +3 -0
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +11 -2
- package/runtime/python/okstra_ctl/review_policy.py +359 -0
- package/runtime/python/okstra_ctl/review_verdicts.py +291 -0
- package/runtime/python/okstra_ctl/run.py +24 -1
- package/runtime/python/okstra_ctl/stage_done.py +11 -0
- package/runtime/python/okstra_ctl/standalone_review_audit.py +28 -0
- package/runtime/python/okstra_ctl/verification_generation.py +23 -1
- package/runtime/python/okstra_ctl/verification_input_types.py +15 -1
- package/runtime/python/okstra_ctl/verification_inputs.py +28 -6
- package/runtime/python/okstra_ctl/verification_invocation.py +7 -1
- package/runtime/python/okstra_ctl/verification_target.py +2 -0
- package/runtime/python/okstra_ctl/worker_prompt_headers.py +24 -9
- package/runtime/python/okstra_ctl/worker_runner.py +66 -10
- package/runtime/python/okstra_ctl/write_policy.py +34 -1
- package/runtime/python/okstra_project/__init__.py +2 -0
- package/runtime/python/okstra_project/dirs.py +18 -0
- package/runtime/python/okstra_project/state.py +25 -20
- package/runtime/schemas/doyouknow-config-v1.schema.json +20 -0
- package/runtime/schemas/doyouknow-v1.schema.json +65 -0
- package/runtime/schemas/final-report-v2.0.schema.json +157 -0
- package/runtime/schemas/final-report-v3.0.schema.json +157 -0
- package/runtime/schemas/memory-entry-contract-v1.json +11 -0
- package/runtime/schemas/record-secret-patterns.json +41 -0
- package/runtime/skills/okstra-code-review/SKILL.md +71 -123
- package/runtime/skills/okstra-code-review/references/census-rules.md +29 -105
- package/runtime/skills/okstra-code-review/references/review-calibration.md +34 -54
- package/runtime/skills/okstra-manager/SKILL.md +7 -0
- package/runtime/skills/okstra-memory/SKILL.md +7 -0
- package/runtime/templates/implementation-worker-preamble.md +4 -9
- package/runtime/templates/worker-prompt-preamble.md +4 -12
- package/runtime/validators/validate-run.py +14 -2
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Literal, NotRequired, TypedDict
|
|
9
|
+
|
|
10
|
+
from ..json_boundary import (
|
|
11
|
+
external_user_json_source,
|
|
12
|
+
load_external_json,
|
|
13
|
+
load_owned_object,
|
|
14
|
+
serialize_owned_object,
|
|
15
|
+
)
|
|
16
|
+
from ..paths import find_asset_root
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DoyouknowError(ValueError):
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SecretRefusal(DoyouknowError):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
MAX_INPUT_BYTES = 2_000_000
|
|
28
|
+
MAX_INPUT_DEPTH = 32
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Source(TypedDict):
|
|
32
|
+
kind: str
|
|
33
|
+
id: str
|
|
34
|
+
revision: str
|
|
35
|
+
locator: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Record(TypedDict):
|
|
39
|
+
title: str
|
|
40
|
+
summary: str
|
|
41
|
+
body: NotRequired[str]
|
|
42
|
+
type: NotRequired[str]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Node(TypedDict):
|
|
46
|
+
key: str
|
|
47
|
+
ref: NotRequired[str]
|
|
48
|
+
kind: str
|
|
49
|
+
scope: str
|
|
50
|
+
title: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
Relation = TypedDict("Relation", {
|
|
54
|
+
"from": str,
|
|
55
|
+
"to": str,
|
|
56
|
+
"kind": str,
|
|
57
|
+
"reason": str,
|
|
58
|
+
"evidence": list[str],
|
|
59
|
+
"standing": Literal["observed", "user-stated", "candidate"],
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Envelope(TypedDict):
|
|
64
|
+
schemaVersion: int
|
|
65
|
+
scope: str
|
|
66
|
+
recordKey: str
|
|
67
|
+
requestId: str
|
|
68
|
+
expectedPreviousRevision: str | None
|
|
69
|
+
source: Source
|
|
70
|
+
record: Record
|
|
71
|
+
nodes: list[Node]
|
|
72
|
+
relations: list[Relation]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def asset_object(filename: str) -> dict:
|
|
76
|
+
relative = ("schemas", filename)
|
|
77
|
+
root = find_asset_root(relative)
|
|
78
|
+
if root is None:
|
|
79
|
+
raise DoyouknowError(f"required schema asset is unavailable: {filename}")
|
|
80
|
+
return load_owned_object(root.joinpath(*relative), artifact="doyouknow schema")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def canonical_json(value: object) -> str:
|
|
84
|
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
|
88
|
+
result = {}
|
|
89
|
+
for key, value in pairs:
|
|
90
|
+
if key in result:
|
|
91
|
+
raise DoyouknowError("duplicate JSON object key")
|
|
92
|
+
result[key] = value
|
|
93
|
+
return result
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def read_envelope(path: Path) -> Envelope:
|
|
97
|
+
try:
|
|
98
|
+
value = load_external_json(
|
|
99
|
+
external_user_json_source(path), artifact="doyouknow input",
|
|
100
|
+
object_pairs_hook=_unique_json_object, max_bytes=MAX_INPUT_BYTES,
|
|
101
|
+
)
|
|
102
|
+
except RecursionError as exc:
|
|
103
|
+
raise DoyouknowError("doyouknow input nesting is too deep") from exc
|
|
104
|
+
return validate_envelope(value)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def validate_envelope(value: object) -> Envelope:
|
|
108
|
+
_check_input_depth(value)
|
|
109
|
+
try:
|
|
110
|
+
encoded = canonical_json(value)
|
|
111
|
+
except (TypeError, ValueError) as exc:
|
|
112
|
+
raise DoyouknowError("doyouknow input is not a JSON value") from exc
|
|
113
|
+
if len(encoded.encode("utf-8")) > MAX_INPUT_BYTES:
|
|
114
|
+
raise DoyouknowError("doyouknow input exceeds 2000000 bytes")
|
|
115
|
+
reject_secrets("\n".join(_decoded_strings(value)))
|
|
116
|
+
serialized = serialize_owned_object(
|
|
117
|
+
Path("doyouknow-input.json"), value, artifact="doyouknow input",
|
|
118
|
+
schema=asset_object("doyouknow-v1.schema.json"),
|
|
119
|
+
)
|
|
120
|
+
envelope = json.loads(serialized)
|
|
121
|
+
_validate_node_references(envelope)
|
|
122
|
+
return envelope
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _decoded_strings(value: object) -> Iterator[str]:
|
|
126
|
+
pending = [value]
|
|
127
|
+
while pending:
|
|
128
|
+
item = pending.pop()
|
|
129
|
+
if isinstance(item, str):
|
|
130
|
+
yield item
|
|
131
|
+
elif isinstance(item, dict):
|
|
132
|
+
pending.extend(item.keys())
|
|
133
|
+
pending.extend(item.values())
|
|
134
|
+
elif isinstance(item, list):
|
|
135
|
+
pending.extend(item)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _check_input_depth(value: object) -> None:
|
|
139
|
+
pending = [(value, 0)]
|
|
140
|
+
while pending:
|
|
141
|
+
item, depth = pending.pop()
|
|
142
|
+
if depth > MAX_INPUT_DEPTH:
|
|
143
|
+
raise DoyouknowError("doyouknow input nesting is too deep")
|
|
144
|
+
if isinstance(item, dict):
|
|
145
|
+
pending.extend((child, depth + 1) for child in item.values())
|
|
146
|
+
elif isinstance(item, list):
|
|
147
|
+
pending.extend((child, depth + 1) for child in item)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _validate_node_references(envelope: Envelope) -> None:
|
|
151
|
+
keys = [node.get("ref", node["key"]) for node in envelope["nodes"]]
|
|
152
|
+
if "$record" in keys or len(keys) != len(set(keys)):
|
|
153
|
+
raise DoyouknowError("node references must be unique and cannot be $record")
|
|
154
|
+
identities = [(node["kind"], node["scope"], node["key"]) for node in envelope["nodes"]]
|
|
155
|
+
if len(identities) != len(set(identities)):
|
|
156
|
+
raise DoyouknowError("canonical node identities must be unique in one input")
|
|
157
|
+
allowed = {"$record", *keys}
|
|
158
|
+
for relation in envelope["relations"]:
|
|
159
|
+
if relation["from"] not in allowed or relation["to"] not in allowed:
|
|
160
|
+
raise DoyouknowError("relation endpoint is not in this input")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def load_record_secret_policy() -> dict:
|
|
164
|
+
policy = asset_object("record-secret-patterns.json")
|
|
165
|
+
try:
|
|
166
|
+
if policy["schemaVersion"] != 1:
|
|
167
|
+
raise DoyouknowError("unsupported record secret policy version")
|
|
168
|
+
except (KeyError, TypeError) as exc:
|
|
169
|
+
raise DoyouknowError("record secret policy is invalid") from exc
|
|
170
|
+
return policy
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def record_secret_policy_digest() -> str:
|
|
174
|
+
return hashlib.sha256(canonical_json(load_record_secret_policy()).encode("utf-8")).hexdigest()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def reject_secrets(text: str) -> None:
|
|
178
|
+
policy = load_record_secret_policy()
|
|
179
|
+
try:
|
|
180
|
+
hits = [row["label"] for row in policy["patterns"] if re.search(row["pattern"], text, re.ASCII)]
|
|
181
|
+
except (KeyError, TypeError, re.error) as exc:
|
|
182
|
+
raise DoyouknowError("record secret policy is invalid") from exc
|
|
183
|
+
if hits:
|
|
184
|
+
raise SecretRefusal("refusing likely secret(s): " + ", ".join(dict.fromkeys(hits)))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from .contracts import Node, canonical_json
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from .source_contracts import SourceRef
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def stable_id(kind: str, *parts: str) -> str:
|
|
13
|
+
digest = hashlib.sha256(canonical_json([kind, *parts]).encode("utf-8")).hexdigest()
|
|
14
|
+
return f"{kind}-{digest}"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def node_id(node: Node) -> str:
|
|
18
|
+
return stable_id(node["kind"], node["scope"], node["key"])
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def source_id(ref: SourceRef) -> str:
|
|
22
|
+
return stable_id("source", ref.kind, ref.owner_id, ref.native_id)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from okstra_project.dirs import okstra_home
|
|
6
|
+
|
|
7
|
+
from .contracts import DoyouknowError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def database_path(home: Path | None = None) -> Path:
|
|
11
|
+
root = home if home is not None else okstra_home()
|
|
12
|
+
return root.expanduser().resolve() / "doyouknow" / "graph.sqlite3"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def require_database_location(path: Path) -> None:
|
|
16
|
+
if path.is_symlink() or path.parent.is_symlink():
|
|
17
|
+
raise DoyouknowError("doyouknow storage cannot use a symlink")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def ensure_storage_root(root: Path) -> None:
|
|
21
|
+
require_database_location(root / "graph.sqlite3")
|
|
22
|
+
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import tempfile
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from uuid import UUID, uuid4
|
|
8
|
+
|
|
9
|
+
from ..json_boundary import load_owned_object
|
|
10
|
+
from .contracts import DoyouknowError
|
|
11
|
+
from .paths import database_path, ensure_storage_root, require_database_location
|
|
12
|
+
from .source_contracts import SourceRef, validate_source_value
|
|
13
|
+
|
|
14
|
+
MAX_TICKET_BYTES = 16_384
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def pending_path(ticket: str, *, home: Path | None = None) -> Path:
|
|
18
|
+
try:
|
|
19
|
+
if str(UUID(ticket)) != ticket:
|
|
20
|
+
raise ValueError
|
|
21
|
+
except (ValueError, TypeError, AttributeError) as exc:
|
|
22
|
+
raise DoyouknowError("invalid registration ticket") from exc
|
|
23
|
+
database = database_path(home)
|
|
24
|
+
require_database_location(database)
|
|
25
|
+
path = database.parent / "pending" / f"{ticket}.json"
|
|
26
|
+
require_database_location(path)
|
|
27
|
+
return path
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _ticket_ref(payload: dict) -> SourceRef:
|
|
31
|
+
try:
|
|
32
|
+
validate_source_value(payload)
|
|
33
|
+
raw = payload["sourceRef"]
|
|
34
|
+
if set(payload) != {"schemaVersion", "sourceRef"} or type(payload["schemaVersion"]) is not int or payload["schemaVersion"] != 1:
|
|
35
|
+
raise ValueError
|
|
36
|
+
if not isinstance(raw, dict) or set(raw) != {"kind", "owner_id", "native_id"}:
|
|
37
|
+
raise ValueError
|
|
38
|
+
if raw["kind"] not in ("memory", "manager", "task"):
|
|
39
|
+
raise ValueError
|
|
40
|
+
if any(not isinstance(value, str) or not value.strip() or len(value) > 4096 for value in raw.values()):
|
|
41
|
+
raise ValueError
|
|
42
|
+
return SourceRef(raw["kind"], raw["owner_id"], raw["native_id"])
|
|
43
|
+
except (KeyError, ValueError, TypeError) as exc:
|
|
44
|
+
raise DoyouknowError("invalid registration reference") from exc
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def enqueue_source(ref: SourceRef, *, home: Path | None = None) -> str:
|
|
48
|
+
payload = {"schemaVersion": 1, "sourceRef": {"kind": ref.kind, "owner_id": ref.owner_id, "native_id": ref.native_id}}
|
|
49
|
+
_ticket_ref(payload)
|
|
50
|
+
encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
51
|
+
if len(encoded) > MAX_TICKET_BYTES:
|
|
52
|
+
raise DoyouknowError("registration reference exceeds input limit")
|
|
53
|
+
ticket = str(uuid4())
|
|
54
|
+
path = pending_path(ticket, home=home)
|
|
55
|
+
ensure_storage_root(path.parent.parent)
|
|
56
|
+
path.parent.mkdir(exist_ok=True, mode=0o700)
|
|
57
|
+
descriptor, name = tempfile.mkstemp(prefix=".ticket-", dir=path.parent)
|
|
58
|
+
temporary = Path(name)
|
|
59
|
+
try:
|
|
60
|
+
with os.fdopen(descriptor, "wb") as handle:
|
|
61
|
+
handle.write(encoded)
|
|
62
|
+
handle.flush()
|
|
63
|
+
os.fsync(handle.fileno())
|
|
64
|
+
os.link(temporary, path)
|
|
65
|
+
finally:
|
|
66
|
+
temporary.unlink(missing_ok=True)
|
|
67
|
+
return ticket
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def read_ticket(ticket: str, *, home: Path | None = None) -> SourceRef:
|
|
71
|
+
path = pending_path(ticket, home=home)
|
|
72
|
+
return _ticket_ref(load_owned_object(path, artifact="registration ticket", max_bytes=MAX_TICKET_BYTES))
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ..json_boundary import JsonBoundaryError, load_owned_object, write_owned_object_atomic
|
|
9
|
+
from ..run_context import dir_flock
|
|
10
|
+
from .config import SourceConfig, load_config
|
|
11
|
+
from .contracts import DoyouknowError, SecretRefusal
|
|
12
|
+
from .identity import source_id
|
|
13
|
+
from .paths import database_path, ensure_storage_root, require_database_location
|
|
14
|
+
from .pending import MAX_TICKET_BYTES, pending_path, read_ticket
|
|
15
|
+
from .source_contracts import PreparedSource, SourceKind, SourceRef, validate_prepared_snapshot, validate_source_value
|
|
16
|
+
from .sources.memory import read_memory_source
|
|
17
|
+
|
|
18
|
+
MAX_FAILURES = 3
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class RegistrationState:
|
|
23
|
+
fingerprint: str | None = None
|
|
24
|
+
attempts: int = 0
|
|
25
|
+
status: str = "pending"
|
|
26
|
+
errorCode: str | None = None
|
|
27
|
+
refusedFingerprint: str | None = None
|
|
28
|
+
preparationAttempts: int = 0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _read_state(path: Path) -> tuple[RegistrationState, dict | None]:
|
|
32
|
+
require_database_location(path)
|
|
33
|
+
if not path.exists():
|
|
34
|
+
return RegistrationState(), None
|
|
35
|
+
previous = load_owned_object(path, artifact="registration state", max_bytes=MAX_TICKET_BYTES)
|
|
36
|
+
validate_source_value(previous)
|
|
37
|
+
raw = {"preparationAttempts": 0, **previous}
|
|
38
|
+
strings = ("fingerprint", "errorCode", "refusedFingerprint")
|
|
39
|
+
if set(raw) != {*strings, "attempts", "status", "preparationAttempts"} or any(raw[key] is not None and not isinstance(raw[key], str) for key in strings):
|
|
40
|
+
raise DoyouknowError("invalid registration state")
|
|
41
|
+
if type(raw["attempts"]) is not int or not 0 <= raw["attempts"] <= MAX_FAILURES:
|
|
42
|
+
raise DoyouknowError("invalid registration state")
|
|
43
|
+
if type(raw["preparationAttempts"]) is not int or not 0 <= raw["preparationAttempts"] <= MAX_FAILURES:
|
|
44
|
+
raise DoyouknowError("invalid registration state")
|
|
45
|
+
if raw["status"] not in ("registered", "pending", "refused", "exhausted"):
|
|
46
|
+
raise DoyouknowError("invalid registration state")
|
|
47
|
+
if raw["errorCode"] not in (None, "registration-unavailable", "source-unavailable", "source-refused", "source-preparation-unavailable", "source-preparation-exhausted"):
|
|
48
|
+
raise DoyouknowError("invalid registration state")
|
|
49
|
+
return RegistrationState(**raw), previous
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _result(ticket: str | None, ref: SourceRef | None, state: RegistrationState) -> dict:
|
|
53
|
+
return {"ok": state.status in ("registered", "excluded"), "ticket": ticket,
|
|
54
|
+
"sourceId": source_id(ref) if ref else None, "status": state.status,
|
|
55
|
+
"attempts": state.attempts, "preparationAttempts": state.preparationAttempts, "errorCode": state.errorCode}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def register_ticket(ticket: str, *, home: Path | None = None, manual: bool = False) -> dict:
|
|
59
|
+
ref = None
|
|
60
|
+
safe_ticket = None
|
|
61
|
+
try:
|
|
62
|
+
pending_path(ticket, home=home)
|
|
63
|
+
safe_ticket = ticket
|
|
64
|
+
ref = read_ticket(ticket, home=home)
|
|
65
|
+
directory = database_path(home).parent / "registrations"
|
|
66
|
+
require_database_location(directory / "state.json")
|
|
67
|
+
require_database_location(directory / f".{source_id(ref)}.lock")
|
|
68
|
+
ensure_storage_root(directory.parent)
|
|
69
|
+
with dir_flock(directory, f".{source_id(ref)}.lock"):
|
|
70
|
+
if not pending_path(ticket, home=home).exists():
|
|
71
|
+
return _result(ticket, ref, RegistrationState(status="registered"))
|
|
72
|
+
return _register_locked(ticket, ref, directory / f"{source_id(ref)}.json", home, manual)
|
|
73
|
+
except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
|
|
74
|
+
return _result(safe_ticket, ref, RegistrationState(errorCode="registration-unavailable"))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _register_locked(ticket: str, ref: SourceRef, path: Path, home: Path | None, manual: bool) -> dict:
|
|
78
|
+
settings = load_config(home=home)
|
|
79
|
+
if (not settings["autoRegister"] and not manual) or ref.kind not in settings["sources"]:
|
|
80
|
+
pending_path(ticket, home=home).unlink(missing_ok=True)
|
|
81
|
+
return _result(ticket, ref, RegistrationState(status="excluded"))
|
|
82
|
+
if ref.kind not in ("memory", "manager"):
|
|
83
|
+
return _result(ticket, ref, RegistrationState(errorCode="unsupported-registration-source"))
|
|
84
|
+
state, previous = _read_state(path)
|
|
85
|
+
if state.preparationAttempts >= MAX_FAILURES and not manual:
|
|
86
|
+
state.status, state.errorCode = "exhausted", "source-preparation-exhausted"
|
|
87
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
88
|
+
return _result(ticket, ref, state)
|
|
89
|
+
state.preparationAttempts = min(state.preparationAttempts + 1, MAX_FAILURES)
|
|
90
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
91
|
+
try:
|
|
92
|
+
prepared = _prepare_source(ref, database_path(home).parent.parent, settings)
|
|
93
|
+
except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
|
|
94
|
+
state.status = "exhausted" if state.preparationAttempts >= MAX_FAILURES else "pending"
|
|
95
|
+
state.errorCode = "source-preparation-unavailable"
|
|
96
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
97
|
+
return _result(ticket, ref, state)
|
|
98
|
+
state.preparationAttempts = 0
|
|
99
|
+
if prepared.outcome == "excluded":
|
|
100
|
+
if previous is None:
|
|
101
|
+
path.unlink(missing_ok=True)
|
|
102
|
+
else:
|
|
103
|
+
write_owned_object_atomic(path, previous, artifact="registration state")
|
|
104
|
+
pending_path(ticket, home=home).unlink(missing_ok=True)
|
|
105
|
+
return _result(ticket, ref, RegistrationState(status="excluded"))
|
|
106
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
107
|
+
if prepared.ref != ref:
|
|
108
|
+
raise DoyouknowError("registration reference does not match its source")
|
|
109
|
+
return _register_prepared(ticket, ref, prepared, path, home, manual)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _prepare_source(ref: SourceRef, home: Path, settings: SourceConfig) -> PreparedSource:
|
|
113
|
+
if ref.kind == "memory":
|
|
114
|
+
return read_memory_source(home, ref, settings)
|
|
115
|
+
from .sources.manager import prepare_manager_source
|
|
116
|
+
|
|
117
|
+
if ref.native_id != "$manager":
|
|
118
|
+
raise DoyouknowError("invalid manager registration reference")
|
|
119
|
+
return prepare_manager_source(home, ref.owner_id, settings)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _register_prepared(ticket: str, ref: SourceRef, prepared: PreparedSource, path: Path, home: Path | None, manual: bool) -> dict:
|
|
123
|
+
state, _ = _read_state(path)
|
|
124
|
+
if prepared.fingerprint is not None and prepared.fingerprint == state.refusedFingerprint:
|
|
125
|
+
state.status, state.errorCode = "refused", "source-refused"
|
|
126
|
+
return _result(ticket, ref, state)
|
|
127
|
+
if prepared.fingerprint is not None and state.fingerprint != prepared.fingerprint:
|
|
128
|
+
state.fingerprint, state.attempts = prepared.fingerprint, 0
|
|
129
|
+
if state.attempts >= MAX_FAILURES and not manual:
|
|
130
|
+
state.status = "exhausted"
|
|
131
|
+
return _result(ticket, ref, state)
|
|
132
|
+
state.attempts = min(state.attempts + 1, MAX_FAILURES)
|
|
133
|
+
state.status, state.errorCode = "pending", None
|
|
134
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
135
|
+
try:
|
|
136
|
+
_store_prepared(prepared, home)
|
|
137
|
+
except SecretRefusal:
|
|
138
|
+
state.status, state.errorCode = "refused", "source-refused"
|
|
139
|
+
state.refusedFingerprint = prepared.fingerprint
|
|
140
|
+
except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
|
|
141
|
+
state.status = "exhausted" if state.attempts >= MAX_FAILURES else "pending"
|
|
142
|
+
state.errorCode = "source-unavailable"
|
|
143
|
+
else:
|
|
144
|
+
state.status, state.attempts, state.errorCode, state.refusedFingerprint = "registered", 0, None, None
|
|
145
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
146
|
+
if state.status == "registered":
|
|
147
|
+
pending_path(ticket, home=home).unlink(missing_ok=True)
|
|
148
|
+
return _result(ticket, ref, state)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _store_prepared(prepared: PreparedSource, home: Path | None) -> None:
|
|
152
|
+
if prepared.outcome != "ready":
|
|
153
|
+
raise DoyouknowError("source unavailable")
|
|
154
|
+
if prepared.scan is not None:
|
|
155
|
+
from .sources.manager import gate_manager_scan
|
|
156
|
+
from .source_store import apply_source_scan
|
|
157
|
+
|
|
158
|
+
scan = gate_manager_scan(prepared.scan, gate=lambda candidate: validate_source_and_update_refusal_cache(candidate, home=home))
|
|
159
|
+
refused = "manager-secret-refused" in scan.errors
|
|
160
|
+
if scan.status != "unavailable":
|
|
161
|
+
apply_source_scan(scan, home=home)
|
|
162
|
+
if refused:
|
|
163
|
+
raise SecretRefusal("manager source refused")
|
|
164
|
+
if scan.status != "complete":
|
|
165
|
+
raise DoyouknowError("source scan incomplete")
|
|
166
|
+
elif prepared.snapshot is not None:
|
|
167
|
+
if not validate_source_and_update_refusal_cache(prepared, home=home):
|
|
168
|
+
raise SecretRefusal("source refused")
|
|
169
|
+
from .source_store import store_source_snapshot
|
|
170
|
+
|
|
171
|
+
store_source_snapshot(prepared.snapshot, home=home)
|
|
172
|
+
else:
|
|
173
|
+
raise DoyouknowError("source snapshot unavailable")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def validate_source_and_update_refusal_cache(prepared: PreparedSource, *, home: Path | None = None) -> bool:
|
|
177
|
+
if prepared.outcome != "ready" or prepared.snapshot is None or prepared.fingerprint is None:
|
|
178
|
+
raise DoyouknowError("source snapshot unavailable")
|
|
179
|
+
database = database_path(home)
|
|
180
|
+
require_database_location(database)
|
|
181
|
+
directory = database.parent / "refusals"
|
|
182
|
+
path = directory / f"{source_id(prepared.ref)}.json"
|
|
183
|
+
require_database_location(path)
|
|
184
|
+
require_database_location(directory / f".{source_id(prepared.ref)}.lock")
|
|
185
|
+
ensure_storage_root(directory.parent)
|
|
186
|
+
with dir_flock(directory, f".{source_id(prepared.ref)}.lock"):
|
|
187
|
+
if path.exists():
|
|
188
|
+
cached = load_owned_object(path, artifact="source refusal", max_bytes=MAX_TICKET_BYTES)
|
|
189
|
+
validate_source_value(cached)
|
|
190
|
+
if set(cached) != {"fingerprint"} or not isinstance(cached["fingerprint"], str):
|
|
191
|
+
raise DoyouknowError("invalid source refusal")
|
|
192
|
+
if cached["fingerprint"] == prepared.fingerprint:
|
|
193
|
+
return False
|
|
194
|
+
try:
|
|
195
|
+
validate_prepared_snapshot(prepared)
|
|
196
|
+
except SecretRefusal:
|
|
197
|
+
write_owned_object_atomic(path, {"fingerprint": prepared.fingerprint}, artifact="source refusal")
|
|
198
|
+
return False
|
|
199
|
+
path.unlink(missing_ok=True)
|
|
200
|
+
return True
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def retry_pending(*, home: Path | None = None, source: SourceKind | None = None, limit: int = 100, manual: bool = False) -> dict:
|
|
204
|
+
if type(limit) is not int or not 1 <= limit <= 1000:
|
|
205
|
+
raise DoyouknowError("pending limit must be between 1 and 1000")
|
|
206
|
+
directory = database_path(home).parent / "pending"
|
|
207
|
+
require_database_location(directory / "ticket.json")
|
|
208
|
+
results = []
|
|
209
|
+
for ticket in _pending_candidates(directory, source, home):
|
|
210
|
+
if len(results) >= limit:
|
|
211
|
+
return {"ok": all(row["ok"] for row in results), "results": results, "truncated": True}
|
|
212
|
+
result = register_ticket(ticket, home=home, manual=manual)
|
|
213
|
+
results.append(result)
|
|
214
|
+
if not result["ok"]:
|
|
215
|
+
try:
|
|
216
|
+
os.utime(pending_path(ticket, home=home), follow_symlinks=False)
|
|
217
|
+
except FileNotFoundError:
|
|
218
|
+
continue
|
|
219
|
+
except (DoyouknowError, OSError):
|
|
220
|
+
result["errorCode"] = "pending-order-unavailable"
|
|
221
|
+
return {"ok": all(row["ok"] for row in results), "results": results, "truncated": False}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _pending_candidates(directory: Path, source: SourceKind | None, home: Path | None) -> Iterator[str]:
|
|
225
|
+
if not directory.exists():
|
|
226
|
+
return
|
|
227
|
+
entries = []
|
|
228
|
+
try:
|
|
229
|
+
for path in directory.iterdir():
|
|
230
|
+
if not path.name.endswith(".json") or path.name.startswith("."):
|
|
231
|
+
continue
|
|
232
|
+
try:
|
|
233
|
+
entries.append((path.lstat().st_mtime_ns, path.name, path))
|
|
234
|
+
except FileNotFoundError:
|
|
235
|
+
# expected-miss: another registrar can acknowledge an enumerated UUID.
|
|
236
|
+
continue
|
|
237
|
+
except OSError as exc:
|
|
238
|
+
raise DoyouknowError("pending queue is unreadable") from exc
|
|
239
|
+
for _, _, path in sorted(entries):
|
|
240
|
+
try:
|
|
241
|
+
path.lstat()
|
|
242
|
+
except FileNotFoundError:
|
|
243
|
+
# expected-miss: an earlier batch can acknowledge this queued UUID.
|
|
244
|
+
continue
|
|
245
|
+
if source is None:
|
|
246
|
+
yield path.stem
|
|
247
|
+
continue
|
|
248
|
+
try:
|
|
249
|
+
ref = read_ticket(path.stem, home=home)
|
|
250
|
+
except JsonBoundaryError as exc:
|
|
251
|
+
if exc.reason == "missing":
|
|
252
|
+
# expected-miss: acknowledgment can race with the source-filter read.
|
|
253
|
+
continue
|
|
254
|
+
yield path.stem
|
|
255
|
+
continue
|
|
256
|
+
except (DoyouknowError, OSError):
|
|
257
|
+
yield path.stem
|
|
258
|
+
continue
|
|
259
|
+
if ref.kind == source:
|
|
260
|
+
yield path.stem
|