okstra 0.231.1 → 0.233.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/README.md +19 -271
  2. package/dist/cli-registry.mjs +14 -1
  3. package/dist/cli-registry.mjs.map +1 -1
  4. package/dist/commands/execute/render-bundle.d.mts +6 -1
  5. package/dist/commands/execute/render-bundle.mjs +139 -32
  6. package/dist/commands/execute/render-bundle.mjs.map +1 -1
  7. package/dist/commands/lifecycle/config.mjs +6 -74
  8. package/dist/commands/lifecycle/config.mjs.map +1 -1
  9. package/dist/commands/memory/memory.mjs +9 -55
  10. package/dist/commands/memory/memory.mjs.map +1 -1
  11. package/dist/commands/trace/capacity.d.mts +12 -0
  12. package/dist/commands/trace/capacity.mjs +21 -0
  13. package/dist/commands/trace/capacity.mjs.map +1 -0
  14. package/dist/commands/trace/model.d.mts +89 -0
  15. package/dist/commands/trace/model.mjs +251 -0
  16. package/dist/commands/trace/model.mjs.map +1 -0
  17. package/dist/commands/trace/progression.d.mts +11 -0
  18. package/dist/commands/trace/progression.mjs +194 -0
  19. package/dist/commands/trace/progression.mjs.map +1 -0
  20. package/dist/commands/trace/review.d.mts +7 -0
  21. package/dist/commands/trace/review.mjs +51 -0
  22. package/dist/commands/trace/review.mjs.map +1 -0
  23. package/dist/commands/trace/task-link.d.mts +18 -0
  24. package/dist/commands/trace/task-link.mjs +72 -0
  25. package/dist/commands/trace/task-link.mjs.map +1 -0
  26. package/dist/commands/trace/trace.d.mts +1 -0
  27. package/dist/commands/trace/trace.mjs +293 -0
  28. package/dist/commands/trace/trace.mjs.map +1 -0
  29. package/dist/lib/bounded-file.d.mts +1 -0
  30. package/dist/lib/bounded-file.mjs +36 -0
  31. package/dist/lib/bounded-file.mjs.map +1 -0
  32. package/dist/lib/config-store.d.mts +34 -0
  33. package/dist/lib/config-store.mjs +70 -0
  34. package/dist/lib/config-store.mjs.map +1 -0
  35. package/dist/lib/file-lock.mjs +4 -5
  36. package/dist/lib/file-lock.mjs.map +1 -1
  37. package/dist/lib/input-limit.d.mts +4 -0
  38. package/dist/lib/input-limit.mjs +8 -0
  39. package/dist/lib/input-limit.mjs.map +1 -0
  40. package/dist/lib/memory-auto-register.d.mts +1 -0
  41. package/dist/lib/memory-auto-register.mjs +98 -0
  42. package/dist/lib/memory-auto-register.mjs.map +1 -0
  43. package/dist/lib/memory-entry-contract.d.mts +12 -0
  44. package/dist/lib/memory-entry-contract.mjs +46 -0
  45. package/dist/lib/memory-entry-contract.mjs.map +1 -0
  46. package/dist/lib/python-helper.d.mts +1 -1
  47. package/dist/lib/python-helper.mjs +3 -2
  48. package/dist/lib/python-helper.mjs.map +1 -1
  49. package/dist/lib/record-secrets.d.mts +1 -0
  50. package/dist/lib/record-secrets.mjs +28 -0
  51. package/dist/lib/record-secrets.mjs.map +1 -0
  52. package/dist/lib/report-language-tag.d.mts +1 -0
  53. package/dist/lib/report-language-tag.mjs +4 -0
  54. package/dist/lib/report-language-tag.mjs.map +1 -0
  55. package/dist/lib/report-language.d.mts +1 -0
  56. package/dist/lib/report-language.mjs +50 -0
  57. package/dist/lib/report-language.mjs.map +1 -0
  58. package/dist/lib/skill-catalog.mjs +3 -0
  59. package/dist/lib/skill-catalog.mjs.map +1 -1
  60. package/dist/lib/types.d.mts +2 -1
  61. package/docs/architecture/storage-model.md +163 -5
  62. package/docs/architecture.md +17 -4
  63. package/docs/cli.md +511 -0
  64. package/docs/contributor-change-matrix.md +1 -2
  65. package/docs/project-structure-overview.md +33 -42
  66. package/package.json +1 -1
  67. package/runtime/BUILD.json +2 -2
  68. package/runtime/agents/common.json +8 -8
  69. package/runtime/agents/operations/trace-interface-review.json +6 -0
  70. package/runtime/agents/operations/trace-scope-review.json +6 -0
  71. package/runtime/agents/roles/verifier.json +2 -4
  72. package/runtime/bin/okstra-provider-exec.py +10 -4
  73. package/runtime/prompts/coding-preflight/architectures/hexagonal.md +23 -30
  74. package/runtime/prompts/coding-preflight/authoring-checks.md +17 -0
  75. package/runtime/prompts/coding-preflight/clean-code.md +88 -286
  76. package/runtime/prompts/coding-preflight/frameworks/node-server.md +2 -3
  77. package/runtime/prompts/coding-preflight/languages/java.md +3 -5
  78. package/runtime/prompts/coding-preflight/languages/javascript-typescript.md +5 -12
  79. package/runtime/prompts/coding-preflight/languages/kotlin.md +2 -4
  80. package/runtime/prompts/coding-preflight/languages/python.md +2 -7
  81. package/runtime/prompts/coding-preflight/languages/rust.md +2 -8
  82. package/runtime/prompts/coding-preflight/languages/sql.md +2 -4
  83. package/runtime/prompts/coding-preflight/overview.md +25 -38
  84. package/runtime/prompts/coding-preflight/review-rules.json +30 -0
  85. package/runtime/prompts/duties/code-reviewer.json +15 -18
  86. package/runtime/prompts/duties/implementation-verifier.json +1 -0
  87. package/runtime/prompts/duties/interface-reviewer.json +32 -0
  88. package/runtime/prompts/host-orchestration/README.md +18 -9
  89. package/runtime/prompts/launch.template.md +3 -2
  90. package/runtime/prompts/lead/context-loader.md +2 -0
  91. package/runtime/prompts/lead/okstra-lead-contract.md +7 -7
  92. package/runtime/prompts/lead/report-writer.md +6 -6
  93. package/runtime/prompts/profiles/_coding-conventions-preflight.md +41 -6
  94. package/runtime/prompts/profiles/_common-contract.md +1 -1
  95. package/runtime/python/okstra_ctl/agent/invocation.py +122 -36
  96. package/runtime/python/okstra_ctl/agent/prompt_cli/cli.py +72 -26
  97. package/runtime/python/okstra_ctl/agent/prompt_cli/corrections.py +23 -8
  98. package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +58 -14
  99. package/runtime/python/okstra_ctl/agent/prompt_cli/results.py +8 -11
  100. package/runtime/python/okstra_ctl/agent/standalone.py +15 -0
  101. package/runtime/python/okstra_ctl/analysis_packet.py +24 -30
  102. package/runtime/python/okstra_ctl/attempt_evidence.py +10 -0
  103. package/runtime/python/okstra_ctl/code_review_census.py +162 -7
  104. package/runtime/python/okstra_ctl/code_review_coverage.py +4 -274
  105. package/runtime/python/okstra_ctl/code_review_integrity.py +154 -0
  106. package/runtime/python/okstra_ctl/code_review_invocation.py +212 -0
  107. package/runtime/python/okstra_ctl/code_review_prepare.py +381 -0
  108. package/runtime/python/okstra_ctl/code_review_run.py +384 -0
  109. package/runtime/python/okstra_ctl/code_review_scope.py +49 -9
  110. package/runtime/python/okstra_ctl/code_review_supervisor.py +41 -0
  111. package/runtime/python/okstra_ctl/code_review_target.py +212 -13
  112. package/runtime/python/okstra_ctl/consumers.py +12 -9
  113. package/runtime/python/okstra_ctl/coverage_census.py +11 -2
  114. package/runtime/python/okstra_ctl/design_review.py +207 -0
  115. package/runtime/python/okstra_ctl/design_review_evidence.py +297 -0
  116. package/runtime/python/okstra_ctl/design_snapshot.py +16 -9
  117. package/runtime/python/okstra_ctl/direct_work.py +45 -3
  118. package/runtime/python/okstra_ctl/dispatch_core.py +31 -15
  119. package/runtime/python/okstra_ctl/dispatch_state.py +9 -2
  120. package/runtime/python/okstra_ctl/doctor.py +8 -25
  121. package/runtime/python/okstra_ctl/domain/role.py +1 -0
  122. package/runtime/python/okstra_ctl/doyouknow/__init__.py +9 -0
  123. package/runtime/python/okstra_ctl/doyouknow/cli.py +91 -0
  124. package/runtime/python/okstra_ctl/doyouknow/config.py +58 -0
  125. package/runtime/python/okstra_ctl/doyouknow/contracts.py +184 -0
  126. package/runtime/python/okstra_ctl/doyouknow/identity.py +22 -0
  127. package/runtime/python/okstra_ctl/doyouknow/paths.py +22 -0
  128. package/runtime/python/okstra_ctl/doyouknow/pending.py +72 -0
  129. package/runtime/python/okstra_ctl/doyouknow/registration.py +260 -0
  130. package/runtime/python/okstra_ctl/doyouknow/source_contracts.py +128 -0
  131. package/runtime/python/okstra_ctl/doyouknow/source_store.py +172 -0
  132. package/runtime/python/okstra_ctl/doyouknow/sources/io.py +36 -0
  133. package/runtime/python/okstra_ctl/doyouknow/sources/manager.py +300 -0
  134. package/runtime/python/okstra_ctl/doyouknow/sources/memory.py +179 -0
  135. package/runtime/python/okstra_ctl/doyouknow/sources/task.py +95 -0
  136. package/runtime/python/okstra_ctl/doyouknow/store.py +260 -0
  137. package/runtime/python/okstra_ctl/doyouknow/sync.py +58 -0
  138. package/runtime/python/okstra_ctl/execution_mutation_audit.py +18 -2
  139. package/runtime/python/okstra_ctl/file_size_cap.py +3 -4
  140. package/runtime/python/okstra_ctl/final_report_schema.py +37 -34
  141. package/runtime/python/okstra_ctl/incremental_carry.py +6 -7
  142. package/runtime/python/okstra_ctl/initial_prompt_materialization.py +9 -2
  143. package/runtime/python/okstra_ctl/json_boundary.py +51 -5
  144. package/runtime/python/okstra_ctl/jsonl.py +21 -0
  145. package/runtime/python/okstra_ctl/manager_cli.py +148 -1
  146. package/runtime/python/okstra_ctl/manager_journal.py +126 -0
  147. package/runtime/python/okstra_ctl/manager_launch.py +3 -3
  148. package/runtime/python/okstra_ctl/manager_log.py +221 -0
  149. package/runtime/python/okstra_ctl/manager_paths.py +8 -0
  150. package/runtime/python/okstra_ctl/manager_registration.py +29 -0
  151. package/runtime/python/okstra_ctl/manager_resume.py +59 -0
  152. package/runtime/python/okstra_ctl/manager_split.py +2 -2
  153. package/runtime/python/okstra_ctl/manager_store.py +116 -17
  154. package/runtime/python/okstra_ctl/manager_sync.py +45 -25
  155. package/runtime/python/okstra_ctl/manager_view.py +35 -2
  156. package/runtime/python/okstra_ctl/material.py +2 -0
  157. package/runtime/python/okstra_ctl/model_cli.py +64 -14
  158. package/runtime/python/okstra_ctl/model_defaults.py +22 -0
  159. package/runtime/python/okstra_ctl/model_io/renderers.py +16 -2
  160. package/runtime/python/okstra_ctl/model_io_cli.py +7 -1
  161. package/runtime/python/okstra_ctl/operation_invocation.py +207 -28
  162. package/runtime/python/okstra_ctl/path_hints.py +14 -3
  163. package/runtime/python/okstra_ctl/paths.py +6 -1
  164. package/runtime/python/okstra_ctl/phases/catalog.py +24 -7
  165. package/runtime/python/okstra_ctl/phases/final_verification/validation.py +20 -0
  166. package/runtime/python/okstra_ctl/phases/implementation/host-rules.md +3 -3
  167. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +10 -0
  168. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-diff-review.md +23 -8
  169. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-self-check.md +6 -4
  170. package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +38 -10
  171. package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation.template.html +13 -0
  172. package/runtime/python/okstra_ctl/phases/implementation/spec.md +25 -0
  173. package/runtime/python/okstra_ctl/phases/implementation/validation.py +6 -0
  174. package/runtime/python/okstra_ctl/phases/implementation_option_selection/profile.md +1 -1
  175. package/runtime/python/okstra_ctl/phases/implementation_option_selection/votes.py +11 -11
  176. package/runtime/python/okstra_ctl/phases/implementation_planning/authoring.py +16 -9
  177. package/runtime/python/okstra_ctl/phases/implementation_planning/host-rules.md +3 -6
  178. package/runtime/python/okstra_ctl/phases/implementation_planning/instructions/plan-body-verification.md +12 -10
  179. package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +6 -6
  180. package/runtime/python/okstra_ctl/phases/improvement_discovery/lenses.py +12 -0
  181. package/runtime/python/okstra_ctl/phases/improvement_discovery/profile.md +7 -6
  182. package/runtime/python/okstra_ctl/phases/improvement_discovery/resolved_census.py +159 -0
  183. package/runtime/python/okstra_ctl/phases/improvement_discovery/spec.md +5 -2
  184. package/runtime/python/okstra_ctl/phases/improvement_discovery/validation.py +21 -7
  185. package/runtime/python/okstra_ctl/phases/release_handoff/boundary.json +1 -1
  186. package/runtime/python/okstra_ctl/phases/release_handoff/profile.md +4 -1
  187. package/runtime/python/okstra_ctl/plan_verify_cli.py +6 -4
  188. package/runtime/python/okstra_ctl/render.py +28 -2
  189. package/runtime/python/okstra_ctl/report_assembly.py +53 -11
  190. package/runtime/python/okstra_ctl/report_contract.py +26 -0
  191. package/runtime/python/okstra_ctl/report_corrections.py +36 -20
  192. package/runtime/python/okstra_ctl/report_inputs.py +1 -0
  193. package/runtime/python/okstra_ctl/report_language.py +4 -1
  194. package/runtime/python/okstra_ctl/report_narrative.py +182 -7
  195. package/runtime/python/okstra_ctl/report_projections.py +64 -18
  196. package/runtime/python/okstra_ctl/report_synthesis_packet.py +97 -38
  197. package/runtime/python/okstra_ctl/review_policy.py +359 -0
  198. package/runtime/python/okstra_ctl/review_verdicts.py +291 -0
  199. package/runtime/python/okstra_ctl/run.py +67 -36
  200. package/runtime/python/okstra_ctl/run_context.py +10 -1
  201. package/runtime/python/okstra_ctl/set_work_status.py +4 -15
  202. package/runtime/python/okstra_ctl/stage_done.py +11 -0
  203. package/runtime/python/okstra_ctl/standalone_review_audit.py +28 -0
  204. package/runtime/python/okstra_ctl/task_list_cli.py +34 -8
  205. package/runtime/python/okstra_ctl/task_query.py +239 -0
  206. package/runtime/python/okstra_ctl/trace_review.py +264 -0
  207. package/runtime/python/okstra_ctl/trace_tasks.py +197 -0
  208. package/runtime/python/okstra_ctl/user_authorization.py +14 -0
  209. package/runtime/python/okstra_ctl/verification_generation.py +23 -1
  210. package/runtime/python/okstra_ctl/verification_input_types.py +15 -1
  211. package/runtime/python/okstra_ctl/verification_inputs.py +28 -6
  212. package/runtime/python/okstra_ctl/verification_invocation.py +7 -1
  213. package/runtime/python/okstra_ctl/verification_target.py +2 -0
  214. package/runtime/python/okstra_ctl/wizard/confirmation.py +7 -3
  215. package/runtime/python/okstra_ctl/wizard/launch.py +52 -0
  216. package/runtime/python/okstra_ctl/wizard/launch_cli.py +39 -0
  217. package/runtime/python/okstra_ctl/wizard/outcome.py +17 -16
  218. package/runtime/python/okstra_ctl/wizard/registry.py +1 -1
  219. package/runtime/python/okstra_ctl/wizard/render.py +8 -0
  220. package/runtime/python/okstra_ctl/wizard/state.py +2 -1
  221. package/runtime/python/okstra_ctl/worker_prompt_headers.py +24 -9
  222. package/runtime/python/okstra_ctl/worker_runner.py +66 -10
  223. package/runtime/python/okstra_ctl/write_policy.py +34 -1
  224. package/runtime/python/okstra_project/__init__.py +2 -0
  225. package/runtime/python/okstra_project/dirs.py +18 -0
  226. package/runtime/python/okstra_project/state.py +45 -20
  227. package/runtime/python/okstra_token_usage/blocks.py +74 -0
  228. package/runtime/python/okstra_token_usage/cli.py +23 -10
  229. package/runtime/python/okstra_token_usage/collect.py +25 -22
  230. package/runtime/python/okstra_token_usage/policy.py +193 -0
  231. package/runtime/schemas/doyouknow-config-v1.schema.json +20 -0
  232. package/runtime/schemas/doyouknow-v1.schema.json +65 -0
  233. package/runtime/schemas/final-report-v2.0.schema.json +157 -0
  234. package/runtime/schemas/final-report-v3.0.schema.json +157 -0
  235. package/runtime/schemas/memory-entry-contract-v1.json +11 -0
  236. package/runtime/schemas/record-secret-patterns.json +41 -0
  237. package/runtime/schemas/report-synthesis-packet-v1.0.schema.json +1 -1
  238. package/runtime/skills/_fragments/wizard-confirm-step.md +1 -1
  239. package/runtime/skills/_fragments/wizard-preflight.md +1 -1
  240. package/runtime/skills/_fragments/wizard-prompt-loop.md +3 -5
  241. package/runtime/skills/_fragments/wizard-relay-protocol.md +1 -1
  242. package/runtime/skills/_fragments/wizard-render-bundle.md +15 -29
  243. package/runtime/skills/okstra-brief-gen/SKILL.md +7 -7
  244. package/runtime/skills/okstra-brief-gen/references/tracker-recursion.md +1 -1
  245. package/runtime/skills/okstra-code-review/SKILL.md +92 -136
  246. package/runtime/skills/okstra-code-review/references/census-rules.md +29 -105
  247. package/runtime/skills/okstra-code-review/references/review-calibration.md +37 -61
  248. package/runtime/skills/okstra-discussion/SKILL.md +22 -38
  249. package/runtime/skills/okstra-explain-flow/SKILL.md +6 -2
  250. package/runtime/skills/okstra-manager/SKILL.md +31 -0
  251. package/runtime/skills/okstra-memory/SKILL.md +8 -1
  252. package/runtime/skills/okstra-pr-gen/SKILL.md +1 -1
  253. package/runtime/skills/okstra-run/SKILL.md +55 -183
  254. package/runtime/skills/okstra-schedule-gen/SKILL.md +9 -13
  255. package/runtime/skills/okstra-setup/SKILL.md +3 -2
  256. package/runtime/skills/okstra-setup/references/project-config.md +2 -4
  257. package/runtime/skills/okstra-tdd/SKILL.md +187 -0
  258. package/runtime/skills/okstra-user-response/SKILL.md +4 -2
  259. package/runtime/templates/implementation-worker-preamble.md +4 -9
  260. package/runtime/templates/manager/view.template.html +4 -0
  261. package/runtime/templates/report-writer-prompt-preamble.md +15 -3
  262. package/runtime/templates/trace/trace.template.md +42 -0
  263. package/runtime/templates/worker-prompt-preamble.md +4 -12
  264. package/runtime/validators/checks/fixtures-02.py +25 -33
  265. package/runtime/validators/validate-run.py +107 -38
@@ -0,0 +1,260 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Iterator
4
+ from dataclasses import dataclass
5
+ import os
6
+ from pathlib import Path
7
+
8
+ from ..json_boundary import JsonBoundaryError, load_owned_object, write_owned_object_atomic
9
+ from ..run_context import dir_flock
10
+ from .config import SourceConfig, load_config
11
+ from .contracts import DoyouknowError, SecretRefusal
12
+ from .identity import source_id
13
+ from .paths import database_path, ensure_storage_root, require_database_location
14
+ from .pending import MAX_TICKET_BYTES, pending_path, read_ticket
15
+ from .source_contracts import PreparedSource, SourceKind, SourceRef, validate_prepared_snapshot, validate_source_value
16
+ from .sources.memory import read_memory_source
17
+
18
+ MAX_FAILURES = 3
19
+
20
+
21
+ @dataclass
22
+ class RegistrationState:
23
+ fingerprint: str | None = None
24
+ attempts: int = 0
25
+ status: str = "pending"
26
+ errorCode: str | None = None
27
+ refusedFingerprint: str | None = None
28
+ preparationAttempts: int = 0
29
+
30
+
31
+ def _read_state(path: Path) -> tuple[RegistrationState, dict | None]:
32
+ require_database_location(path)
33
+ if not path.exists():
34
+ return RegistrationState(), None
35
+ previous = load_owned_object(path, artifact="registration state", max_bytes=MAX_TICKET_BYTES)
36
+ validate_source_value(previous)
37
+ raw = {"preparationAttempts": 0, **previous}
38
+ strings = ("fingerprint", "errorCode", "refusedFingerprint")
39
+ if set(raw) != {*strings, "attempts", "status", "preparationAttempts"} or any(raw[key] is not None and not isinstance(raw[key], str) for key in strings):
40
+ raise DoyouknowError("invalid registration state")
41
+ if type(raw["attempts"]) is not int or not 0 <= raw["attempts"] <= MAX_FAILURES:
42
+ raise DoyouknowError("invalid registration state")
43
+ if type(raw["preparationAttempts"]) is not int or not 0 <= raw["preparationAttempts"] <= MAX_FAILURES:
44
+ raise DoyouknowError("invalid registration state")
45
+ if raw["status"] not in ("registered", "pending", "refused", "exhausted"):
46
+ raise DoyouknowError("invalid registration state")
47
+ if raw["errorCode"] not in (None, "registration-unavailable", "source-unavailable", "source-refused", "source-preparation-unavailable", "source-preparation-exhausted"):
48
+ raise DoyouknowError("invalid registration state")
49
+ return RegistrationState(**raw), previous
50
+
51
+
52
+ def _result(ticket: str | None, ref: SourceRef | None, state: RegistrationState) -> dict:
53
+ return {"ok": state.status in ("registered", "excluded"), "ticket": ticket,
54
+ "sourceId": source_id(ref) if ref else None, "status": state.status,
55
+ "attempts": state.attempts, "preparationAttempts": state.preparationAttempts, "errorCode": state.errorCode}
56
+
57
+
58
+ def register_ticket(ticket: str, *, home: Path | None = None, manual: bool = False) -> dict:
59
+ ref = None
60
+ safe_ticket = None
61
+ try:
62
+ pending_path(ticket, home=home)
63
+ safe_ticket = ticket
64
+ ref = read_ticket(ticket, home=home)
65
+ directory = database_path(home).parent / "registrations"
66
+ require_database_location(directory / "state.json")
67
+ require_database_location(directory / f".{source_id(ref)}.lock")
68
+ ensure_storage_root(directory.parent)
69
+ with dir_flock(directory, f".{source_id(ref)}.lock"):
70
+ if not pending_path(ticket, home=home).exists():
71
+ return _result(ticket, ref, RegistrationState(status="registered"))
72
+ return _register_locked(ticket, ref, directory / f"{source_id(ref)}.json", home, manual)
73
+ except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
74
+ return _result(safe_ticket, ref, RegistrationState(errorCode="registration-unavailable"))
75
+
76
+
77
+ def _register_locked(ticket: str, ref: SourceRef, path: Path, home: Path | None, manual: bool) -> dict:
78
+ settings = load_config(home=home)
79
+ if (not settings["autoRegister"] and not manual) or ref.kind not in settings["sources"]:
80
+ pending_path(ticket, home=home).unlink(missing_ok=True)
81
+ return _result(ticket, ref, RegistrationState(status="excluded"))
82
+ if ref.kind not in ("memory", "manager"):
83
+ return _result(ticket, ref, RegistrationState(errorCode="unsupported-registration-source"))
84
+ state, previous = _read_state(path)
85
+ if state.preparationAttempts >= MAX_FAILURES and not manual:
86
+ state.status, state.errorCode = "exhausted", "source-preparation-exhausted"
87
+ write_owned_object_atomic(path, vars(state), artifact="registration state")
88
+ return _result(ticket, ref, state)
89
+ state.preparationAttempts = min(state.preparationAttempts + 1, MAX_FAILURES)
90
+ write_owned_object_atomic(path, vars(state), artifact="registration state")
91
+ try:
92
+ prepared = _prepare_source(ref, database_path(home).parent.parent, settings)
93
+ except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
94
+ state.status = "exhausted" if state.preparationAttempts >= MAX_FAILURES else "pending"
95
+ state.errorCode = "source-preparation-unavailable"
96
+ write_owned_object_atomic(path, vars(state), artifact="registration state")
97
+ return _result(ticket, ref, state)
98
+ state.preparationAttempts = 0
99
+ if prepared.outcome == "excluded":
100
+ if previous is None:
101
+ path.unlink(missing_ok=True)
102
+ else:
103
+ write_owned_object_atomic(path, previous, artifact="registration state")
104
+ pending_path(ticket, home=home).unlink(missing_ok=True)
105
+ return _result(ticket, ref, RegistrationState(status="excluded"))
106
+ write_owned_object_atomic(path, vars(state), artifact="registration state")
107
+ if prepared.ref != ref:
108
+ raise DoyouknowError("registration reference does not match its source")
109
+ return _register_prepared(ticket, ref, prepared, path, home, manual)
110
+
111
+
112
+ def _prepare_source(ref: SourceRef, home: Path, settings: SourceConfig) -> PreparedSource:
113
+ if ref.kind == "memory":
114
+ return read_memory_source(home, ref, settings)
115
+ from .sources.manager import prepare_manager_source
116
+
117
+ if ref.native_id != "$manager":
118
+ raise DoyouknowError("invalid manager registration reference")
119
+ return prepare_manager_source(home, ref.owner_id, settings)
120
+
121
+
122
+ def _register_prepared(ticket: str, ref: SourceRef, prepared: PreparedSource, path: Path, home: Path | None, manual: bool) -> dict:
123
+ state, _ = _read_state(path)
124
+ if prepared.fingerprint is not None and prepared.fingerprint == state.refusedFingerprint:
125
+ state.status, state.errorCode = "refused", "source-refused"
126
+ return _result(ticket, ref, state)
127
+ if prepared.fingerprint is not None and state.fingerprint != prepared.fingerprint:
128
+ state.fingerprint, state.attempts = prepared.fingerprint, 0
129
+ if state.attempts >= MAX_FAILURES and not manual:
130
+ state.status = "exhausted"
131
+ return _result(ticket, ref, state)
132
+ state.attempts = min(state.attempts + 1, MAX_FAILURES)
133
+ state.status, state.errorCode = "pending", None
134
+ write_owned_object_atomic(path, vars(state), artifact="registration state")
135
+ try:
136
+ _store_prepared(prepared, home)
137
+ except SecretRefusal:
138
+ state.status, state.errorCode = "refused", "source-refused"
139
+ state.refusedFingerprint = prepared.fingerprint
140
+ except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
141
+ state.status = "exhausted" if state.attempts >= MAX_FAILURES else "pending"
142
+ state.errorCode = "source-unavailable"
143
+ else:
144
+ state.status, state.attempts, state.errorCode, state.refusedFingerprint = "registered", 0, None, None
145
+ write_owned_object_atomic(path, vars(state), artifact="registration state")
146
+ if state.status == "registered":
147
+ pending_path(ticket, home=home).unlink(missing_ok=True)
148
+ return _result(ticket, ref, state)
149
+
150
+
151
+ def _store_prepared(prepared: PreparedSource, home: Path | None) -> None:
152
+ if prepared.outcome != "ready":
153
+ raise DoyouknowError("source unavailable")
154
+ if prepared.scan is not None:
155
+ from .sources.manager import gate_manager_scan
156
+ from .source_store import apply_source_scan
157
+
158
+ scan = gate_manager_scan(prepared.scan, gate=lambda candidate: validate_source_and_update_refusal_cache(candidate, home=home))
159
+ refused = "manager-secret-refused" in scan.errors
160
+ if scan.status != "unavailable":
161
+ apply_source_scan(scan, home=home)
162
+ if refused:
163
+ raise SecretRefusal("manager source refused")
164
+ if scan.status != "complete":
165
+ raise DoyouknowError("source scan incomplete")
166
+ elif prepared.snapshot is not None:
167
+ if not validate_source_and_update_refusal_cache(prepared, home=home):
168
+ raise SecretRefusal("source refused")
169
+ from .source_store import store_source_snapshot
170
+
171
+ store_source_snapshot(prepared.snapshot, home=home)
172
+ else:
173
+ raise DoyouknowError("source snapshot unavailable")
174
+
175
+
176
+ def validate_source_and_update_refusal_cache(prepared: PreparedSource, *, home: Path | None = None) -> bool:
177
+ if prepared.outcome != "ready" or prepared.snapshot is None or prepared.fingerprint is None:
178
+ raise DoyouknowError("source snapshot unavailable")
179
+ database = database_path(home)
180
+ require_database_location(database)
181
+ directory = database.parent / "refusals"
182
+ path = directory / f"{source_id(prepared.ref)}.json"
183
+ require_database_location(path)
184
+ require_database_location(directory / f".{source_id(prepared.ref)}.lock")
185
+ ensure_storage_root(directory.parent)
186
+ with dir_flock(directory, f".{source_id(prepared.ref)}.lock"):
187
+ if path.exists():
188
+ cached = load_owned_object(path, artifact="source refusal", max_bytes=MAX_TICKET_BYTES)
189
+ validate_source_value(cached)
190
+ if set(cached) != {"fingerprint"} or not isinstance(cached["fingerprint"], str):
191
+ raise DoyouknowError("invalid source refusal")
192
+ if cached["fingerprint"] == prepared.fingerprint:
193
+ return False
194
+ try:
195
+ validate_prepared_snapshot(prepared)
196
+ except SecretRefusal:
197
+ write_owned_object_atomic(path, {"fingerprint": prepared.fingerprint}, artifact="source refusal")
198
+ return False
199
+ path.unlink(missing_ok=True)
200
+ return True
201
+
202
+
203
+ def retry_pending(*, home: Path | None = None, source: SourceKind | None = None, limit: int = 100, manual: bool = False) -> dict:
204
+ if type(limit) is not int or not 1 <= limit <= 1000:
205
+ raise DoyouknowError("pending limit must be between 1 and 1000")
206
+ directory = database_path(home).parent / "pending"
207
+ require_database_location(directory / "ticket.json")
208
+ results = []
209
+ for ticket in _pending_candidates(directory, source, home):
210
+ if len(results) >= limit:
211
+ return {"ok": all(row["ok"] for row in results), "results": results, "truncated": True}
212
+ result = register_ticket(ticket, home=home, manual=manual)
213
+ results.append(result)
214
+ if not result["ok"]:
215
+ try:
216
+ os.utime(pending_path(ticket, home=home), follow_symlinks=False)
217
+ except FileNotFoundError:
218
+ continue
219
+ except (DoyouknowError, OSError):
220
+ result["errorCode"] = "pending-order-unavailable"
221
+ return {"ok": all(row["ok"] for row in results), "results": results, "truncated": False}
222
+
223
+
224
+ def _pending_candidates(directory: Path, source: SourceKind | None, home: Path | None) -> Iterator[str]:
225
+ if not directory.exists():
226
+ return
227
+ entries = []
228
+ try:
229
+ for path in directory.iterdir():
230
+ if not path.name.endswith(".json") or path.name.startswith("."):
231
+ continue
232
+ try:
233
+ entries.append((path.lstat().st_mtime_ns, path.name, path))
234
+ except FileNotFoundError:
235
+ # expected-miss: another registrar can acknowledge an enumerated UUID.
236
+ continue
237
+ except OSError as exc:
238
+ raise DoyouknowError("pending queue is unreadable") from exc
239
+ for _, _, path in sorted(entries):
240
+ try:
241
+ path.lstat()
242
+ except FileNotFoundError:
243
+ # expected-miss: an earlier batch can acknowledge this queued UUID.
244
+ continue
245
+ if source is None:
246
+ yield path.stem
247
+ continue
248
+ try:
249
+ ref = read_ticket(path.stem, home=home)
250
+ except JsonBoundaryError as exc:
251
+ if exc.reason == "missing":
252
+ # expected-miss: acknowledgment can race with the source-filter read.
253
+ continue
254
+ yield path.stem
255
+ continue
256
+ except (DoyouknowError, OSError):
257
+ yield path.stem
258
+ continue
259
+ if ref.kind == source:
260
+ yield path.stem
@@ -0,0 +1,128 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from dataclasses import asdict, dataclass, field
5
+ from pathlib import Path
6
+ from typing import Literal
7
+
8
+ from ..json_boundary import serialize_owned_object
9
+ from .contracts import (DoyouknowError, MAX_INPUT_BYTES, Node, Relation, _check_input_depth,
10
+ _decoded_strings, asset_object, canonical_json, record_secret_policy_digest, reject_secrets)
11
+ from .identity import stable_id
12
+
13
+ SourceKind = Literal["memory", "manager", "task"]
14
+ Lifecycle = Literal["active", "archived", "unlinked", "missing"]
15
+ JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class SourceRef:
20
+ kind: SourceKind
21
+ owner_id: str
22
+ native_id: str
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class SourceDomain:
27
+ kind: SourceKind
28
+ owner_id: str
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class SourceSnapshot:
33
+ ref: SourceRef
34
+ primary_node: Node
35
+ locator: str
36
+ metadata: dict[str, JsonValue] = field(default_factory=dict)
37
+ nodes: tuple[Node, ...] = ()
38
+ relations: tuple[Relation, ...] = ()
39
+ lifecycle: Lifecycle = "active"
40
+ source_updated_at: str | None = None
41
+ scope: str | None = None
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class SourceScan:
46
+ domain: SourceDomain
47
+ snapshots: tuple[SourceSnapshot, ...]
48
+ observed_refs: tuple[SourceRef, ...]
49
+ status: Literal["complete", "partial", "unavailable"]
50
+ errors: tuple[str, ...] = ()
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class PreparedSource:
55
+ ref: SourceRef
56
+ fingerprint: str | None
57
+ snapshot: SourceSnapshot | None = field(default=None, repr=False)
58
+ outcome: Literal["ready", "excluded", "unavailable"] = "ready"
59
+ secret_texts: tuple[str, ...] = field(default=(), repr=False)
60
+ scan: SourceScan | None = field(default=None, repr=False)
61
+ error_code: str | None = None
62
+
63
+
64
+ def prepared_snapshot(snapshot: SourceSnapshot) -> PreparedSource:
65
+ return PreparedSource(snapshot.ref, stable_id("source-content", canonical_json(asdict(snapshot)), record_secret_policy_digest()), snapshot)
66
+
67
+
68
+ def validate_prepared_snapshot(prepared: PreparedSource) -> None:
69
+ if prepared.outcome != "ready" or prepared.snapshot is None:
70
+ raise DoyouknowError("source snapshot unavailable")
71
+ for text in prepared.secret_texts:
72
+ reject_secrets(text)
73
+ snapshot_payload(prepared.snapshot)
74
+
75
+
76
+ def validate_source_value(value: object) -> str:
77
+ _check_input_depth(value)
78
+ try:
79
+ encoded = canonical_json(value)
80
+ except (ValueError, TypeError) as exc:
81
+ raise DoyouknowError("source input is not a JSON value") from exc
82
+ if len(encoded.encode("utf-8")) > MAX_INPUT_BYTES:
83
+ raise DoyouknowError("source input exceeds 2000000 bytes")
84
+ reject_secrets("\n".join(_decoded_strings(value)))
85
+ return encoded
86
+
87
+
88
+ def snapshot_payload(snapshot: SourceSnapshot) -> dict:
89
+ payload = json.loads(validate_source_value({
90
+ "ref": {"kind": snapshot.ref.kind, "owner_id": snapshot.ref.owner_id, "native_id": snapshot.ref.native_id},
91
+ "primary_node": snapshot.primary_node, "locator": snapshot.locator,
92
+ "metadata": snapshot.metadata, "nodes": [snapshot.primary_node, *snapshot.nodes],
93
+ "relations": list(snapshot.relations), "lifecycle": snapshot.lifecycle,
94
+ "source_updated_at": snapshot.source_updated_at, "scope": snapshot.scope,
95
+ }))
96
+ _require_snapshot_fields(snapshot)
97
+ _require_snapshot_graph(payload)
98
+ return payload
99
+
100
+
101
+ def _require_snapshot_fields(snapshot: SourceSnapshot) -> None:
102
+ if snapshot.ref.kind not in ("memory", "manager", "task") or snapshot.lifecycle not in ("active", "archived", "unlinked", "missing"):
103
+ raise DoyouknowError("invalid source kind or lifecycle")
104
+ for value in (snapshot.ref.owner_id, snapshot.ref.native_id, snapshot.locator):
105
+ if not isinstance(value, str) or not value.strip():
106
+ raise DoyouknowError("source identity and locator must be nonempty strings")
107
+ if snapshot.scope is not None and (not isinstance(snapshot.scope, str) or not snapshot.scope.strip()):
108
+ raise DoyouknowError("source scope must be a nonempty string or null")
109
+ if snapshot.source_updated_at is not None and not isinstance(snapshot.source_updated_at, str):
110
+ raise DoyouknowError("source update time must be a string or null")
111
+ if not isinstance(snapshot.metadata, dict):
112
+ raise DoyouknowError("source metadata must be an object")
113
+
114
+
115
+ def _require_snapshot_graph(payload: dict) -> None:
116
+ schema = asset_object("doyouknow-v1.schema.json")
117
+ if len(payload["nodes"]) > 1000 or len(payload["relations"]) > 4000:
118
+ raise DoyouknowError("source graph exceeds input limits")
119
+ for node in payload["nodes"]:
120
+ serialize_owned_object(Path("source-node.json"), node, artifact="source node", schema={**schema, **schema["$defs"]["node"]})
121
+ refs = [node.get("ref", node["key"]) for node in payload["nodes"]]
122
+ identities = [(node["kind"], node["scope"], node["key"]) for node in payload["nodes"]]
123
+ if "$source" in refs or "$record" in refs or len(refs) != len(set(refs)) or len(identities) != len(set(identities)):
124
+ raise DoyouknowError("source node references and identities must be unique")
125
+ for relation in payload["relations"]:
126
+ serialize_owned_object(Path("source-relation.json"), relation, artifact="source relation", schema={**schema, **schema["$defs"]["relation"]})
127
+ if relation["from"] not in {"$source", *refs} or relation["to"] not in {"$source", *refs}:
128
+ raise DoyouknowError("source relation endpoint is not in this snapshot")
@@ -0,0 +1,172 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import sqlite3
5
+ from contextlib import closing
6
+ from datetime import datetime, timezone
7
+ from pathlib import Path
8
+
9
+ from .contracts import DoyouknowError, MAX_INPUT_BYTES, canonical_json
10
+ from .identity import node_id, source_id, stable_id
11
+ from .source_contracts import SourceRef, SourceScan, SourceSnapshot, snapshot_payload, validate_source_value
12
+ from .store import SCHEMA_VERSION, SQLITE_TIMEOUT_SECONDS, _digest, _insert_node, _require_database_location, database_path, write_connection
13
+
14
+ SOURCE_SCHEMA_STATEMENTS = (
15
+ """create table if not exists source (
16
+ id text primary key, kind text not null, owner_id text not null,
17
+ native_id text not null, node_id text not null references node(id),
18
+ latest_snapshot text not null, scope text,
19
+ lifecycle text not null check(lifecycle in ('active', 'archived', 'unlinked', 'missing')),
20
+ checked_at text not null, attempted_at text not null,
21
+ scan_status text not null check(scan_status in ('complete', 'partial', 'unavailable')),
22
+ unique(kind, owner_id, native_id)
23
+ )""",
24
+ """create table if not exists source_snapshot (
25
+ id text primary key, source_id text not null references source(id),
26
+ previous_snapshot text references source_snapshot(id),
27
+ content_digest text not null, payload text not null, created_at text not null
28
+ )""",
29
+ """create table if not exists source_relation (
30
+ id text primary key, snapshot_id text not null references source_snapshot(id),
31
+ from_node text not null references node(id), to_node text not null references node(id),
32
+ kind text not null, reason text not null, evidence text not null,
33
+ standing text not null check(standing in ('observed', 'user-stated', 'candidate'))
34
+ )""",
35
+ "create index if not exists source_relation_snapshot on source_relation(snapshot_id)",
36
+ )
37
+
38
+
39
+ def store_source_snapshot(snapshot: SourceSnapshot, *, home: Path | None = None) -> dict:
40
+ payload = snapshot_payload(snapshot)
41
+ with write_connection(database_path(home)) as connection:
42
+ _write_source(connection, source_id(snapshot.ref), payload, datetime.now(timezone.utc).isoformat())
43
+ return {"ok": True, "sourceId": source_id(snapshot.ref)}
44
+
45
+
46
+ def apply_source_scan(scan: SourceScan, *, home: Path | None = None) -> dict:
47
+ payloads, refs = _validated_scan(scan)
48
+ now = datetime.now(timezone.utc).isoformat()
49
+ with write_connection(database_path(home)) as connection:
50
+ for snapshot, payload in zip(scan.snapshots, payloads):
51
+ _write_source(connection, source_id(snapshot.ref), payload, now)
52
+ _mark_unrefreshed_sources(connection, scan, refs, now)
53
+ return {"ok": True, "status": scan.status, "registered": len(payloads), "errors": list(scan.errors)}
54
+
55
+
56
+ def _validated_scan(scan: SourceScan) -> tuple[list[dict], list[SourceRef]]:
57
+ payloads = [snapshot_payload(snapshot) for snapshot in scan.snapshots]
58
+ refs = [*scan.observed_refs, *(snapshot.ref for snapshot in scan.snapshots)]
59
+ validate_source_value({"domain": [scan.domain.kind, scan.domain.owner_id], "refs": [[ref.kind, ref.owner_id, ref.native_id] for ref in refs], "errors": list(scan.errors)})
60
+ if scan.status not in ("complete", "partial", "unavailable"):
61
+ raise DoyouknowError("invalid source scan status")
62
+ if scan.status == "complete" and scan.errors:
63
+ raise DoyouknowError("a complete source scan cannot contain errors")
64
+ if scan.domain.kind not in ("memory", "manager", "task") or not scan.domain.owner_id.strip():
65
+ raise DoyouknowError("invalid source scan domain")
66
+ if any((ref.kind, ref.owner_id) != (scan.domain.kind, scan.domain.owner_id) for ref in refs):
67
+ raise DoyouknowError("source scan contains a different domain")
68
+ if any(not isinstance(ref.native_id, str) or not ref.native_id.strip() for ref in refs):
69
+ raise DoyouknowError("source identity must be a nonempty string")
70
+ if len({source_id(snapshot.ref) for snapshot in scan.snapshots}) != len(scan.snapshots):
71
+ raise DoyouknowError("source scan contains duplicate snapshots")
72
+ return payloads, refs
73
+
74
+
75
+ def _mark_unrefreshed_sources(connection: sqlite3.Connection, scan: SourceScan, refs: list[SourceRef], now: str) -> None:
76
+ seen = {source_id(ref) for ref in refs}
77
+ updated = {source_id(snapshot.ref) for snapshot in scan.snapshots}
78
+ rows = connection.execute("select id from source where kind = ? and owner_id = ?", (scan.domain.kind, scan.domain.owner_id)).fetchall()
79
+ for (identity,) in rows:
80
+ if identity in updated:
81
+ continue
82
+ if scan.status == "complete" and identity not in seen:
83
+ connection.execute("update source set lifecycle = 'missing', attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
84
+ else:
85
+ connection.execute("update source set attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
86
+
87
+
88
+ def _write_source(connection: sqlite3.Connection, identity: str, payload: dict, now: str) -> None:
89
+ previous = connection.execute("select source.latest_snapshot, source_snapshot.content_digest from source inner join source_snapshot on source.latest_snapshot = source_snapshot.id where source.id = ?", (identity,)).fetchone()
90
+ digest = _digest(payload)
91
+ if previous and previous[1] == digest:
92
+ connection.execute("update source set checked_at = ?, attempted_at = ?, scan_status = 'complete', lifecycle = ? where id = ?", (now, now, payload["lifecycle"], identity))
93
+ return
94
+ nodes = payload["nodes"]
95
+ for node in nodes:
96
+ _insert_node(connection, node_id(node), node["kind"], node["scope"], node["key"], node["title"])
97
+ snapshot_id = stable_id("source-snapshot", identity, previous[0] if previous else "", digest)
98
+ ref = payload["ref"]
99
+ connection.execute("insert into source (id, kind, owner_id, native_id, node_id, latest_snapshot, scope, lifecycle, checked_at, attempted_at, scan_status) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'complete') on conflict(id) do update set node_id=excluded.node_id, latest_snapshot=excluded.latest_snapshot, scope=excluded.scope, lifecycle=excluded.lifecycle, checked_at=excluded.checked_at, attempted_at=excluded.attempted_at, scan_status=excluded.scan_status", (identity, ref["kind"], ref["owner_id"], ref["native_id"], node_id(payload["primary_node"]), snapshot_id, payload["scope"], payload["lifecycle"], now, now))
100
+ connection.execute("insert into source_snapshot (id, source_id, previous_snapshot, content_digest, payload, created_at) values (?, ?, ?, ?, ?, ?)", (snapshot_id, identity, previous[0] if previous else None, digest, canonical_json(payload), now))
101
+ identities = {node.get("ref", node["key"]): node_id(node) for node in nodes}
102
+ identities["$source"] = node_id(payload["primary_node"])
103
+ for index, relation in enumerate(payload["relations"]):
104
+ connection.execute("insert into source_relation (id, snapshot_id, from_node, to_node, kind, reason, evidence, standing) values (?, ?, ?, ?, ?, ?, ?, ?)", (stable_id("source-relation", snapshot_id, str(index)), snapshot_id, identities[relation["from"]], identities[relation["to"]], relation["kind"], relation["reason"], canonical_json(relation["evidence"]), relation["standing"]))
105
+
106
+
107
+ def list_sources(*, home: Path | None = None, scope: str | None = None, kind: str | None = None, limit: int = 100, after: str | None = None) -> dict:
108
+ if not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= 1000:
109
+ raise DoyouknowError("source limit must be between 1 and 1000")
110
+ if after is not None and (not isinstance(after, str) or not after.strip()):
111
+ raise DoyouknowError("source cursor must be a nonempty string")
112
+ path = database_path(home)
113
+ _require_database_location(path)
114
+ if not path.is_file():
115
+ return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
116
+ try:
117
+ with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=SQLITE_TIMEOUT_SECONDS)) as connection:
118
+ connection.execute("begin")
119
+ version = connection.execute("pragma user_version").fetchone()[0]
120
+ if version == 1:
121
+ return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
122
+ if version != SCHEMA_VERSION:
123
+ raise DoyouknowError(f"unsupported doyouknow database version: {version}")
124
+ rows = connection.execute(
125
+ """select source.id, source.node_id, source.latest_snapshot,
126
+ source.lifecycle, source.checked_at, source.attempted_at,
127
+ source.scan_status, source_snapshot.payload
128
+ from source inner join source_snapshot
129
+ on source.latest_snapshot = source_snapshot.id
130
+ where source.id > ? and (? is null or source.kind = ?)
131
+ and (? is null or source.scope = ?)
132
+ order by source.id limit ?""",
133
+ (after or "", kind, kind, scope, scope, limit + 1),
134
+ )
135
+ return _source_page(connection, rows, limit)
136
+ except sqlite3.Error as exc:
137
+ raise DoyouknowError(f"doyouknow source lookup failed: {exc}") from exc
138
+
139
+
140
+ def _source_page(connection: sqlite3.Connection, rows: sqlite3.Cursor, limit: int) -> dict:
141
+ result = {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
142
+ item_bytes = 0
143
+ for row in rows:
144
+ if len(result["sources"]) >= limit:
145
+ result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
146
+ break
147
+ view = _source_view(connection, row)
148
+ view_bytes = len(canonical_json(view).encode("utf-8"))
149
+ header = {"ok": True, "sources": [], "truncated": True, "nextCursor": view["sourceId"]}
150
+ header_bytes = len(canonical_json(header).encode("utf-8"))
151
+ if header_bytes + view_bytes > MAX_INPUT_BYTES:
152
+ view = {key: view[key] for key in ("sourceId", "nodeId", "snapshotId", "lifecycle", "checkedAt", "attemptedAt", "scanStatus")}
153
+ view["detailsOmitted"] = True
154
+ view_bytes = len(canonical_json(view).encode("utf-8"))
155
+ required = header_bytes + item_bytes + view_bytes
156
+ if required > MAX_INPUT_BYTES:
157
+ result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
158
+ break
159
+ result["sources"].append(view)
160
+ item_bytes += view_bytes + 2
161
+ return result
162
+
163
+
164
+ def _source_view(connection: sqlite3.Connection, row: tuple) -> dict:
165
+ payload = json.loads(row[7])
166
+ relations = connection.execute("select from_node, to_node, kind, reason, evidence, standing from source_relation where snapshot_id = ? order by rowid", (row[2],)).fetchall()
167
+ return {"sourceId": row[0], "nodeId": row[1], "snapshotId": row[2],
168
+ "lifecycle": row[3], "checkedAt": row[4], "attemptedAt": row[5], "scanStatus": row[6],
169
+ "ref": payload["ref"], "scope": payload["scope"], "locator": payload["locator"],
170
+ "sourceUpdatedAt": payload["source_updated_at"], "metadata": payload["metadata"],
171
+ "nodes": [{"id": node_id(node), **node} for node in payload["nodes"]],
172
+ "relations": [{"fromNodeId": relation[0], "toNodeId": relation[1], "kind": relation[2], "reason": relation[3], "evidence": json.loads(relation[4]), "standing": relation[5]} for relation in relations]}
@@ -0,0 +1,36 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ from okstra_project.dirs import contained_relative_path
6
+
7
+ from ...json_boundary import JsonBoundaryError, load_owned_object, read_bounded_bytes
8
+ from ..contracts import DoyouknowError, MAX_INPUT_BYTES
9
+
10
+
11
+ def safe_source_path(root: Path, relative: str | Path) -> Path:
12
+ base = root.resolve()
13
+ contained = contained_relative_path(str(relative), base)
14
+ if contained is None:
15
+ raise DoyouknowError("source path is outside its registered root")
16
+ return base / contained
17
+
18
+
19
+ def read_source_text(root: Path, relative: str | Path) -> str:
20
+ path = safe_source_path(root, relative)
21
+ if not path.is_file():
22
+ raise DoyouknowError("source text is not a regular file")
23
+ try:
24
+ return read_bounded_bytes(path, max_bytes=MAX_INPUT_BYTES, artifact="source text").decode("utf-8")
25
+ except (OSError, UnicodeError, JsonBoundaryError) as exc:
26
+ raise DoyouknowError("source text is unreadable or exceeds the input limit") from exc
27
+
28
+
29
+ def read_source_object(root: Path, relative: str | Path) -> dict:
30
+ path = safe_source_path(root, relative)
31
+ if not path.is_file():
32
+ raise DoyouknowError("source object is not a regular file")
33
+ try:
34
+ return load_owned_object(path, artifact="source object", max_bytes=MAX_INPUT_BYTES)
35
+ except (OSError, JsonBoundaryError, RecursionError) as exc:
36
+ raise DoyouknowError("source object is unreadable or exceeds the input limit") from exc