okstra 0.231.1 → 0.233.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -271
- package/dist/cli-registry.mjs +14 -1
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/execute/render-bundle.d.mts +6 -1
- package/dist/commands/execute/render-bundle.mjs +139 -32
- package/dist/commands/execute/render-bundle.mjs.map +1 -1
- package/dist/commands/lifecycle/config.mjs +6 -74
- package/dist/commands/lifecycle/config.mjs.map +1 -1
- package/dist/commands/memory/memory.mjs +9 -55
- package/dist/commands/memory/memory.mjs.map +1 -1
- package/dist/commands/trace/capacity.d.mts +12 -0
- package/dist/commands/trace/capacity.mjs +21 -0
- package/dist/commands/trace/capacity.mjs.map +1 -0
- package/dist/commands/trace/model.d.mts +89 -0
- package/dist/commands/trace/model.mjs +251 -0
- package/dist/commands/trace/model.mjs.map +1 -0
- package/dist/commands/trace/progression.d.mts +11 -0
- package/dist/commands/trace/progression.mjs +194 -0
- package/dist/commands/trace/progression.mjs.map +1 -0
- package/dist/commands/trace/review.d.mts +7 -0
- package/dist/commands/trace/review.mjs +51 -0
- package/dist/commands/trace/review.mjs.map +1 -0
- package/dist/commands/trace/task-link.d.mts +18 -0
- package/dist/commands/trace/task-link.mjs +72 -0
- package/dist/commands/trace/task-link.mjs.map +1 -0
- package/dist/commands/trace/trace.d.mts +1 -0
- package/dist/commands/trace/trace.mjs +293 -0
- package/dist/commands/trace/trace.mjs.map +1 -0
- package/dist/lib/bounded-file.d.mts +1 -0
- package/dist/lib/bounded-file.mjs +36 -0
- package/dist/lib/bounded-file.mjs.map +1 -0
- package/dist/lib/config-store.d.mts +34 -0
- package/dist/lib/config-store.mjs +70 -0
- package/dist/lib/config-store.mjs.map +1 -0
- package/dist/lib/file-lock.mjs +4 -5
- package/dist/lib/file-lock.mjs.map +1 -1
- package/dist/lib/input-limit.d.mts +4 -0
- package/dist/lib/input-limit.mjs +8 -0
- package/dist/lib/input-limit.mjs.map +1 -0
- package/dist/lib/memory-auto-register.d.mts +1 -0
- package/dist/lib/memory-auto-register.mjs +98 -0
- package/dist/lib/memory-auto-register.mjs.map +1 -0
- package/dist/lib/memory-entry-contract.d.mts +12 -0
- package/dist/lib/memory-entry-contract.mjs +46 -0
- package/dist/lib/memory-entry-contract.mjs.map +1 -0
- package/dist/lib/python-helper.d.mts +1 -1
- package/dist/lib/python-helper.mjs +3 -2
- package/dist/lib/python-helper.mjs.map +1 -1
- package/dist/lib/record-secrets.d.mts +1 -0
- package/dist/lib/record-secrets.mjs +28 -0
- package/dist/lib/record-secrets.mjs.map +1 -0
- package/dist/lib/report-language-tag.d.mts +1 -0
- package/dist/lib/report-language-tag.mjs +4 -0
- package/dist/lib/report-language-tag.mjs.map +1 -0
- package/dist/lib/report-language.d.mts +1 -0
- package/dist/lib/report-language.mjs +50 -0
- package/dist/lib/report-language.mjs.map +1 -0
- package/dist/lib/skill-catalog.mjs +3 -0
- package/dist/lib/skill-catalog.mjs.map +1 -1
- package/dist/lib/types.d.mts +2 -1
- package/docs/architecture/storage-model.md +163 -5
- package/docs/architecture.md +17 -4
- package/docs/cli.md +511 -0
- package/docs/contributor-change-matrix.md +1 -2
- package/docs/project-structure-overview.md +33 -42
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/agents/common.json +8 -8
- package/runtime/agents/operations/trace-interface-review.json +6 -0
- package/runtime/agents/operations/trace-scope-review.json +6 -0
- package/runtime/agents/roles/verifier.json +2 -4
- package/runtime/bin/okstra-provider-exec.py +10 -4
- package/runtime/prompts/coding-preflight/architectures/hexagonal.md +23 -30
- package/runtime/prompts/coding-preflight/authoring-checks.md +17 -0
- package/runtime/prompts/coding-preflight/clean-code.md +88 -286
- package/runtime/prompts/coding-preflight/frameworks/node-server.md +2 -3
- package/runtime/prompts/coding-preflight/languages/java.md +3 -5
- package/runtime/prompts/coding-preflight/languages/javascript-typescript.md +5 -12
- package/runtime/prompts/coding-preflight/languages/kotlin.md +2 -4
- package/runtime/prompts/coding-preflight/languages/python.md +2 -7
- package/runtime/prompts/coding-preflight/languages/rust.md +2 -8
- package/runtime/prompts/coding-preflight/languages/sql.md +2 -4
- package/runtime/prompts/coding-preflight/overview.md +25 -38
- package/runtime/prompts/coding-preflight/review-rules.json +30 -0
- package/runtime/prompts/duties/code-reviewer.json +15 -18
- package/runtime/prompts/duties/implementation-verifier.json +1 -0
- package/runtime/prompts/duties/interface-reviewer.json +32 -0
- package/runtime/prompts/host-orchestration/README.md +18 -9
- package/runtime/prompts/launch.template.md +3 -2
- package/runtime/prompts/lead/context-loader.md +2 -0
- package/runtime/prompts/lead/okstra-lead-contract.md +7 -7
- package/runtime/prompts/lead/report-writer.md +6 -6
- package/runtime/prompts/profiles/_coding-conventions-preflight.md +41 -6
- package/runtime/prompts/profiles/_common-contract.md +1 -1
- package/runtime/python/okstra_ctl/agent/invocation.py +122 -36
- package/runtime/python/okstra_ctl/agent/prompt_cli/cli.py +72 -26
- package/runtime/python/okstra_ctl/agent/prompt_cli/corrections.py +23 -8
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +58 -14
- package/runtime/python/okstra_ctl/agent/prompt_cli/results.py +8 -11
- package/runtime/python/okstra_ctl/agent/standalone.py +15 -0
- package/runtime/python/okstra_ctl/analysis_packet.py +24 -30
- package/runtime/python/okstra_ctl/attempt_evidence.py +10 -0
- package/runtime/python/okstra_ctl/code_review_census.py +162 -7
- package/runtime/python/okstra_ctl/code_review_coverage.py +4 -274
- package/runtime/python/okstra_ctl/code_review_integrity.py +154 -0
- package/runtime/python/okstra_ctl/code_review_invocation.py +212 -0
- package/runtime/python/okstra_ctl/code_review_prepare.py +381 -0
- package/runtime/python/okstra_ctl/code_review_run.py +384 -0
- package/runtime/python/okstra_ctl/code_review_scope.py +49 -9
- package/runtime/python/okstra_ctl/code_review_supervisor.py +41 -0
- package/runtime/python/okstra_ctl/code_review_target.py +212 -13
- package/runtime/python/okstra_ctl/consumers.py +12 -9
- package/runtime/python/okstra_ctl/coverage_census.py +11 -2
- package/runtime/python/okstra_ctl/design_review.py +207 -0
- package/runtime/python/okstra_ctl/design_review_evidence.py +297 -0
- package/runtime/python/okstra_ctl/design_snapshot.py +16 -9
- package/runtime/python/okstra_ctl/direct_work.py +45 -3
- package/runtime/python/okstra_ctl/dispatch_core.py +31 -15
- package/runtime/python/okstra_ctl/dispatch_state.py +9 -2
- package/runtime/python/okstra_ctl/doctor.py +8 -25
- package/runtime/python/okstra_ctl/domain/role.py +1 -0
- package/runtime/python/okstra_ctl/doyouknow/__init__.py +9 -0
- package/runtime/python/okstra_ctl/doyouknow/cli.py +91 -0
- package/runtime/python/okstra_ctl/doyouknow/config.py +58 -0
- package/runtime/python/okstra_ctl/doyouknow/contracts.py +184 -0
- package/runtime/python/okstra_ctl/doyouknow/identity.py +22 -0
- package/runtime/python/okstra_ctl/doyouknow/paths.py +22 -0
- package/runtime/python/okstra_ctl/doyouknow/pending.py +72 -0
- package/runtime/python/okstra_ctl/doyouknow/registration.py +260 -0
- package/runtime/python/okstra_ctl/doyouknow/source_contracts.py +128 -0
- package/runtime/python/okstra_ctl/doyouknow/source_store.py +172 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/io.py +36 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/manager.py +300 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/memory.py +179 -0
- package/runtime/python/okstra_ctl/doyouknow/sources/task.py +95 -0
- package/runtime/python/okstra_ctl/doyouknow/store.py +260 -0
- package/runtime/python/okstra_ctl/doyouknow/sync.py +58 -0
- package/runtime/python/okstra_ctl/execution_mutation_audit.py +18 -2
- package/runtime/python/okstra_ctl/file_size_cap.py +3 -4
- package/runtime/python/okstra_ctl/final_report_schema.py +37 -34
- package/runtime/python/okstra_ctl/incremental_carry.py +6 -7
- package/runtime/python/okstra_ctl/initial_prompt_materialization.py +9 -2
- package/runtime/python/okstra_ctl/json_boundary.py +51 -5
- package/runtime/python/okstra_ctl/jsonl.py +21 -0
- package/runtime/python/okstra_ctl/manager_cli.py +148 -1
- package/runtime/python/okstra_ctl/manager_journal.py +126 -0
- package/runtime/python/okstra_ctl/manager_launch.py +3 -3
- package/runtime/python/okstra_ctl/manager_log.py +221 -0
- package/runtime/python/okstra_ctl/manager_paths.py +8 -0
- package/runtime/python/okstra_ctl/manager_registration.py +29 -0
- package/runtime/python/okstra_ctl/manager_resume.py +59 -0
- package/runtime/python/okstra_ctl/manager_split.py +2 -2
- package/runtime/python/okstra_ctl/manager_store.py +116 -17
- package/runtime/python/okstra_ctl/manager_sync.py +45 -25
- package/runtime/python/okstra_ctl/manager_view.py +35 -2
- package/runtime/python/okstra_ctl/material.py +2 -0
- package/runtime/python/okstra_ctl/model_cli.py +64 -14
- package/runtime/python/okstra_ctl/model_defaults.py +22 -0
- package/runtime/python/okstra_ctl/model_io/renderers.py +16 -2
- package/runtime/python/okstra_ctl/model_io_cli.py +7 -1
- package/runtime/python/okstra_ctl/operation_invocation.py +207 -28
- package/runtime/python/okstra_ctl/path_hints.py +14 -3
- package/runtime/python/okstra_ctl/paths.py +6 -1
- package/runtime/python/okstra_ctl/phases/catalog.py +24 -7
- package/runtime/python/okstra_ctl/phases/final_verification/validation.py +20 -0
- package/runtime/python/okstra_ctl/phases/implementation/host-rules.md +3 -3
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +10 -0
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-diff-review.md +23 -8
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-self-check.md +6 -4
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +38 -10
- package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation.template.html +13 -0
- package/runtime/python/okstra_ctl/phases/implementation/spec.md +25 -0
- package/runtime/python/okstra_ctl/phases/implementation/validation.py +6 -0
- package/runtime/python/okstra_ctl/phases/implementation_option_selection/profile.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation_option_selection/votes.py +11 -11
- package/runtime/python/okstra_ctl/phases/implementation_planning/authoring.py +16 -9
- package/runtime/python/okstra_ctl/phases/implementation_planning/host-rules.md +3 -6
- package/runtime/python/okstra_ctl/phases/implementation_planning/instructions/plan-body-verification.md +12 -10
- package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +6 -6
- package/runtime/python/okstra_ctl/phases/improvement_discovery/lenses.py +12 -0
- package/runtime/python/okstra_ctl/phases/improvement_discovery/profile.md +7 -6
- package/runtime/python/okstra_ctl/phases/improvement_discovery/resolved_census.py +159 -0
- package/runtime/python/okstra_ctl/phases/improvement_discovery/spec.md +5 -2
- package/runtime/python/okstra_ctl/phases/improvement_discovery/validation.py +21 -7
- package/runtime/python/okstra_ctl/phases/release_handoff/boundary.json +1 -1
- package/runtime/python/okstra_ctl/phases/release_handoff/profile.md +4 -1
- package/runtime/python/okstra_ctl/plan_verify_cli.py +6 -4
- package/runtime/python/okstra_ctl/render.py +28 -2
- package/runtime/python/okstra_ctl/report_assembly.py +53 -11
- package/runtime/python/okstra_ctl/report_contract.py +26 -0
- package/runtime/python/okstra_ctl/report_corrections.py +36 -20
- package/runtime/python/okstra_ctl/report_inputs.py +1 -0
- package/runtime/python/okstra_ctl/report_language.py +4 -1
- package/runtime/python/okstra_ctl/report_narrative.py +182 -7
- package/runtime/python/okstra_ctl/report_projections.py +64 -18
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +97 -38
- package/runtime/python/okstra_ctl/review_policy.py +359 -0
- package/runtime/python/okstra_ctl/review_verdicts.py +291 -0
- package/runtime/python/okstra_ctl/run.py +67 -36
- package/runtime/python/okstra_ctl/run_context.py +10 -1
- package/runtime/python/okstra_ctl/set_work_status.py +4 -15
- package/runtime/python/okstra_ctl/stage_done.py +11 -0
- package/runtime/python/okstra_ctl/standalone_review_audit.py +28 -0
- package/runtime/python/okstra_ctl/task_list_cli.py +34 -8
- package/runtime/python/okstra_ctl/task_query.py +239 -0
- package/runtime/python/okstra_ctl/trace_review.py +264 -0
- package/runtime/python/okstra_ctl/trace_tasks.py +197 -0
- package/runtime/python/okstra_ctl/user_authorization.py +14 -0
- package/runtime/python/okstra_ctl/verification_generation.py +23 -1
- package/runtime/python/okstra_ctl/verification_input_types.py +15 -1
- package/runtime/python/okstra_ctl/verification_inputs.py +28 -6
- package/runtime/python/okstra_ctl/verification_invocation.py +7 -1
- package/runtime/python/okstra_ctl/verification_target.py +2 -0
- package/runtime/python/okstra_ctl/wizard/confirmation.py +7 -3
- package/runtime/python/okstra_ctl/wizard/launch.py +52 -0
- package/runtime/python/okstra_ctl/wizard/launch_cli.py +39 -0
- package/runtime/python/okstra_ctl/wizard/outcome.py +17 -16
- package/runtime/python/okstra_ctl/wizard/registry.py +1 -1
- package/runtime/python/okstra_ctl/wizard/render.py +8 -0
- package/runtime/python/okstra_ctl/wizard/state.py +2 -1
- package/runtime/python/okstra_ctl/worker_prompt_headers.py +24 -9
- package/runtime/python/okstra_ctl/worker_runner.py +66 -10
- package/runtime/python/okstra_ctl/write_policy.py +34 -1
- package/runtime/python/okstra_project/__init__.py +2 -0
- package/runtime/python/okstra_project/dirs.py +18 -0
- package/runtime/python/okstra_project/state.py +45 -20
- package/runtime/python/okstra_token_usage/blocks.py +74 -0
- package/runtime/python/okstra_token_usage/cli.py +23 -10
- package/runtime/python/okstra_token_usage/collect.py +25 -22
- package/runtime/python/okstra_token_usage/policy.py +193 -0
- package/runtime/schemas/doyouknow-config-v1.schema.json +20 -0
- package/runtime/schemas/doyouknow-v1.schema.json +65 -0
- package/runtime/schemas/final-report-v2.0.schema.json +157 -0
- package/runtime/schemas/final-report-v3.0.schema.json +157 -0
- package/runtime/schemas/memory-entry-contract-v1.json +11 -0
- package/runtime/schemas/record-secret-patterns.json +41 -0
- package/runtime/schemas/report-synthesis-packet-v1.0.schema.json +1 -1
- package/runtime/skills/_fragments/wizard-confirm-step.md +1 -1
- package/runtime/skills/_fragments/wizard-preflight.md +1 -1
- package/runtime/skills/_fragments/wizard-prompt-loop.md +3 -5
- package/runtime/skills/_fragments/wizard-relay-protocol.md +1 -1
- package/runtime/skills/_fragments/wizard-render-bundle.md +15 -29
- package/runtime/skills/okstra-brief-gen/SKILL.md +7 -7
- package/runtime/skills/okstra-brief-gen/references/tracker-recursion.md +1 -1
- package/runtime/skills/okstra-code-review/SKILL.md +92 -136
- package/runtime/skills/okstra-code-review/references/census-rules.md +29 -105
- package/runtime/skills/okstra-code-review/references/review-calibration.md +37 -61
- package/runtime/skills/okstra-discussion/SKILL.md +22 -38
- package/runtime/skills/okstra-explain-flow/SKILL.md +6 -2
- package/runtime/skills/okstra-manager/SKILL.md +31 -0
- package/runtime/skills/okstra-memory/SKILL.md +8 -1
- package/runtime/skills/okstra-pr-gen/SKILL.md +1 -1
- package/runtime/skills/okstra-run/SKILL.md +55 -183
- package/runtime/skills/okstra-schedule-gen/SKILL.md +9 -13
- package/runtime/skills/okstra-setup/SKILL.md +3 -2
- package/runtime/skills/okstra-setup/references/project-config.md +2 -4
- package/runtime/skills/okstra-tdd/SKILL.md +187 -0
- package/runtime/skills/okstra-user-response/SKILL.md +4 -2
- package/runtime/templates/implementation-worker-preamble.md +4 -9
- package/runtime/templates/manager/view.template.html +4 -0
- package/runtime/templates/report-writer-prompt-preamble.md +15 -3
- package/runtime/templates/trace/trace.template.md +42 -0
- package/runtime/templates/worker-prompt-preamble.md +4 -12
- package/runtime/validators/checks/fixtures-02.py +25 -33
- package/runtime/validators/validate-run.py +107 -38
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ..json_boundary import JsonBoundaryError, load_owned_object, write_owned_object_atomic
|
|
9
|
+
from ..run_context import dir_flock
|
|
10
|
+
from .config import SourceConfig, load_config
|
|
11
|
+
from .contracts import DoyouknowError, SecretRefusal
|
|
12
|
+
from .identity import source_id
|
|
13
|
+
from .paths import database_path, ensure_storage_root, require_database_location
|
|
14
|
+
from .pending import MAX_TICKET_BYTES, pending_path, read_ticket
|
|
15
|
+
from .source_contracts import PreparedSource, SourceKind, SourceRef, validate_prepared_snapshot, validate_source_value
|
|
16
|
+
from .sources.memory import read_memory_source
|
|
17
|
+
|
|
18
|
+
MAX_FAILURES = 3
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class RegistrationState:
|
|
23
|
+
fingerprint: str | None = None
|
|
24
|
+
attempts: int = 0
|
|
25
|
+
status: str = "pending"
|
|
26
|
+
errorCode: str | None = None
|
|
27
|
+
refusedFingerprint: str | None = None
|
|
28
|
+
preparationAttempts: int = 0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _read_state(path: Path) -> tuple[RegistrationState, dict | None]:
|
|
32
|
+
require_database_location(path)
|
|
33
|
+
if not path.exists():
|
|
34
|
+
return RegistrationState(), None
|
|
35
|
+
previous = load_owned_object(path, artifact="registration state", max_bytes=MAX_TICKET_BYTES)
|
|
36
|
+
validate_source_value(previous)
|
|
37
|
+
raw = {"preparationAttempts": 0, **previous}
|
|
38
|
+
strings = ("fingerprint", "errorCode", "refusedFingerprint")
|
|
39
|
+
if set(raw) != {*strings, "attempts", "status", "preparationAttempts"} or any(raw[key] is not None and not isinstance(raw[key], str) for key in strings):
|
|
40
|
+
raise DoyouknowError("invalid registration state")
|
|
41
|
+
if type(raw["attempts"]) is not int or not 0 <= raw["attempts"] <= MAX_FAILURES:
|
|
42
|
+
raise DoyouknowError("invalid registration state")
|
|
43
|
+
if type(raw["preparationAttempts"]) is not int or not 0 <= raw["preparationAttempts"] <= MAX_FAILURES:
|
|
44
|
+
raise DoyouknowError("invalid registration state")
|
|
45
|
+
if raw["status"] not in ("registered", "pending", "refused", "exhausted"):
|
|
46
|
+
raise DoyouknowError("invalid registration state")
|
|
47
|
+
if raw["errorCode"] not in (None, "registration-unavailable", "source-unavailable", "source-refused", "source-preparation-unavailable", "source-preparation-exhausted"):
|
|
48
|
+
raise DoyouknowError("invalid registration state")
|
|
49
|
+
return RegistrationState(**raw), previous
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _result(ticket: str | None, ref: SourceRef | None, state: RegistrationState) -> dict:
|
|
53
|
+
return {"ok": state.status in ("registered", "excluded"), "ticket": ticket,
|
|
54
|
+
"sourceId": source_id(ref) if ref else None, "status": state.status,
|
|
55
|
+
"attempts": state.attempts, "preparationAttempts": state.preparationAttempts, "errorCode": state.errorCode}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def register_ticket(ticket: str, *, home: Path | None = None, manual: bool = False) -> dict:
|
|
59
|
+
ref = None
|
|
60
|
+
safe_ticket = None
|
|
61
|
+
try:
|
|
62
|
+
pending_path(ticket, home=home)
|
|
63
|
+
safe_ticket = ticket
|
|
64
|
+
ref = read_ticket(ticket, home=home)
|
|
65
|
+
directory = database_path(home).parent / "registrations"
|
|
66
|
+
require_database_location(directory / "state.json")
|
|
67
|
+
require_database_location(directory / f".{source_id(ref)}.lock")
|
|
68
|
+
ensure_storage_root(directory.parent)
|
|
69
|
+
with dir_flock(directory, f".{source_id(ref)}.lock"):
|
|
70
|
+
if not pending_path(ticket, home=home).exists():
|
|
71
|
+
return _result(ticket, ref, RegistrationState(status="registered"))
|
|
72
|
+
return _register_locked(ticket, ref, directory / f"{source_id(ref)}.json", home, manual)
|
|
73
|
+
except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
|
|
74
|
+
return _result(safe_ticket, ref, RegistrationState(errorCode="registration-unavailable"))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _register_locked(ticket: str, ref: SourceRef, path: Path, home: Path | None, manual: bool) -> dict:
|
|
78
|
+
settings = load_config(home=home)
|
|
79
|
+
if (not settings["autoRegister"] and not manual) or ref.kind not in settings["sources"]:
|
|
80
|
+
pending_path(ticket, home=home).unlink(missing_ok=True)
|
|
81
|
+
return _result(ticket, ref, RegistrationState(status="excluded"))
|
|
82
|
+
if ref.kind not in ("memory", "manager"):
|
|
83
|
+
return _result(ticket, ref, RegistrationState(errorCode="unsupported-registration-source"))
|
|
84
|
+
state, previous = _read_state(path)
|
|
85
|
+
if state.preparationAttempts >= MAX_FAILURES and not manual:
|
|
86
|
+
state.status, state.errorCode = "exhausted", "source-preparation-exhausted"
|
|
87
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
88
|
+
return _result(ticket, ref, state)
|
|
89
|
+
state.preparationAttempts = min(state.preparationAttempts + 1, MAX_FAILURES)
|
|
90
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
91
|
+
try:
|
|
92
|
+
prepared = _prepare_source(ref, database_path(home).parent.parent, settings)
|
|
93
|
+
except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
|
|
94
|
+
state.status = "exhausted" if state.preparationAttempts >= MAX_FAILURES else "pending"
|
|
95
|
+
state.errorCode = "source-preparation-unavailable"
|
|
96
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
97
|
+
return _result(ticket, ref, state)
|
|
98
|
+
state.preparationAttempts = 0
|
|
99
|
+
if prepared.outcome == "excluded":
|
|
100
|
+
if previous is None:
|
|
101
|
+
path.unlink(missing_ok=True)
|
|
102
|
+
else:
|
|
103
|
+
write_owned_object_atomic(path, previous, artifact="registration state")
|
|
104
|
+
pending_path(ticket, home=home).unlink(missing_ok=True)
|
|
105
|
+
return _result(ticket, ref, RegistrationState(status="excluded"))
|
|
106
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
107
|
+
if prepared.ref != ref:
|
|
108
|
+
raise DoyouknowError("registration reference does not match its source")
|
|
109
|
+
return _register_prepared(ticket, ref, prepared, path, home, manual)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _prepare_source(ref: SourceRef, home: Path, settings: SourceConfig) -> PreparedSource:
|
|
113
|
+
if ref.kind == "memory":
|
|
114
|
+
return read_memory_source(home, ref, settings)
|
|
115
|
+
from .sources.manager import prepare_manager_source
|
|
116
|
+
|
|
117
|
+
if ref.native_id != "$manager":
|
|
118
|
+
raise DoyouknowError("invalid manager registration reference")
|
|
119
|
+
return prepare_manager_source(home, ref.owner_id, settings)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _register_prepared(ticket: str, ref: SourceRef, prepared: PreparedSource, path: Path, home: Path | None, manual: bool) -> dict:
|
|
123
|
+
state, _ = _read_state(path)
|
|
124
|
+
if prepared.fingerprint is not None and prepared.fingerprint == state.refusedFingerprint:
|
|
125
|
+
state.status, state.errorCode = "refused", "source-refused"
|
|
126
|
+
return _result(ticket, ref, state)
|
|
127
|
+
if prepared.fingerprint is not None and state.fingerprint != prepared.fingerprint:
|
|
128
|
+
state.fingerprint, state.attempts = prepared.fingerprint, 0
|
|
129
|
+
if state.attempts >= MAX_FAILURES and not manual:
|
|
130
|
+
state.status = "exhausted"
|
|
131
|
+
return _result(ticket, ref, state)
|
|
132
|
+
state.attempts = min(state.attempts + 1, MAX_FAILURES)
|
|
133
|
+
state.status, state.errorCode = "pending", None
|
|
134
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
135
|
+
try:
|
|
136
|
+
_store_prepared(prepared, home)
|
|
137
|
+
except SecretRefusal:
|
|
138
|
+
state.status, state.errorCode = "refused", "source-refused"
|
|
139
|
+
state.refusedFingerprint = prepared.fingerprint
|
|
140
|
+
except (DoyouknowError, JsonBoundaryError, OSError, ImportError, RecursionError):
|
|
141
|
+
state.status = "exhausted" if state.attempts >= MAX_FAILURES else "pending"
|
|
142
|
+
state.errorCode = "source-unavailable"
|
|
143
|
+
else:
|
|
144
|
+
state.status, state.attempts, state.errorCode, state.refusedFingerprint = "registered", 0, None, None
|
|
145
|
+
write_owned_object_atomic(path, vars(state), artifact="registration state")
|
|
146
|
+
if state.status == "registered":
|
|
147
|
+
pending_path(ticket, home=home).unlink(missing_ok=True)
|
|
148
|
+
return _result(ticket, ref, state)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _store_prepared(prepared: PreparedSource, home: Path | None) -> None:
|
|
152
|
+
if prepared.outcome != "ready":
|
|
153
|
+
raise DoyouknowError("source unavailable")
|
|
154
|
+
if prepared.scan is not None:
|
|
155
|
+
from .sources.manager import gate_manager_scan
|
|
156
|
+
from .source_store import apply_source_scan
|
|
157
|
+
|
|
158
|
+
scan = gate_manager_scan(prepared.scan, gate=lambda candidate: validate_source_and_update_refusal_cache(candidate, home=home))
|
|
159
|
+
refused = "manager-secret-refused" in scan.errors
|
|
160
|
+
if scan.status != "unavailable":
|
|
161
|
+
apply_source_scan(scan, home=home)
|
|
162
|
+
if refused:
|
|
163
|
+
raise SecretRefusal("manager source refused")
|
|
164
|
+
if scan.status != "complete":
|
|
165
|
+
raise DoyouknowError("source scan incomplete")
|
|
166
|
+
elif prepared.snapshot is not None:
|
|
167
|
+
if not validate_source_and_update_refusal_cache(prepared, home=home):
|
|
168
|
+
raise SecretRefusal("source refused")
|
|
169
|
+
from .source_store import store_source_snapshot
|
|
170
|
+
|
|
171
|
+
store_source_snapshot(prepared.snapshot, home=home)
|
|
172
|
+
else:
|
|
173
|
+
raise DoyouknowError("source snapshot unavailable")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def validate_source_and_update_refusal_cache(prepared: PreparedSource, *, home: Path | None = None) -> bool:
|
|
177
|
+
if prepared.outcome != "ready" or prepared.snapshot is None or prepared.fingerprint is None:
|
|
178
|
+
raise DoyouknowError("source snapshot unavailable")
|
|
179
|
+
database = database_path(home)
|
|
180
|
+
require_database_location(database)
|
|
181
|
+
directory = database.parent / "refusals"
|
|
182
|
+
path = directory / f"{source_id(prepared.ref)}.json"
|
|
183
|
+
require_database_location(path)
|
|
184
|
+
require_database_location(directory / f".{source_id(prepared.ref)}.lock")
|
|
185
|
+
ensure_storage_root(directory.parent)
|
|
186
|
+
with dir_flock(directory, f".{source_id(prepared.ref)}.lock"):
|
|
187
|
+
if path.exists():
|
|
188
|
+
cached = load_owned_object(path, artifact="source refusal", max_bytes=MAX_TICKET_BYTES)
|
|
189
|
+
validate_source_value(cached)
|
|
190
|
+
if set(cached) != {"fingerprint"} or not isinstance(cached["fingerprint"], str):
|
|
191
|
+
raise DoyouknowError("invalid source refusal")
|
|
192
|
+
if cached["fingerprint"] == prepared.fingerprint:
|
|
193
|
+
return False
|
|
194
|
+
try:
|
|
195
|
+
validate_prepared_snapshot(prepared)
|
|
196
|
+
except SecretRefusal:
|
|
197
|
+
write_owned_object_atomic(path, {"fingerprint": prepared.fingerprint}, artifact="source refusal")
|
|
198
|
+
return False
|
|
199
|
+
path.unlink(missing_ok=True)
|
|
200
|
+
return True
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def retry_pending(*, home: Path | None = None, source: SourceKind | None = None, limit: int = 100, manual: bool = False) -> dict:
|
|
204
|
+
if type(limit) is not int or not 1 <= limit <= 1000:
|
|
205
|
+
raise DoyouknowError("pending limit must be between 1 and 1000")
|
|
206
|
+
directory = database_path(home).parent / "pending"
|
|
207
|
+
require_database_location(directory / "ticket.json")
|
|
208
|
+
results = []
|
|
209
|
+
for ticket in _pending_candidates(directory, source, home):
|
|
210
|
+
if len(results) >= limit:
|
|
211
|
+
return {"ok": all(row["ok"] for row in results), "results": results, "truncated": True}
|
|
212
|
+
result = register_ticket(ticket, home=home, manual=manual)
|
|
213
|
+
results.append(result)
|
|
214
|
+
if not result["ok"]:
|
|
215
|
+
try:
|
|
216
|
+
os.utime(pending_path(ticket, home=home), follow_symlinks=False)
|
|
217
|
+
except FileNotFoundError:
|
|
218
|
+
continue
|
|
219
|
+
except (DoyouknowError, OSError):
|
|
220
|
+
result["errorCode"] = "pending-order-unavailable"
|
|
221
|
+
return {"ok": all(row["ok"] for row in results), "results": results, "truncated": False}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _pending_candidates(directory: Path, source: SourceKind | None, home: Path | None) -> Iterator[str]:
|
|
225
|
+
if not directory.exists():
|
|
226
|
+
return
|
|
227
|
+
entries = []
|
|
228
|
+
try:
|
|
229
|
+
for path in directory.iterdir():
|
|
230
|
+
if not path.name.endswith(".json") or path.name.startswith("."):
|
|
231
|
+
continue
|
|
232
|
+
try:
|
|
233
|
+
entries.append((path.lstat().st_mtime_ns, path.name, path))
|
|
234
|
+
except FileNotFoundError:
|
|
235
|
+
# expected-miss: another registrar can acknowledge an enumerated UUID.
|
|
236
|
+
continue
|
|
237
|
+
except OSError as exc:
|
|
238
|
+
raise DoyouknowError("pending queue is unreadable") from exc
|
|
239
|
+
for _, _, path in sorted(entries):
|
|
240
|
+
try:
|
|
241
|
+
path.lstat()
|
|
242
|
+
except FileNotFoundError:
|
|
243
|
+
# expected-miss: an earlier batch can acknowledge this queued UUID.
|
|
244
|
+
continue
|
|
245
|
+
if source is None:
|
|
246
|
+
yield path.stem
|
|
247
|
+
continue
|
|
248
|
+
try:
|
|
249
|
+
ref = read_ticket(path.stem, home=home)
|
|
250
|
+
except JsonBoundaryError as exc:
|
|
251
|
+
if exc.reason == "missing":
|
|
252
|
+
# expected-miss: acknowledgment can race with the source-filter read.
|
|
253
|
+
continue
|
|
254
|
+
yield path.stem
|
|
255
|
+
continue
|
|
256
|
+
except (DoyouknowError, OSError):
|
|
257
|
+
yield path.stem
|
|
258
|
+
continue
|
|
259
|
+
if ref.kind == source:
|
|
260
|
+
yield path.stem
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import asdict, dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
from ..json_boundary import serialize_owned_object
|
|
9
|
+
from .contracts import (DoyouknowError, MAX_INPUT_BYTES, Node, Relation, _check_input_depth,
|
|
10
|
+
_decoded_strings, asset_object, canonical_json, record_secret_policy_digest, reject_secrets)
|
|
11
|
+
from .identity import stable_id
|
|
12
|
+
|
|
13
|
+
SourceKind = Literal["memory", "manager", "task"]
|
|
14
|
+
Lifecycle = Literal["active", "archived", "unlinked", "missing"]
|
|
15
|
+
JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SourceRef:
|
|
20
|
+
kind: SourceKind
|
|
21
|
+
owner_id: str
|
|
22
|
+
native_id: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class SourceDomain:
|
|
27
|
+
kind: SourceKind
|
|
28
|
+
owner_id: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class SourceSnapshot:
|
|
33
|
+
ref: SourceRef
|
|
34
|
+
primary_node: Node
|
|
35
|
+
locator: str
|
|
36
|
+
metadata: dict[str, JsonValue] = field(default_factory=dict)
|
|
37
|
+
nodes: tuple[Node, ...] = ()
|
|
38
|
+
relations: tuple[Relation, ...] = ()
|
|
39
|
+
lifecycle: Lifecycle = "active"
|
|
40
|
+
source_updated_at: str | None = None
|
|
41
|
+
scope: str | None = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class SourceScan:
|
|
46
|
+
domain: SourceDomain
|
|
47
|
+
snapshots: tuple[SourceSnapshot, ...]
|
|
48
|
+
observed_refs: tuple[SourceRef, ...]
|
|
49
|
+
status: Literal["complete", "partial", "unavailable"]
|
|
50
|
+
errors: tuple[str, ...] = ()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class PreparedSource:
|
|
55
|
+
ref: SourceRef
|
|
56
|
+
fingerprint: str | None
|
|
57
|
+
snapshot: SourceSnapshot | None = field(default=None, repr=False)
|
|
58
|
+
outcome: Literal["ready", "excluded", "unavailable"] = "ready"
|
|
59
|
+
secret_texts: tuple[str, ...] = field(default=(), repr=False)
|
|
60
|
+
scan: SourceScan | None = field(default=None, repr=False)
|
|
61
|
+
error_code: str | None = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def prepared_snapshot(snapshot: SourceSnapshot) -> PreparedSource:
|
|
65
|
+
return PreparedSource(snapshot.ref, stable_id("source-content", canonical_json(asdict(snapshot)), record_secret_policy_digest()), snapshot)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def validate_prepared_snapshot(prepared: PreparedSource) -> None:
|
|
69
|
+
if prepared.outcome != "ready" or prepared.snapshot is None:
|
|
70
|
+
raise DoyouknowError("source snapshot unavailable")
|
|
71
|
+
for text in prepared.secret_texts:
|
|
72
|
+
reject_secrets(text)
|
|
73
|
+
snapshot_payload(prepared.snapshot)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def validate_source_value(value: object) -> str:
|
|
77
|
+
_check_input_depth(value)
|
|
78
|
+
try:
|
|
79
|
+
encoded = canonical_json(value)
|
|
80
|
+
except (ValueError, TypeError) as exc:
|
|
81
|
+
raise DoyouknowError("source input is not a JSON value") from exc
|
|
82
|
+
if len(encoded.encode("utf-8")) > MAX_INPUT_BYTES:
|
|
83
|
+
raise DoyouknowError("source input exceeds 2000000 bytes")
|
|
84
|
+
reject_secrets("\n".join(_decoded_strings(value)))
|
|
85
|
+
return encoded
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def snapshot_payload(snapshot: SourceSnapshot) -> dict:
|
|
89
|
+
payload = json.loads(validate_source_value({
|
|
90
|
+
"ref": {"kind": snapshot.ref.kind, "owner_id": snapshot.ref.owner_id, "native_id": snapshot.ref.native_id},
|
|
91
|
+
"primary_node": snapshot.primary_node, "locator": snapshot.locator,
|
|
92
|
+
"metadata": snapshot.metadata, "nodes": [snapshot.primary_node, *snapshot.nodes],
|
|
93
|
+
"relations": list(snapshot.relations), "lifecycle": snapshot.lifecycle,
|
|
94
|
+
"source_updated_at": snapshot.source_updated_at, "scope": snapshot.scope,
|
|
95
|
+
}))
|
|
96
|
+
_require_snapshot_fields(snapshot)
|
|
97
|
+
_require_snapshot_graph(payload)
|
|
98
|
+
return payload
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _require_snapshot_fields(snapshot: SourceSnapshot) -> None:
|
|
102
|
+
if snapshot.ref.kind not in ("memory", "manager", "task") or snapshot.lifecycle not in ("active", "archived", "unlinked", "missing"):
|
|
103
|
+
raise DoyouknowError("invalid source kind or lifecycle")
|
|
104
|
+
for value in (snapshot.ref.owner_id, snapshot.ref.native_id, snapshot.locator):
|
|
105
|
+
if not isinstance(value, str) or not value.strip():
|
|
106
|
+
raise DoyouknowError("source identity and locator must be nonempty strings")
|
|
107
|
+
if snapshot.scope is not None and (not isinstance(snapshot.scope, str) or not snapshot.scope.strip()):
|
|
108
|
+
raise DoyouknowError("source scope must be a nonempty string or null")
|
|
109
|
+
if snapshot.source_updated_at is not None and not isinstance(snapshot.source_updated_at, str):
|
|
110
|
+
raise DoyouknowError("source update time must be a string or null")
|
|
111
|
+
if not isinstance(snapshot.metadata, dict):
|
|
112
|
+
raise DoyouknowError("source metadata must be an object")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _require_snapshot_graph(payload: dict) -> None:
|
|
116
|
+
schema = asset_object("doyouknow-v1.schema.json")
|
|
117
|
+
if len(payload["nodes"]) > 1000 or len(payload["relations"]) > 4000:
|
|
118
|
+
raise DoyouknowError("source graph exceeds input limits")
|
|
119
|
+
for node in payload["nodes"]:
|
|
120
|
+
serialize_owned_object(Path("source-node.json"), node, artifact="source node", schema={**schema, **schema["$defs"]["node"]})
|
|
121
|
+
refs = [node.get("ref", node["key"]) for node in payload["nodes"]]
|
|
122
|
+
identities = [(node["kind"], node["scope"], node["key"]) for node in payload["nodes"]]
|
|
123
|
+
if "$source" in refs or "$record" in refs or len(refs) != len(set(refs)) or len(identities) != len(set(identities)):
|
|
124
|
+
raise DoyouknowError("source node references and identities must be unique")
|
|
125
|
+
for relation in payload["relations"]:
|
|
126
|
+
serialize_owned_object(Path("source-relation.json"), relation, artifact="source relation", schema={**schema, **schema["$defs"]["relation"]})
|
|
127
|
+
if relation["from"] not in {"$source", *refs} or relation["to"] not in {"$source", *refs}:
|
|
128
|
+
raise DoyouknowError("source relation endpoint is not in this snapshot")
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from contextlib import closing
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .contracts import DoyouknowError, MAX_INPUT_BYTES, canonical_json
|
|
10
|
+
from .identity import node_id, source_id, stable_id
|
|
11
|
+
from .source_contracts import SourceRef, SourceScan, SourceSnapshot, snapshot_payload, validate_source_value
|
|
12
|
+
from .store import SCHEMA_VERSION, SQLITE_TIMEOUT_SECONDS, _digest, _insert_node, _require_database_location, database_path, write_connection
|
|
13
|
+
|
|
14
|
+
SOURCE_SCHEMA_STATEMENTS = (
|
|
15
|
+
"""create table if not exists source (
|
|
16
|
+
id text primary key, kind text not null, owner_id text not null,
|
|
17
|
+
native_id text not null, node_id text not null references node(id),
|
|
18
|
+
latest_snapshot text not null, scope text,
|
|
19
|
+
lifecycle text not null check(lifecycle in ('active', 'archived', 'unlinked', 'missing')),
|
|
20
|
+
checked_at text not null, attempted_at text not null,
|
|
21
|
+
scan_status text not null check(scan_status in ('complete', 'partial', 'unavailable')),
|
|
22
|
+
unique(kind, owner_id, native_id)
|
|
23
|
+
)""",
|
|
24
|
+
"""create table if not exists source_snapshot (
|
|
25
|
+
id text primary key, source_id text not null references source(id),
|
|
26
|
+
previous_snapshot text references source_snapshot(id),
|
|
27
|
+
content_digest text not null, payload text not null, created_at text not null
|
|
28
|
+
)""",
|
|
29
|
+
"""create table if not exists source_relation (
|
|
30
|
+
id text primary key, snapshot_id text not null references source_snapshot(id),
|
|
31
|
+
from_node text not null references node(id), to_node text not null references node(id),
|
|
32
|
+
kind text not null, reason text not null, evidence text not null,
|
|
33
|
+
standing text not null check(standing in ('observed', 'user-stated', 'candidate'))
|
|
34
|
+
)""",
|
|
35
|
+
"create index if not exists source_relation_snapshot on source_relation(snapshot_id)",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def store_source_snapshot(snapshot: SourceSnapshot, *, home: Path | None = None) -> dict:
|
|
40
|
+
payload = snapshot_payload(snapshot)
|
|
41
|
+
with write_connection(database_path(home)) as connection:
|
|
42
|
+
_write_source(connection, source_id(snapshot.ref), payload, datetime.now(timezone.utc).isoformat())
|
|
43
|
+
return {"ok": True, "sourceId": source_id(snapshot.ref)}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def apply_source_scan(scan: SourceScan, *, home: Path | None = None) -> dict:
|
|
47
|
+
payloads, refs = _validated_scan(scan)
|
|
48
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
49
|
+
with write_connection(database_path(home)) as connection:
|
|
50
|
+
for snapshot, payload in zip(scan.snapshots, payloads):
|
|
51
|
+
_write_source(connection, source_id(snapshot.ref), payload, now)
|
|
52
|
+
_mark_unrefreshed_sources(connection, scan, refs, now)
|
|
53
|
+
return {"ok": True, "status": scan.status, "registered": len(payloads), "errors": list(scan.errors)}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _validated_scan(scan: SourceScan) -> tuple[list[dict], list[SourceRef]]:
|
|
57
|
+
payloads = [snapshot_payload(snapshot) for snapshot in scan.snapshots]
|
|
58
|
+
refs = [*scan.observed_refs, *(snapshot.ref for snapshot in scan.snapshots)]
|
|
59
|
+
validate_source_value({"domain": [scan.domain.kind, scan.domain.owner_id], "refs": [[ref.kind, ref.owner_id, ref.native_id] for ref in refs], "errors": list(scan.errors)})
|
|
60
|
+
if scan.status not in ("complete", "partial", "unavailable"):
|
|
61
|
+
raise DoyouknowError("invalid source scan status")
|
|
62
|
+
if scan.status == "complete" and scan.errors:
|
|
63
|
+
raise DoyouknowError("a complete source scan cannot contain errors")
|
|
64
|
+
if scan.domain.kind not in ("memory", "manager", "task") or not scan.domain.owner_id.strip():
|
|
65
|
+
raise DoyouknowError("invalid source scan domain")
|
|
66
|
+
if any((ref.kind, ref.owner_id) != (scan.domain.kind, scan.domain.owner_id) for ref in refs):
|
|
67
|
+
raise DoyouknowError("source scan contains a different domain")
|
|
68
|
+
if any(not isinstance(ref.native_id, str) or not ref.native_id.strip() for ref in refs):
|
|
69
|
+
raise DoyouknowError("source identity must be a nonempty string")
|
|
70
|
+
if len({source_id(snapshot.ref) for snapshot in scan.snapshots}) != len(scan.snapshots):
|
|
71
|
+
raise DoyouknowError("source scan contains duplicate snapshots")
|
|
72
|
+
return payloads, refs
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _mark_unrefreshed_sources(connection: sqlite3.Connection, scan: SourceScan, refs: list[SourceRef], now: str) -> None:
|
|
76
|
+
seen = {source_id(ref) for ref in refs}
|
|
77
|
+
updated = {source_id(snapshot.ref) for snapshot in scan.snapshots}
|
|
78
|
+
rows = connection.execute("select id from source where kind = ? and owner_id = ?", (scan.domain.kind, scan.domain.owner_id)).fetchall()
|
|
79
|
+
for (identity,) in rows:
|
|
80
|
+
if identity in updated:
|
|
81
|
+
continue
|
|
82
|
+
if scan.status == "complete" and identity not in seen:
|
|
83
|
+
connection.execute("update source set lifecycle = 'missing', attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
|
|
84
|
+
else:
|
|
85
|
+
connection.execute("update source set attempted_at = ?, scan_status = ? where id = ?", (now, scan.status, identity))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _write_source(connection: sqlite3.Connection, identity: str, payload: dict, now: str) -> None:
|
|
89
|
+
previous = connection.execute("select source.latest_snapshot, source_snapshot.content_digest from source inner join source_snapshot on source.latest_snapshot = source_snapshot.id where source.id = ?", (identity,)).fetchone()
|
|
90
|
+
digest = _digest(payload)
|
|
91
|
+
if previous and previous[1] == digest:
|
|
92
|
+
connection.execute("update source set checked_at = ?, attempted_at = ?, scan_status = 'complete', lifecycle = ? where id = ?", (now, now, payload["lifecycle"], identity))
|
|
93
|
+
return
|
|
94
|
+
nodes = payload["nodes"]
|
|
95
|
+
for node in nodes:
|
|
96
|
+
_insert_node(connection, node_id(node), node["kind"], node["scope"], node["key"], node["title"])
|
|
97
|
+
snapshot_id = stable_id("source-snapshot", identity, previous[0] if previous else "", digest)
|
|
98
|
+
ref = payload["ref"]
|
|
99
|
+
connection.execute("insert into source (id, kind, owner_id, native_id, node_id, latest_snapshot, scope, lifecycle, checked_at, attempted_at, scan_status) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'complete') on conflict(id) do update set node_id=excluded.node_id, latest_snapshot=excluded.latest_snapshot, scope=excluded.scope, lifecycle=excluded.lifecycle, checked_at=excluded.checked_at, attempted_at=excluded.attempted_at, scan_status=excluded.scan_status", (identity, ref["kind"], ref["owner_id"], ref["native_id"], node_id(payload["primary_node"]), snapshot_id, payload["scope"], payload["lifecycle"], now, now))
|
|
100
|
+
connection.execute("insert into source_snapshot (id, source_id, previous_snapshot, content_digest, payload, created_at) values (?, ?, ?, ?, ?, ?)", (snapshot_id, identity, previous[0] if previous else None, digest, canonical_json(payload), now))
|
|
101
|
+
identities = {node.get("ref", node["key"]): node_id(node) for node in nodes}
|
|
102
|
+
identities["$source"] = node_id(payload["primary_node"])
|
|
103
|
+
for index, relation in enumerate(payload["relations"]):
|
|
104
|
+
connection.execute("insert into source_relation (id, snapshot_id, from_node, to_node, kind, reason, evidence, standing) values (?, ?, ?, ?, ?, ?, ?, ?)", (stable_id("source-relation", snapshot_id, str(index)), snapshot_id, identities[relation["from"]], identities[relation["to"]], relation["kind"], relation["reason"], canonical_json(relation["evidence"]), relation["standing"]))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def list_sources(*, home: Path | None = None, scope: str | None = None, kind: str | None = None, limit: int = 100, after: str | None = None) -> dict:
|
|
108
|
+
if not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= 1000:
|
|
109
|
+
raise DoyouknowError("source limit must be between 1 and 1000")
|
|
110
|
+
if after is not None and (not isinstance(after, str) or not after.strip()):
|
|
111
|
+
raise DoyouknowError("source cursor must be a nonempty string")
|
|
112
|
+
path = database_path(home)
|
|
113
|
+
_require_database_location(path)
|
|
114
|
+
if not path.is_file():
|
|
115
|
+
return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
|
|
116
|
+
try:
|
|
117
|
+
with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=SQLITE_TIMEOUT_SECONDS)) as connection:
|
|
118
|
+
connection.execute("begin")
|
|
119
|
+
version = connection.execute("pragma user_version").fetchone()[0]
|
|
120
|
+
if version == 1:
|
|
121
|
+
return {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
|
|
122
|
+
if version != SCHEMA_VERSION:
|
|
123
|
+
raise DoyouknowError(f"unsupported doyouknow database version: {version}")
|
|
124
|
+
rows = connection.execute(
|
|
125
|
+
"""select source.id, source.node_id, source.latest_snapshot,
|
|
126
|
+
source.lifecycle, source.checked_at, source.attempted_at,
|
|
127
|
+
source.scan_status, source_snapshot.payload
|
|
128
|
+
from source inner join source_snapshot
|
|
129
|
+
on source.latest_snapshot = source_snapshot.id
|
|
130
|
+
where source.id > ? and (? is null or source.kind = ?)
|
|
131
|
+
and (? is null or source.scope = ?)
|
|
132
|
+
order by source.id limit ?""",
|
|
133
|
+
(after or "", kind, kind, scope, scope, limit + 1),
|
|
134
|
+
)
|
|
135
|
+
return _source_page(connection, rows, limit)
|
|
136
|
+
except sqlite3.Error as exc:
|
|
137
|
+
raise DoyouknowError(f"doyouknow source lookup failed: {exc}") from exc
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _source_page(connection: sqlite3.Connection, rows: sqlite3.Cursor, limit: int) -> dict:
|
|
141
|
+
result = {"ok": True, "sources": [], "truncated": False, "nextCursor": None}
|
|
142
|
+
item_bytes = 0
|
|
143
|
+
for row in rows:
|
|
144
|
+
if len(result["sources"]) >= limit:
|
|
145
|
+
result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
|
|
146
|
+
break
|
|
147
|
+
view = _source_view(connection, row)
|
|
148
|
+
view_bytes = len(canonical_json(view).encode("utf-8"))
|
|
149
|
+
header = {"ok": True, "sources": [], "truncated": True, "nextCursor": view["sourceId"]}
|
|
150
|
+
header_bytes = len(canonical_json(header).encode("utf-8"))
|
|
151
|
+
if header_bytes + view_bytes > MAX_INPUT_BYTES:
|
|
152
|
+
view = {key: view[key] for key in ("sourceId", "nodeId", "snapshotId", "lifecycle", "checkedAt", "attemptedAt", "scanStatus")}
|
|
153
|
+
view["detailsOmitted"] = True
|
|
154
|
+
view_bytes = len(canonical_json(view).encode("utf-8"))
|
|
155
|
+
required = header_bytes + item_bytes + view_bytes
|
|
156
|
+
if required > MAX_INPUT_BYTES:
|
|
157
|
+
result.update(truncated=True, nextCursor=result["sources"][-1]["sourceId"])
|
|
158
|
+
break
|
|
159
|
+
result["sources"].append(view)
|
|
160
|
+
item_bytes += view_bytes + 2
|
|
161
|
+
return result
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _source_view(connection: sqlite3.Connection, row: tuple) -> dict:
|
|
165
|
+
payload = json.loads(row[7])
|
|
166
|
+
relations = connection.execute("select from_node, to_node, kind, reason, evidence, standing from source_relation where snapshot_id = ? order by rowid", (row[2],)).fetchall()
|
|
167
|
+
return {"sourceId": row[0], "nodeId": row[1], "snapshotId": row[2],
|
|
168
|
+
"lifecycle": row[3], "checkedAt": row[4], "attemptedAt": row[5], "scanStatus": row[6],
|
|
169
|
+
"ref": payload["ref"], "scope": payload["scope"], "locator": payload["locator"],
|
|
170
|
+
"sourceUpdatedAt": payload["source_updated_at"], "metadata": payload["metadata"],
|
|
171
|
+
"nodes": [{"id": node_id(node), **node} for node in payload["nodes"]],
|
|
172
|
+
"relations": [{"fromNodeId": relation[0], "toNodeId": relation[1], "kind": relation[2], "reason": relation[3], "evidence": json.loads(relation[4]), "standing": relation[5]} for relation in relations]}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from okstra_project.dirs import contained_relative_path
|
|
6
|
+
|
|
7
|
+
from ...json_boundary import JsonBoundaryError, load_owned_object, read_bounded_bytes
|
|
8
|
+
from ..contracts import DoyouknowError, MAX_INPUT_BYTES
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def safe_source_path(root: Path, relative: str | Path) -> Path:
|
|
12
|
+
base = root.resolve()
|
|
13
|
+
contained = contained_relative_path(str(relative), base)
|
|
14
|
+
if contained is None:
|
|
15
|
+
raise DoyouknowError("source path is outside its registered root")
|
|
16
|
+
return base / contained
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def read_source_text(root: Path, relative: str | Path) -> str:
|
|
20
|
+
path = safe_source_path(root, relative)
|
|
21
|
+
if not path.is_file():
|
|
22
|
+
raise DoyouknowError("source text is not a regular file")
|
|
23
|
+
try:
|
|
24
|
+
return read_bounded_bytes(path, max_bytes=MAX_INPUT_BYTES, artifact="source text").decode("utf-8")
|
|
25
|
+
except (OSError, UnicodeError, JsonBoundaryError) as exc:
|
|
26
|
+
raise DoyouknowError("source text is unreadable or exceeds the input limit") from exc
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def read_source_object(root: Path, relative: str | Path) -> dict:
|
|
30
|
+
path = safe_source_path(root, relative)
|
|
31
|
+
if not path.is_file():
|
|
32
|
+
raise DoyouknowError("source object is not a regular file")
|
|
33
|
+
try:
|
|
34
|
+
return load_owned_object(path, artifact="source object", max_bytes=MAX_INPUT_BYTES)
|
|
35
|
+
except (OSError, JsonBoundaryError, RecursionError) as exc:
|
|
36
|
+
raise DoyouknowError("source object is unreadable or exceeds the input limit") from exc
|