okstra 0.233.0 → 0.234.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-registry.mjs +6 -0
- package/dist/cli-registry.mjs.map +1 -1
- package/docs/architecture/storage-model.md +19 -0
- package/docs/cli.md +41 -1
- package/docs/project-structure-overview.md +3 -2
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/python/okstra_ctl/business_flow/cli.py +60 -6
- package/runtime/python/okstra_ctl/business_flow/contracts.py +12 -1
- package/runtime/python/okstra_ctl/business_flow/engine.py +17 -5
- package/runtime/python/okstra_ctl/business_flow/history.py +60 -0
- package/runtime/python/okstra_ctl/business_flow/source.py +35 -2
- package/runtime/python/okstra_ctl/business_flow/store.py +51 -14
- package/runtime/python/okstra_ctl/business_flow/sweep.py +124 -0
- package/runtime/python/okstra_ctl/doyouknow/cli.py +9 -9
- package/runtime/python/okstra_ctl/doyouknow/config.py +2 -2
- package/runtime/python/okstra_ctl/doyouknow/group_view.py +152 -0
- package/runtime/python/okstra_ctl/doyouknow/source_contracts.py +4 -3
- package/runtime/python/okstra_ctl/doyouknow/source_store.py +2 -2
- package/runtime/python/okstra_ctl/doyouknow/sources/business.py +116 -0
- package/runtime/python/okstra_ctl/doyouknow/sync.py +34 -9
- package/runtime/python/okstra_ctl/fixed_text.py +10 -0
- package/runtime/python/okstra_ctl/project_group.py +175 -0
- package/runtime/python/okstra_ctl/project_group_cli.py +62 -0
- package/runtime/schemas/business-flow-v1.schema.json +3 -0
- package/runtime/schemas/doyouknow-config-v1.schema.json +1 -1
- package/runtime/skills/okstra-explain-flow/SKILL.md +27 -2
- package/runtime/templates/doyouknow/group-view.css +103 -0
- package/runtime/templates/doyouknow/group-view.js +317 -0
- package/runtime/templates/doyouknow/group-view.template.html +72 -0
|
@@ -25,7 +25,7 @@ from .contracts import (
|
|
|
25
25
|
read_artifact,
|
|
26
26
|
write_artifact,
|
|
27
27
|
)
|
|
28
|
-
from .source import candidate_projects, capture_project
|
|
28
|
+
from .source import candidate_projects, capture_project, knowledge_projects
|
|
29
29
|
|
|
30
30
|
CONFIRMED_LEVELS = frozenset({"static", "execution-verified"})
|
|
31
31
|
PACKET_CLAIM_LIMIT = 40
|
|
@@ -160,12 +160,17 @@ def _mark_conflicts(claims: list[StoredClaim]) -> None:
|
|
|
160
160
|
|
|
161
161
|
|
|
162
162
|
def resolve_conflicts(
|
|
163
|
-
project_root: Path,
|
|
163
|
+
project_root: Path,
|
|
164
|
+
resolutions: list[BusinessResolution],
|
|
165
|
+
producer: ClaimProducer,
|
|
166
|
+
project_group: str,
|
|
164
167
|
) -> None:
|
|
165
168
|
path = artifact_path(project_root, "knowledge.json")
|
|
166
169
|
with dir_flock(path.parent, ".knowledge.lock"):
|
|
167
170
|
data = read_knowledge(project_root)
|
|
168
|
-
validate_resolutions(
|
|
171
|
+
validate_resolutions(
|
|
172
|
+
read_shared_knowledge(project_root, project_group), resolutions
|
|
173
|
+
)
|
|
169
174
|
by_id = {row["id"]: row for row in data["claims"]}
|
|
170
175
|
for resolution in resolutions:
|
|
171
176
|
ids = resolution["claimIds"]
|
|
@@ -238,9 +243,11 @@ def _resolution_rows(
|
|
|
238
243
|
return rows
|
|
239
244
|
|
|
240
245
|
|
|
241
|
-
def read_shared_knowledge(project_root: Path) -> dict[str, Any]:
|
|
246
|
+
def read_shared_knowledge(project_root: Path, project_group: str) -> dict[str, Any]:
|
|
242
247
|
claims, resolutions, errors = [], [], []
|
|
243
|
-
candidates =
|
|
248
|
+
candidates = knowledge_projects(
|
|
249
|
+
FlowRequest(project_root, "business", project_group=project_group)
|
|
250
|
+
)
|
|
244
251
|
for candidate in candidates:
|
|
245
252
|
root = Path(candidate["root"])
|
|
246
253
|
try:
|
|
@@ -289,17 +296,40 @@ def _apply_shared_resolutions(
|
|
|
289
296
|
|
|
290
297
|
|
|
291
298
|
def query_knowledge(
|
|
292
|
-
project_root: Path,
|
|
299
|
+
project_root: Path,
|
|
300
|
+
query: str,
|
|
301
|
+
source_root: Path | None = None,
|
|
302
|
+
project_group: str = "",
|
|
303
|
+
*,
|
|
304
|
+
members_only: bool = True,
|
|
293
305
|
) -> dict[str, Any]:
|
|
306
|
+
"""Select stored claims for a question.
|
|
307
|
+
|
|
308
|
+
With a group, `members_only` keeps the claims its members own. Their status still
|
|
309
|
+
comes from the whole knowledge scope, so a conflict with an outside claim shows.
|
|
310
|
+
"""
|
|
294
311
|
sources = {
|
|
295
312
|
row["projectId"]: row["root"]
|
|
296
|
-
for row in
|
|
297
|
-
FlowRequest(
|
|
313
|
+
for row in knowledge_projects(
|
|
314
|
+
FlowRequest(
|
|
315
|
+
project_root,
|
|
316
|
+
query or "business",
|
|
317
|
+
source_root=source_root,
|
|
318
|
+
project_group=project_group,
|
|
319
|
+
)
|
|
298
320
|
)
|
|
299
321
|
}
|
|
300
322
|
tokens = set(re.findall(r"[\w-]{2,}", query.casefold()))
|
|
301
|
-
shared = read_shared_knowledge(project_root)
|
|
323
|
+
shared = read_shared_knowledge(project_root, project_group)
|
|
302
324
|
claims, errors = shared["claims"], shared["errors"]
|
|
325
|
+
if project_group and members_only:
|
|
326
|
+
member_roots = {
|
|
327
|
+
row["root"]
|
|
328
|
+
for row in candidate_projects(
|
|
329
|
+
FlowRequest(project_root, "business", project_group=project_group)
|
|
330
|
+
)
|
|
331
|
+
}
|
|
332
|
+
claims = [row for row in claims if row["ownerRoot"] in member_roots]
|
|
303
333
|
selected = [
|
|
304
334
|
row
|
|
305
335
|
for row in claims
|
|
@@ -328,15 +358,22 @@ def query_knowledge(
|
|
|
328
358
|
selected, keys = expanded, {row["key"] for row in expanded}
|
|
329
359
|
states: dict[str, dict[str, Any]] = {}
|
|
330
360
|
for row in selected:
|
|
331
|
-
row["freshness"] =
|
|
361
|
+
row["freshness"] = source_freshness(row["sourceStates"], states, sources)
|
|
332
362
|
row["selectionReason"] = "business terms or a connected claim key"
|
|
333
363
|
return {"claims": selected, "errors": errors, "query": query}
|
|
334
364
|
|
|
335
365
|
|
|
336
|
-
def
|
|
337
|
-
|
|
366
|
+
def source_freshness(
|
|
367
|
+
source_states: dict[str, Any], cache: dict[str, dict[str, Any]], sources: dict[str, str]
|
|
338
368
|
) -> str:
|
|
339
|
-
|
|
369
|
+
"""Compare recorded source versions with the projects as they are now.
|
|
370
|
+
|
|
371
|
+
`cache` keeps one capture per root across calls; `sources` maps a projectId to its root.
|
|
372
|
+
"""
|
|
373
|
+
for project_id, expected in source_states.items():
|
|
374
|
+
# A version that was not captured when it was recorded cannot be compared.
|
|
375
|
+
if not expected.get("digest"):
|
|
376
|
+
return "unverified"
|
|
340
377
|
root = sources.get(project_id)
|
|
341
378
|
if root is None:
|
|
342
379
|
return "unverified"
|
|
@@ -347,7 +384,7 @@ def _freshness(
|
|
|
347
384
|
return "unverified"
|
|
348
385
|
if current["digest"] != expected["digest"]:
|
|
349
386
|
return "historical"
|
|
350
|
-
return "current" if
|
|
387
|
+
return "current" if source_states else "unverified"
|
|
351
388
|
|
|
352
389
|
|
|
353
390
|
def knowledge_packet(
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Explain a list of questions across a project group, skipping the ones still current."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ..paths import okstra_home
|
|
8
|
+
from ..json_boundary import JsonBoundaryError
|
|
9
|
+
from ..project_group import ProjectGroupError, show_group
|
|
10
|
+
from .contracts import BusinessFlowError, FlowRequest, flow_report_language
|
|
11
|
+
from .engine import explain_flow
|
|
12
|
+
from .history import flow_id, latest_explanations, recorded_source_states
|
|
13
|
+
from .invocation import FlowExecutor
|
|
14
|
+
from .source import registered_roots
|
|
15
|
+
from .store import source_freshness
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _questions(rows: object) -> list[tuple[str, str]]:
|
|
19
|
+
if not isinstance(rows, list):
|
|
20
|
+
raise BusinessFlowError("sweep questions must be a JSON array")
|
|
21
|
+
questions = []
|
|
22
|
+
for row in rows:
|
|
23
|
+
if (
|
|
24
|
+
not isinstance(row, dict)
|
|
25
|
+
or not isinstance(row.get("projectId"), str)
|
|
26
|
+
or not isinstance(row.get("question"), str)
|
|
27
|
+
or not row["question"].strip()
|
|
28
|
+
):
|
|
29
|
+
raise BusinessFlowError('each sweep question needs a "projectId" and a non-empty "question"')
|
|
30
|
+
questions.append((row["projectId"], row["question"]))
|
|
31
|
+
return questions
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def plan_sweep(project_group: str, questions: object, *, stale: bool, home: Path | None = None) -> dict[str, Any]:
|
|
35
|
+
"""Decide what a sweep would explain. No model is called.
|
|
36
|
+
|
|
37
|
+
A listed question is a target unless its newest explanation is still `current`; its
|
|
38
|
+
reason is `new` (never explained), `historical` (a recorded project changed) or
|
|
39
|
+
`unverified` (a recorded version cannot be compared, so it is a target on every sweep).
|
|
40
|
+
With `stale`, every stored explanation that is `historical` is a target too.
|
|
41
|
+
A member whose root is missing or unreadable is reported in `problems`, and each of
|
|
42
|
+
its listed questions is reported there as not planned.
|
|
43
|
+
"""
|
|
44
|
+
home = home or okstra_home()
|
|
45
|
+
try:
|
|
46
|
+
group = show_group(home, project_group)
|
|
47
|
+
except ProjectGroupError as exc:
|
|
48
|
+
raise BusinessFlowError(str(exc)) from exc
|
|
49
|
+
members = {member["projectId"]: member for member in group["projects"]}
|
|
50
|
+
roots = registered_roots(group["projectGroup"], with_run_history=True, home=home)
|
|
51
|
+
captured: dict[str, dict[str, Any]] = {}
|
|
52
|
+
targets: dict[tuple[str, str], dict[str, str]] = {}
|
|
53
|
+
skipped: dict[tuple[str, str], dict[str, str]] = {}
|
|
54
|
+
problems = []
|
|
55
|
+
stored: dict[str, dict[str, tuple[str, str]]] = {}
|
|
56
|
+
for project_id, member in sorted(members.items()):
|
|
57
|
+
if not member["rootExists"]:
|
|
58
|
+
problems.append(f"{project_id}: registered root is missing: {member['projectRoot']}")
|
|
59
|
+
continue
|
|
60
|
+
try:
|
|
61
|
+
latest, refused = latest_explanations(Path(member["projectRoot"]))
|
|
62
|
+
except (OSError, BusinessFlowError) as exc:
|
|
63
|
+
problems.append(f"{project_id}: stored explanations could not be listed: {exc}")
|
|
64
|
+
continue
|
|
65
|
+
if refused:
|
|
66
|
+
problems.append(f"{project_id}: {refused} stored explanation record(s) could not be read")
|
|
67
|
+
stored[project_id] = {
|
|
68
|
+
key: (execution["request"]["question"], source_freshness(recorded_source_states(execution), captured, roots))
|
|
69
|
+
for key, (_, execution) in latest.items()
|
|
70
|
+
}
|
|
71
|
+
for project_id, question in _questions(questions):
|
|
72
|
+
if project_id not in members:
|
|
73
|
+
raise BusinessFlowError(f"project {project_id} is not in project-group {group['projectGroup']}")
|
|
74
|
+
if project_id not in stored:
|
|
75
|
+
problems.append(f"{project_id}: not planned because the project could not be read: {question}")
|
|
76
|
+
continue
|
|
77
|
+
key = flow_id(question)
|
|
78
|
+
freshness = stored[project_id].get(key, (question, "new"))[1]
|
|
79
|
+
row = {"projectId": project_id, "question": question, "reason": freshness}
|
|
80
|
+
(skipped if freshness == "current" else targets)[(project_id, key)] = row
|
|
81
|
+
if stale:
|
|
82
|
+
for project_id, flows in stored.items():
|
|
83
|
+
for key, (question, freshness) in flows.items():
|
|
84
|
+
if freshness == "historical":
|
|
85
|
+
targets.setdefault((project_id, key), {"projectId": project_id, "question": question, "reason": freshness})
|
|
86
|
+
return {
|
|
87
|
+
"projectGroup": group["projectGroup"],
|
|
88
|
+
"targets": [targets[key] for key in sorted(targets)],
|
|
89
|
+
"skipped": [skipped[key] for key in sorted(skipped)],
|
|
90
|
+
"problems": problems,
|
|
91
|
+
"roots": {name: member["projectRoot"] for name, member in members.items()},
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def run_sweep(
|
|
96
|
+
plan: dict[str, Any], *, report_language: str, host_runtime: str, executor: FlowExecutor | None = None
|
|
97
|
+
) -> dict[str, Any]:
|
|
98
|
+
"""Explain every planned target in order. One failure does not stop the rest.
|
|
99
|
+
|
|
100
|
+
A target that fails before its execution record exists is reported with an empty
|
|
101
|
+
`executionId`.
|
|
102
|
+
"""
|
|
103
|
+
results = []
|
|
104
|
+
for target in plan["targets"]:
|
|
105
|
+
root = Path(plan["roots"][target["projectId"]])
|
|
106
|
+
try:
|
|
107
|
+
execution = explain_flow(
|
|
108
|
+
FlowRequest(
|
|
109
|
+
root,
|
|
110
|
+
target["question"],
|
|
111
|
+
report_language=flow_report_language(root, report_language),
|
|
112
|
+
host_runtime=host_runtime,
|
|
113
|
+
project_group=plan["projectGroup"],
|
|
114
|
+
),
|
|
115
|
+
executor,
|
|
116
|
+
)
|
|
117
|
+
except (OSError, BusinessFlowError, JsonBoundaryError) as exc:
|
|
118
|
+
# explain_flow validates the request and writes its first record before it starts catching.
|
|
119
|
+
results.append({**target, "executionId": "", "status": "failed", "error": str(exc)})
|
|
120
|
+
continue
|
|
121
|
+
results.append({
|
|
122
|
+
**target, "executionId": execution["id"], "status": execution["status"], "error": execution.get("error", ""),
|
|
123
|
+
})
|
|
124
|
+
return {**plan, "results": results, "status": "failed" if any(row["status"] == "failed" for row in results) else "complete"}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
|
-
import json
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
from typing import get_args
|
|
7
6
|
|
|
8
|
-
from ..fixed_text import
|
|
7
|
+
from ..fixed_text import emit_result
|
|
9
8
|
from ..json_boundary import JsonBoundaryError, external_user_json_source, load_external_json
|
|
10
9
|
from .config import load_config, save_config
|
|
11
10
|
from .contracts import DoyouknowError, MAX_INPUT_BYTES, read_envelope
|
|
@@ -16,7 +15,7 @@ from .source_contracts import SourceKind
|
|
|
16
15
|
def _parser() -> argparse.ArgumentParser:
|
|
17
16
|
parser = argparse.ArgumentParser(prog="okstra doyouknow", description="Store linked work records without starting a run.")
|
|
18
17
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
19
|
-
for name in ("capture", "ingest", "show", "sync", "list", "config", "register", "retry"):
|
|
18
|
+
for name in ("capture", "ingest", "show", "sync", "list", "config", "register", "retry", "view"):
|
|
20
19
|
command = commands.add_parser(name)
|
|
21
20
|
if name == "show":
|
|
22
21
|
command.add_argument("record_id")
|
|
@@ -26,6 +25,8 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
26
25
|
command.add_argument("--file", type=Path, help="replace source settings from JSON")
|
|
27
26
|
elif name == "register":
|
|
28
27
|
command.add_argument("--ticket", required=True, help="pending registration UUID")
|
|
28
|
+
elif name == "view":
|
|
29
|
+
command.add_argument("--project-group", required=True, help="registered project group to render")
|
|
29
30
|
if name in ("sync", "list", "retry"):
|
|
30
31
|
command.add_argument("--source", choices=get_args(SourceKind))
|
|
31
32
|
if name == "retry":
|
|
@@ -41,12 +42,7 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
def _emit(payload: dict, *, text: bool) -> None:
|
|
44
|
-
|
|
45
|
-
print(json.dumps(payload, ensure_ascii=False, allow_nan=False))
|
|
46
|
-
return
|
|
47
|
-
rows = ["Okstra doyouknow\n", line("OK", payload["ok"])]
|
|
48
|
-
rows.extend(value_lines("Result", payload))
|
|
49
|
-
print("".join(rows), end="")
|
|
45
|
+
emit_result(payload, text=text, title="Okstra doyouknow")
|
|
50
46
|
|
|
51
47
|
|
|
52
48
|
def main(argv: list[str] | None = None) -> int:
|
|
@@ -62,6 +58,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
62
58
|
result = sync_sources(source=args.source)
|
|
63
59
|
elif args.command == "register":
|
|
64
60
|
result = register_ticket(args.ticket)
|
|
61
|
+
elif args.command == "view":
|
|
62
|
+
from .group_view import write_group_view
|
|
63
|
+
|
|
64
|
+
result = write_group_view(args.project_group)
|
|
65
65
|
elif args.command == "retry":
|
|
66
66
|
result = retry_pending(source=args.source, limit=args.limit)
|
|
67
67
|
elif args.command == "list":
|
|
@@ -6,7 +6,7 @@ from typing import TypedDict
|
|
|
6
6
|
from ..json_boundary import load_owned_object, serialize_owned_object, write_owned_object_atomic
|
|
7
7
|
from ..run_context import dir_flock
|
|
8
8
|
from .contracts import MAX_INPUT_BYTES, asset_object
|
|
9
|
-
from .source_contracts import validate_source_value
|
|
9
|
+
from .source_contracts import SOURCE_KINDS, validate_source_value
|
|
10
10
|
from .paths import database_path, ensure_storage_root, require_database_location as _require_database_location
|
|
11
11
|
|
|
12
12
|
|
|
@@ -26,7 +26,7 @@ def load_config(*, home: Path | None = None) -> SourceConfig:
|
|
|
26
26
|
_require_database_location(path)
|
|
27
27
|
if not path.exists():
|
|
28
28
|
return {"schemaVersion": 1, "autoRegister": True, "scope": None,
|
|
29
|
-
"sources":
|
|
29
|
+
"sources": list(SOURCE_KINDS), "includeProjects": [],
|
|
30
30
|
"excludeProjects": [], "includeGroups": [], "excludeGroups": []}
|
|
31
31
|
payload = load_owned_object(path, artifact="doyouknow config", max_bytes=MAX_INPUT_BYTES)
|
|
32
32
|
validate_source_value(payload)
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""One project group's projects, business flows and linked records as a single HTML page.
|
|
2
|
+
|
|
3
|
+
The page is a derived view: it reads the group registry and the sources `okstra doyouknow
|
|
4
|
+
sync` already stored, and compares each flow's recorded source versions with the
|
|
5
|
+
projects as they are now. It collects nothing itself.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import html
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from ..business_flow.contracts import BusinessFlowError
|
|
16
|
+
from ..business_flow.source import registered_roots
|
|
17
|
+
from ..business_flow.store import source_freshness
|
|
18
|
+
from ..paths import find_asset_root
|
|
19
|
+
from ..project_group import ProjectGroupError, group_path, show_group
|
|
20
|
+
from .contracts import DoyouknowError
|
|
21
|
+
from .paths import database_path, ensure_storage_root
|
|
22
|
+
from .source_store import list_sources
|
|
23
|
+
|
|
24
|
+
ASSET_DIRECTORY = ("templates", "doyouknow")
|
|
25
|
+
PAGE, STYLE, SCRIPT = "group-view.template.html", "group-view.css", "group-view.js"
|
|
26
|
+
_MARKER = re.compile(r"\{\{(TITLE|STYLE|SCRIPT|DATA)\}\}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _stored_sources(home: Path) -> tuple[list[dict], list[str]]:
|
|
30
|
+
"""Every active stored source, and a note for each one too large to be listed."""
|
|
31
|
+
sources, problems, after = [], [], None
|
|
32
|
+
while True:
|
|
33
|
+
listing = list_sources(home=home, limit=1000, after=after)
|
|
34
|
+
for source in listing["sources"]:
|
|
35
|
+
if source["lifecycle"] != "active":
|
|
36
|
+
continue
|
|
37
|
+
if source.get("detailsOmitted"):
|
|
38
|
+
# An omitted item carries no project, so it cannot be matched to a group.
|
|
39
|
+
problems.append(f"A stored source is too large to list, so its project group is unknown: {source['sourceId']}")
|
|
40
|
+
else:
|
|
41
|
+
sources.append(source)
|
|
42
|
+
if not listing["truncated"]:
|
|
43
|
+
return sources, problems
|
|
44
|
+
after = listing["nextCursor"]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _title(source: dict) -> str:
|
|
48
|
+
return next(node["title"] for node in source["nodes"] if node["id"] == source["nodeId"])
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _project_ids(source: dict) -> set[str]:
|
|
52
|
+
return {node["key"] for node in source["nodes"] if node["kind"] == "project"}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _flow(source: dict, freshness: str) -> dict:
|
|
56
|
+
metadata = source["metadata"]
|
|
57
|
+
return {
|
|
58
|
+
"id": source["sourceId"], "title": _title(source), "owner": metadata["projectId"], "locator": source["locator"],
|
|
59
|
+
"updatedAt": source["sourceUpdatedAt"] or "", "freshness": freshness, "projects": sorted(_project_ids(source)),
|
|
60
|
+
**{key: metadata[key] for key in (
|
|
61
|
+
"question", "summary", "status", "terms", "steps", "facts", "gaps", "questions",
|
|
62
|
+
"inspectedProjects", "relationships",
|
|
63
|
+
)},
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _edges(source: dict) -> list[dict]:
|
|
68
|
+
projects = {node["id"]: node["key"] for node in source["nodes"] if node["kind"] == "project"}
|
|
69
|
+
return [
|
|
70
|
+
{
|
|
71
|
+
"from": projects[relation["fromNodeId"]], "to": projects[relation["toNodeId"]],
|
|
72
|
+
"kind": relation["kind"], "reason": relation["reason"], "evidence": relation["evidence"],
|
|
73
|
+
"standing": relation["standing"], "flowId": source["sourceId"],
|
|
74
|
+
}
|
|
75
|
+
for relation in source["relations"]
|
|
76
|
+
if relation["fromNodeId"] in projects and relation["toNodeId"] in projects
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def group_view_data(home: Path, project_group: str, *, now: str | None = None) -> dict:
|
|
81
|
+
try:
|
|
82
|
+
group = show_group(home, project_group)
|
|
83
|
+
# The same projects the knowledge query compares against, so a flow gets one freshness in both.
|
|
84
|
+
roots = registered_roots(group["projectGroup"], with_run_history=True, home=home)
|
|
85
|
+
except (ProjectGroupError, BusinessFlowError) as exc:
|
|
86
|
+
raise DoyouknowError(str(exc)) from exc
|
|
87
|
+
members = {member["projectId"]: member for member in group["projects"]}
|
|
88
|
+
stored, problems = _stored_sources(home)
|
|
89
|
+
captured: dict[str, dict] = {}
|
|
90
|
+
flows, edges, records = [], [], []
|
|
91
|
+
for source in stored:
|
|
92
|
+
if source["ref"]["kind"] != "business":
|
|
93
|
+
named_members = _project_ids(source) & members.keys()
|
|
94
|
+
if named_members:
|
|
95
|
+
records.append({
|
|
96
|
+
"kind": source["ref"]["kind"], "title": _title(source), "projects": sorted(named_members),
|
|
97
|
+
"locator": source["locator"], "updatedAt": source["sourceUpdatedAt"] or "",
|
|
98
|
+
})
|
|
99
|
+
elif source["ref"]["owner_id"] in members:
|
|
100
|
+
flows.append(_flow(source, source_freshness(source["metadata"]["sourceStates"], captured, roots)))
|
|
101
|
+
edges.extend(_edges(source))
|
|
102
|
+
inspected = {name for flow in flows for name in flow["inspectedProjects"]}
|
|
103
|
+
return {
|
|
104
|
+
"projectGroup": group["projectGroup"],
|
|
105
|
+
"generatedAt": now or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
106
|
+
"projects": [
|
|
107
|
+
{
|
|
108
|
+
"projectId": name, "projectRoot": member["projectRoot"], "rootExists": member["rootExists"],
|
|
109
|
+
"investigated": name in inspected, "flowCount": sum(flow["owner"] == name for flow in flows),
|
|
110
|
+
}
|
|
111
|
+
for name, member in sorted(members.items())
|
|
112
|
+
],
|
|
113
|
+
"flows": sorted(flows, key=lambda flow: flow["title"].casefold()),
|
|
114
|
+
"edges": edges,
|
|
115
|
+
"records": sorted(records, key=lambda record: record["updatedAt"], reverse=True),
|
|
116
|
+
"problems": problems,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _asset(name: str) -> str:
|
|
121
|
+
asset_root = find_asset_root((*ASSET_DIRECTORY, name))
|
|
122
|
+
if asset_root is None:
|
|
123
|
+
raise DoyouknowError("group view asset not found: " + "/".join((*ASSET_DIRECTORY, name)))
|
|
124
|
+
return asset_root.joinpath(*ASSET_DIRECTORY, name).read_text(encoding="utf-8")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def render_group_view(home: Path, project_group: str, *, now: str | None = None) -> tuple[str, dict]:
|
|
128
|
+
data = group_view_data(home, project_group, now=now)
|
|
129
|
+
replacements = {
|
|
130
|
+
"TITLE": html.escape("Okstra project group · " + data["projectGroup"]),
|
|
131
|
+
"STYLE": _asset(STYLE),
|
|
132
|
+
"SCRIPT": _asset(SCRIPT),
|
|
133
|
+
# Inside a script element only `<` can end the element or open a comment.
|
|
134
|
+
"DATA": json.dumps(data, ensure_ascii=False).replace("<", "\\u003c"),
|
|
135
|
+
}
|
|
136
|
+
# One pass: replacing in turn would rescan stored text that happens to contain a marker.
|
|
137
|
+
return _MARKER.sub(lambda match: replacements[match.group(1)], _asset(PAGE)), data
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def write_group_view(project_group: str, *, home: Path | None = None, now: str | None = None) -> dict:
|
|
141
|
+
database = database_path(home)
|
|
142
|
+
okstra_home_root = database.parent.parent
|
|
143
|
+
html_page, data = render_group_view(okstra_home_root, project_group, now=now)
|
|
144
|
+
ensure_storage_root(database.parent)
|
|
145
|
+
path = database.parent / "view" / group_path(okstra_home_root, data["projectGroup"]).stem / "index.html"
|
|
146
|
+
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
147
|
+
path.write_text(html_page, encoding="utf-8")
|
|
148
|
+
return {
|
|
149
|
+
"ok": True, "projectGroup": data["projectGroup"], "viewPath": str(path), "viewUrl": path.as_uri(),
|
|
150
|
+
"projects": len(data["projects"]), "flows": len(data["flows"]), "relationships": len(data["edges"]),
|
|
151
|
+
"records": len(data["records"]), "problems": data["problems"],
|
|
152
|
+
}
|
|
@@ -3,14 +3,15 @@ from __future__ import annotations
|
|
|
3
3
|
import json
|
|
4
4
|
from dataclasses import asdict, dataclass, field
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Literal
|
|
6
|
+
from typing import Literal, get_args
|
|
7
7
|
|
|
8
8
|
from ..json_boundary import serialize_owned_object
|
|
9
9
|
from .contracts import (DoyouknowError, MAX_INPUT_BYTES, Node, Relation, _check_input_depth,
|
|
10
10
|
_decoded_strings, asset_object, canonical_json, record_secret_policy_digest, reject_secrets)
|
|
11
11
|
from .identity import stable_id
|
|
12
12
|
|
|
13
|
-
SourceKind = Literal["memory", "manager", "task"]
|
|
13
|
+
SourceKind = Literal["memory", "manager", "task", "business"]
|
|
14
|
+
SOURCE_KINDS = get_args(SourceKind)
|
|
14
15
|
Lifecycle = Literal["active", "archived", "unlinked", "missing"]
|
|
15
16
|
JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
|
|
16
17
|
|
|
@@ -99,7 +100,7 @@ def snapshot_payload(snapshot: SourceSnapshot) -> dict:
|
|
|
99
100
|
|
|
100
101
|
|
|
101
102
|
def _require_snapshot_fields(snapshot: SourceSnapshot) -> None:
|
|
102
|
-
if snapshot.ref.kind not in
|
|
103
|
+
if snapshot.ref.kind not in SOURCE_KINDS or snapshot.lifecycle not in ("active", "archived", "unlinked", "missing"):
|
|
103
104
|
raise DoyouknowError("invalid source kind or lifecycle")
|
|
104
105
|
for value in (snapshot.ref.owner_id, snapshot.ref.native_id, snapshot.locator):
|
|
105
106
|
if not isinstance(value, str) or not value.strip():
|
|
@@ -8,7 +8,7 @@ from pathlib import Path
|
|
|
8
8
|
|
|
9
9
|
from .contracts import DoyouknowError, MAX_INPUT_BYTES, canonical_json
|
|
10
10
|
from .identity import node_id, source_id, stable_id
|
|
11
|
-
from .source_contracts import SourceRef, SourceScan, SourceSnapshot, snapshot_payload, validate_source_value
|
|
11
|
+
from .source_contracts import SOURCE_KINDS, SourceRef, SourceScan, SourceSnapshot, snapshot_payload, validate_source_value
|
|
12
12
|
from .store import SCHEMA_VERSION, SQLITE_TIMEOUT_SECONDS, _digest, _insert_node, _require_database_location, database_path, write_connection
|
|
13
13
|
|
|
14
14
|
SOURCE_SCHEMA_STATEMENTS = (
|
|
@@ -61,7 +61,7 @@ def _validated_scan(scan: SourceScan) -> tuple[list[dict], list[SourceRef]]:
|
|
|
61
61
|
raise DoyouknowError("invalid source scan status")
|
|
62
62
|
if scan.status == "complete" and scan.errors:
|
|
63
63
|
raise DoyouknowError("a complete source scan cannot contain errors")
|
|
64
|
-
if scan.domain.kind not in
|
|
64
|
+
if scan.domain.kind not in SOURCE_KINDS or not scan.domain.owner_id.strip():
|
|
65
65
|
raise DoyouknowError("invalid source scan domain")
|
|
66
66
|
if any((ref.kind, ref.owner_id) != (scan.domain.kind, scan.domain.owner_id) for ref in refs):
|
|
67
67
|
raise DoyouknowError("source scan contains a different domain")
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
from okstra_project.dirs import PROJECT_JSON_RELATIVE
|
|
7
|
+
|
|
8
|
+
from ...business_flow.contracts import BusinessFact, BusinessFlowError, BusinessResult, FlowExecution, business_root, schema_definition
|
|
9
|
+
from ...business_flow.history import flow_id, latest_explanations, recorded_source_states
|
|
10
|
+
from ...business_flow.store import CONFIRMED_LEVELS
|
|
11
|
+
from ...json_boundary import JsonBoundaryError, load_owned_object
|
|
12
|
+
from ..config import SourceConfig, includes_project
|
|
13
|
+
from ..contracts import MAX_INPUT_BYTES, DoyouknowError, Node, Relation
|
|
14
|
+
from ..source_contracts import SourceDomain, SourceRef, SourceScan, SourceSnapshot, snapshot_payload
|
|
15
|
+
from .io import read_source_object, safe_source_path
|
|
16
|
+
|
|
17
|
+
_UNREADABLE = (DoyouknowError, BusinessFlowError, JsonBoundaryError, OSError, RecursionError)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _evidence_refs(fact: BusinessFact) -> list[str]:
|
|
21
|
+
return [f"{row['projectId']}:{row['path']}:{row['line']}" for row in fact["evidence"]]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _project_node(project_id: str) -> Node:
|
|
25
|
+
return {"kind": "project", "scope": "okstra-project", "key": project_id, "title": project_id, "ref": "project:" + project_id}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _graph(project_id: str, result: BusinessResult, locator: str) -> tuple[list[Node], list[Relation]]:
|
|
29
|
+
facts = {fact["key"]: fact for fact in result["facts"]}
|
|
30
|
+
involved = {project_id, *result["inspectedProjects"]}
|
|
31
|
+
for step in result["steps"]:
|
|
32
|
+
involved.update(step["projectIds"])
|
|
33
|
+
for row in result["relationships"]:
|
|
34
|
+
involved.update((row["from"], row["to"]))
|
|
35
|
+
nodes = [_project_node(name) for name in sorted(involved)]
|
|
36
|
+
relations: list[Relation] = [
|
|
37
|
+
{
|
|
38
|
+
"from": "$source", "to": "project:" + name,
|
|
39
|
+
"kind": "belongs-to" if name == project_id else "involves",
|
|
40
|
+
"reason": "The project stores this explanation." if name == project_id else "The explanation covers this project.",
|
|
41
|
+
"evidence": [locator], "standing": "observed",
|
|
42
|
+
}
|
|
43
|
+
for name in sorted(involved)
|
|
44
|
+
]
|
|
45
|
+
# The explanation schema allows a blank term or meaning; a graph node and its relation do not.
|
|
46
|
+
terms = {row["term"].casefold(): row for row in result["terms"] if row["term"].strip() and row["meaning"].strip()}
|
|
47
|
+
for index, (key, row) in enumerate(sorted(terms.items())):
|
|
48
|
+
ref = f"term:{index}"
|
|
49
|
+
nodes.append({"kind": "concept", "scope": project_id, "key": key, "title": row["term"], "ref": ref})
|
|
50
|
+
relations.append({"from": "$source", "to": ref, "kind": "defines", "reason": row["meaning"], "evidence": [locator], "standing": "observed"})
|
|
51
|
+
for row in result["relationships"]:
|
|
52
|
+
linked = [facts[key] for key in row["factKeys"] if key in facts]
|
|
53
|
+
relations.append({
|
|
54
|
+
"from": "project:" + row["from"], "to": "project:" + row["to"], "kind": row["kind"],
|
|
55
|
+
"reason": row["data"],
|
|
56
|
+
"evidence": [ref for fact in linked for ref in _evidence_refs(fact)] or [locator],
|
|
57
|
+
"standing": "observed" if linked and all(fact["level"] in CONFIRMED_LEVELS for fact in linked) else "candidate",
|
|
58
|
+
})
|
|
59
|
+
return nodes, relations
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _snapshot(project_id: str, locator: str, execution: FlowExecution, settings: SourceConfig) -> SourceSnapshot:
|
|
63
|
+
result, request = execution["result"], execution["request"]
|
|
64
|
+
nodes, relations = _graph(project_id, result, locator)
|
|
65
|
+
metadata = {
|
|
66
|
+
"projectId": project_id, "executionId": execution["id"], "question": request["question"], "mode": request["mode"],
|
|
67
|
+
"status": execution["status"], "summary": result["summary"],
|
|
68
|
+
"terms": result["terms"], "steps": result["steps"], "relationships": result["relationships"],
|
|
69
|
+
# Excerpts are source text; the graph keeps where each fact was read, not the text.
|
|
70
|
+
"facts": [{**{key: fact[key] for key in ("key", "business", "value", "level")}, "evidence": _evidence_refs(fact)} for fact in result["facts"]],
|
|
71
|
+
"inspectedProjects": result["inspectedProjects"], "gaps": result["gaps"], "questions": result["questions"],
|
|
72
|
+
"sourceStates": recorded_source_states(execution),
|
|
73
|
+
}
|
|
74
|
+
native_id = flow_id(request["question"])
|
|
75
|
+
snapshot = SourceSnapshot(
|
|
76
|
+
ref=SourceRef("business", project_id, native_id),
|
|
77
|
+
primary_node={"kind": "flow", "scope": project_id, "key": native_id, "ref": "primary", "title": result["title"]},
|
|
78
|
+
locator=locator, metadata=metadata, nodes=tuple(nodes), relations=tuple(relations),
|
|
79
|
+
source_updated_at=execution["createdAt"], scope=settings["scope"],
|
|
80
|
+
)
|
|
81
|
+
snapshot_payload(snapshot)
|
|
82
|
+
return snapshot
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _latest_explanations(project_root: Path) -> tuple[dict[str, tuple[Path, FlowExecution]], int]:
|
|
86
|
+
"""The owner's selection, read with this store's containment and size limits."""
|
|
87
|
+
safe_source_path(project_root, business_root(project_root) / "executions")
|
|
88
|
+
|
|
89
|
+
def read(path: Path) -> FlowExecution:
|
|
90
|
+
return cast(FlowExecution, load_owned_object(
|
|
91
|
+
safe_source_path(project_root, path), artifact="business flow execution",
|
|
92
|
+
schema=schema_definition("execution"), max_bytes=MAX_INPUT_BYTES,
|
|
93
|
+
))
|
|
94
|
+
|
|
95
|
+
return latest_explanations(project_root, read=read, unreadable=(DoyouknowError,))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def scan_business_flows(project_root: Path, project_id: str, settings: SourceConfig) -> SourceScan:
|
|
99
|
+
domain = SourceDomain("business", project_id)
|
|
100
|
+
if "business" not in settings["sources"] or not includes_project(settings, project_id):
|
|
101
|
+
return SourceScan(domain, (), (), "partial")
|
|
102
|
+
try:
|
|
103
|
+
if read_source_object(project_root, PROJECT_JSON_RELATIVE).get("projectId") != project_id:
|
|
104
|
+
raise DoyouknowError("registered project identity does not match its source")
|
|
105
|
+
latest, unreadable = _latest_explanations(project_root)
|
|
106
|
+
except _UNREADABLE:
|
|
107
|
+
return SourceScan(domain, (), (), "unavailable", ("project-source-unavailable",))
|
|
108
|
+
snapshots, observed = [], []
|
|
109
|
+
errors = ["business-source-unavailable"] * unreadable
|
|
110
|
+
for native_id, (locator, execution) in sorted(latest.items()):
|
|
111
|
+
observed.append(SourceRef("business", project_id, native_id))
|
|
112
|
+
try:
|
|
113
|
+
snapshots.append(_snapshot(project_id, str(locator), execution, settings))
|
|
114
|
+
except _UNREADABLE:
|
|
115
|
+
errors.append("business-source-unavailable")
|
|
116
|
+
return SourceScan(domain, tuple(snapshots), tuple(observed), "partial" if errors else "complete", tuple(errors))
|