@antoneeo/mkt-agentic-sdlc-skill 0.23.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +3 -3
- package/gemini-extension.json +1 -1
- package/package.json +1 -1
- package/skills/mkt-agentic-sdlc/ENFORCEMENT.md +22 -10
- package/skills/mkt-agentic-sdlc/SKILL.md +3 -2
- package/skills/mkt-agentic-sdlc/docs_site/app.js +7 -1
- package/skills/mkt-agentic-sdlc/docs_site/strings.js +9 -0
- package/skills/mkt-agentic-sdlc/docs_site/style.css +2 -0
- package/skills/mkt-agentic-sdlc/guides.md +15 -1
- package/skills/mkt-agentic-sdlc/memory.md +1 -1
- package/skills/mkt-agentic-sdlc/review.md +9 -1
- package/skills/mkt-agentic-sdlc/routing.md +4 -0
- package/skills/mkt-agentic-sdlc/scripts/docs_site.py +89 -6
- package/skills/mkt-agentic-sdlc/scripts/knowledge.py +1444 -50
- package/skills/mkt-agentic-sdlc/scripts/mkt_check.py +2 -1
- package/skills/mkt-agentic-sdlc/scripts/sdlc_core.py +1115 -75
|
@@ -5,12 +5,16 @@ Extracted from the KB overlay; the kb_* API and stored formats remain compatible
|
|
|
5
5
|
Every distribution ships this module alongside its entry point and sdlc_core.py.
|
|
6
6
|
"""
|
|
7
7
|
import argparse
|
|
8
|
+
import codecs
|
|
9
|
+
import contextlib
|
|
10
|
+
import io
|
|
8
11
|
import os
|
|
9
12
|
import hashlib
|
|
10
13
|
import ntpath
|
|
11
14
|
import posixpath
|
|
12
15
|
import re
|
|
13
16
|
import sys
|
|
17
|
+
import time
|
|
14
18
|
from pathlib import Path
|
|
15
19
|
import sdlc_core
|
|
16
20
|
from sdlc_core import * # compatibility with the existing KB implementation
|
|
@@ -22,7 +26,7 @@ from sdlc_core import _map_refs
|
|
|
22
26
|
|
|
23
27
|
CLAIM_COLUMNS = ("id", "claim", "valid", "qty", "about", "source", "prov", "state")
|
|
24
28
|
CLAIM_HEADING = "## Claims"
|
|
25
|
-
PROVENANCES = ("GIVEN", "ELICITED", "DERIVED", "RULING", "IMPORTED")
|
|
29
|
+
PROVENANCES = ("GIVEN", "ELICITED", "DERIVED", "RULING", "IMPORTED", "DECLARED")
|
|
26
30
|
SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
|
|
27
31
|
OWNS_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}/[a-z0-9][a-z0-9-]{0,63}$")
|
|
28
32
|
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
@@ -112,6 +116,22 @@ def kb_claim_id(source_path, locator, qty_key=""):
|
|
|
112
116
|
# reviewer-comment shape ("states X, that Y, and that Z").
|
|
113
117
|
KB_BUNDLE_HINTS = (", and that ", "; ", ", while ", " and also ")
|
|
114
118
|
|
|
119
|
+
# F-080: `about` holds relations joined by ';', and one reserved predicate whose
|
|
120
|
+
# target is the Standalone ANALYSIS recording a deferred fix of code that does
|
|
121
|
+
# not yet honor the claim. The mark has no reverse direction: nothing lists,
|
|
122
|
+
# counts or selects claims by it -- only the claim checks and export read it.
|
|
123
|
+
KB_NOT_HONORED = "not-yet-honored-by"
|
|
124
|
+
KB_ABOUT_RE = re.compile(r"^([a-z0-9-]+) -> ([a-z0-9-]+)$")
|
|
125
|
+
KB_NOT_HONORED_RE = re.compile(r"^%s -> ((?:ANALYSIS|SHADOW)_[A-Za-z0-9_-]+)$"
|
|
126
|
+
% KB_NOT_HONORED)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def kb_about_parts(cell):
|
|
130
|
+
"""The ';'-joined parts of an `about` cell; [] for '-' or empty."""
|
|
131
|
+
if cell.strip() in ("", "-"):
|
|
132
|
+
return []
|
|
133
|
+
return [part.strip() for part in cell.split(";")]
|
|
134
|
+
|
|
115
135
|
|
|
116
136
|
def kb_unquote(value):
|
|
117
137
|
"""Strip ONE pair of wrapping YAML quotes (then re-strip whitespace).
|
|
@@ -363,6 +383,7 @@ def kb_check_claims(root):
|
|
|
363
383
|
src_cache[path_s] = (t, t is not None and t.is_file())
|
|
364
384
|
return src_cache[path_s]
|
|
365
385
|
per_file_rows = []
|
|
386
|
+
marked = [] # (where, row, [ANALYSIS_ targets]) -- F-080
|
|
366
387
|
for p in sorted(topics.glob("*.md")):
|
|
367
388
|
rel = "topics/" + p.name
|
|
368
389
|
text = sdlc_core.read_text(p)
|
|
@@ -401,6 +422,40 @@ def kb_check_claims(root):
|
|
|
401
422
|
meta, meta_where = (None, "")
|
|
402
423
|
if "#" in first:
|
|
403
424
|
meta, meta_where = _declaring_frontmatter(first.rsplit("#", 1)[0])
|
|
425
|
+
# F-084: a declaration (a document this project SENT) can never pass
|
|
426
|
+
# for evidence, nor evidence for a declaration -- checked on EVERY
|
|
427
|
+
# source position, because corroboration appends sources to a row.
|
|
428
|
+
declared_srcs = []
|
|
429
|
+
for src in sources:
|
|
430
|
+
if "#" not in src:
|
|
431
|
+
continue
|
|
432
|
+
path_s = src.rsplit("#", 1)[0]
|
|
433
|
+
if _resolved(path_s)[1]:
|
|
434
|
+
smeta, swhere = _declaring_frontmatter(path_s)
|
|
435
|
+
if ((smeta or {}).get("provenance") or "").strip().upper() == "DECLARED":
|
|
436
|
+
declared_srcs.append((path_s, smeta, swhere))
|
|
437
|
+
if prov == "DECLARED":
|
|
438
|
+
decl_paths = set(d[0] for d in declared_srcs)
|
|
439
|
+
for src in sources:
|
|
440
|
+
path_s = src.rsplit("#", 1)[0] if "#" in src else src
|
|
441
|
+
if not _resolved(path_s)[1]:
|
|
442
|
+
continue # unresolvable source already reported
|
|
443
|
+
if not path_s.startswith("corpus/given/") or path_s not in decl_paths:
|
|
444
|
+
errors.append("%s: DECLARED row cites %s, which is not a "
|
|
445
|
+
"corpus/given/ artifact whose sidecar says "
|
|
446
|
+
"'provenance: DECLARED' -- a declaration rests "
|
|
447
|
+
"only on the document that was sent" % (where, path_s))
|
|
448
|
+
for path_s, smeta, swhere in declared_srcs:
|
|
449
|
+
if not (smeta.get("recipient") or "").strip():
|
|
450
|
+
errors.append("%s: DECLARED source (%s) carries no "
|
|
451
|
+
"'recipient:' -- a declaration names to whom it "
|
|
452
|
+
"was made" % (where, swhere))
|
|
453
|
+
elif declared_srcs:
|
|
454
|
+
errors.append("%s: prov %s, but %s declares 'provenance: DECLARED' -- "
|
|
455
|
+
"a document this project sent is a declaration, not "
|
|
456
|
+
"evidence: file the row as DECLARED on that artifact "
|
|
457
|
+
"alone (a declaration never corroborates another row)"
|
|
458
|
+
% (where, prov, declared_srcs[0][2]))
|
|
404
459
|
if prov not in PROVENANCES:
|
|
405
460
|
errors.append("%s: prov %r not in %s" % (where, prov, "/".join(PROVENANCES)))
|
|
406
461
|
elif prov in ("DERIVED", "RULING", "ELICITED", "IMPORTED"):
|
|
@@ -410,6 +465,16 @@ def kb_check_claims(root):
|
|
|
410
465
|
errors.append("%s: DERIVED claim's source (%s) carries no "
|
|
411
466
|
"'derived_from:' — model knowledge disguised as "
|
|
412
467
|
"a source" % (where, meta_where))
|
|
468
|
+
elif prov == "RULING" and meta.get("imported_from"):
|
|
469
|
+
# F-083: relabelling an imported row RULING would make another
|
|
470
|
+
# owner's note carry a decision here. Ratifying is deciding:
|
|
471
|
+
# a note of your own, with your own basis.
|
|
472
|
+
errors.append("%s: RULING source (%s) names a foreign origin "
|
|
473
|
+
"(imported_from: %s) — another project's note "
|
|
474
|
+
"cannot carry your ruling. Write your own note "
|
|
475
|
+
"with your own 'basis:' and put the RULING on it "
|
|
476
|
+
"(portability.md §4)"
|
|
477
|
+
% (where, meta_where, meta.get("imported_from")))
|
|
413
478
|
elif prov == "RULING" and not meta.get("basis"):
|
|
414
479
|
errors.append("%s: RULING source (%s) carries no 'basis:' — a "
|
|
415
480
|
"preference is not a fact; no basis, no ruling"
|
|
@@ -429,7 +494,7 @@ def kb_check_claims(root):
|
|
|
429
494
|
# prose, and prose is not a check -- which is how three rows
|
|
430
495
|
# whose evidence was a reading of an image passed as GIVEN.
|
|
431
496
|
declared = ((meta or {}).get("provenance") or "").strip()
|
|
432
|
-
if declared and declared.upper()
|
|
497
|
+
if declared and declared.upper() not in ("GIVEN", "DECLARED"):
|
|
433
498
|
warnings.append(
|
|
434
499
|
"%s: prov GIVEN, but %s declares 'provenance: %s' — the "
|
|
435
500
|
"row reads as first-hand evidence and its artifact does "
|
|
@@ -445,11 +510,22 @@ def kb_check_claims(root):
|
|
|
445
510
|
except ValueError as e:
|
|
446
511
|
errors.append("%s: %s" % (where, e))
|
|
447
512
|
qk = None
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
513
|
+
marks = []
|
|
514
|
+
for part in kb_about_parts(row["about"]):
|
|
515
|
+
mark = KB_NOT_HONORED_RE.match(part)
|
|
516
|
+
rel_m = KB_ABOUT_RE.match(part)
|
|
517
|
+
if mark:
|
|
518
|
+
marks.append(mark.group(1))
|
|
519
|
+
elif part.split(" -> ")[0].strip() == KB_NOT_HONORED:
|
|
520
|
+
errors.append("%s: about %r: '%s' needs an ANALYSIS_<name> "
|
|
521
|
+
"target (Hybrid: SHADOW_<doc_key>, unversioned) -- the "
|
|
522
|
+
"analysis or design that records the deferred "
|
|
523
|
+
"fix" % (where, part, KB_NOT_HONORED))
|
|
524
|
+
elif not rel_m:
|
|
525
|
+
errors.append("%s: about must be '<predicate> -> <slug>' "
|
|
526
|
+
"parts joined by ';', or '-': %r" % (where, part))
|
|
527
|
+
if marks:
|
|
528
|
+
marked.append((where, row, marks))
|
|
453
529
|
# --- atomicity hint (F-047): a row atomic as provenance can be
|
|
454
530
|
# plural as meaning. Text cannot PROVE plurality, so this is a
|
|
455
531
|
# note — never a warning or an error, never the exit code.
|
|
@@ -541,13 +617,18 @@ def kb_check_claims(root):
|
|
|
541
617
|
state = row["state"]
|
|
542
618
|
if state == "OK":
|
|
543
619
|
continue
|
|
544
|
-
|
|
545
|
-
if not
|
|
546
|
-
errors.append("%s: state must be OK, 'CONTESTED <id>[,..]'
|
|
547
|
-
"'SUPERSEDED <id>[,..]': %r"
|
|
620
|
+
kind, targets = _kb_states(state)
|
|
621
|
+
if not kind:
|
|
622
|
+
errors.append("%s: state must be OK, 'CONTESTED <id>[,..]', "
|
|
623
|
+
"'SUPERSEDED <id>[,..]' or 'DIVERGES <id>[,..]': %r"
|
|
624
|
+
% (where, state))
|
|
625
|
+
continue
|
|
626
|
+
row_decl = _kb_is_declared(row)
|
|
627
|
+
if kind == "DIVERGES" and not row_decl:
|
|
628
|
+
errors.append("%s: DIVERGES on a %s row -- only a DECLARED row "
|
|
629
|
+
"diverges; two sources that disagree are CONTESTED"
|
|
630
|
+
% (where, row["prov"].strip() or "?"))
|
|
548
631
|
continue
|
|
549
|
-
kind = m.group(1)
|
|
550
|
-
targets = [t.strip() for t in m.group(2).split(",") if t.strip()]
|
|
551
632
|
for t in targets:
|
|
552
633
|
other = all_rows.get(t)
|
|
553
634
|
if other is None:
|
|
@@ -556,18 +637,38 @@ def kb_check_claims(root):
|
|
|
556
637
|
"the check, it does not clean up" % (where, kind, t))
|
|
557
638
|
continue
|
|
558
639
|
orow, _ = other
|
|
640
|
+
if kind == "DIVERGES":
|
|
641
|
+
if _kb_is_declared(orow):
|
|
642
|
+
errors.append("%s: DIVERGES points at %s, a DECLARED row -- a "
|
|
643
|
+
"declaration diverges from the ledger, never from "
|
|
644
|
+
"another declaration (two sends that disagree "
|
|
645
|
+
"are CONTESTED)" % (where, t))
|
|
646
|
+
elif orow["state"].startswith("SUPERSEDED"):
|
|
647
|
+
errors.append("%s: DIVERGES points at SUPERSEDED row %s -- "
|
|
648
|
+
"re-judge the declaration against its successor"
|
|
649
|
+
" (OK if it now agrees)" % (where, t))
|
|
650
|
+
continue
|
|
651
|
+
if kind in ("SUPERSEDED", "CONTESTED") and row_decl != _kb_is_declared(orow):
|
|
652
|
+
errors.append("%s: %s %s across a declaration -- a declaration is "
|
|
653
|
+
"superseded only by a later declaration and never "
|
|
654
|
+
"settles or contests a fact; a declaration that "
|
|
655
|
+
"contradicts the ledger is DIVERGES"
|
|
656
|
+
% (where, kind, t))
|
|
657
|
+
continue
|
|
559
658
|
if kind == "SUPERSEDED" and orow["prov"] == "IMPORTED":
|
|
560
659
|
# F-030, owner ruling 2026-08-03: knowledge crosses a project
|
|
561
660
|
# boundary, authority does not. An IMPORTED row carries another
|
|
562
661
|
# owner's decision; letting it supersede a local row would make
|
|
563
662
|
# that decision binding here without anyone here granting it.
|
|
564
|
-
#
|
|
565
|
-
#
|
|
663
|
+
# Ratify first (F-083): a note of your own with your own basis,
|
|
664
|
+
# a RULING row on it, and every row contested with the imported
|
|
665
|
+
# one SUPERSEDED by that row.
|
|
566
666
|
errors.append("%s: SUPERSEDED by %s, which is IMPORTED — a "
|
|
567
667
|
"foreign decision cannot settle a local row. "
|
|
568
|
-
"
|
|
569
|
-
"RULING
|
|
570
|
-
%
|
|
668
|
+
"Ratify it: your own note with your own "
|
|
669
|
+
"'basis:', a RULING row on it, and every row "
|
|
670
|
+
"CONTESTED with %s SUPERSEDED by that row"
|
|
671
|
+
% (where, t, t))
|
|
571
672
|
if kind == "CONTESTED":
|
|
572
673
|
if orow["state"].startswith("SUPERSEDED"):
|
|
573
674
|
errors.append("%s: CONTESTED points at SUPERSEDED row %s — "
|
|
@@ -579,9 +680,123 @@ def kb_check_claims(root):
|
|
|
579
680
|
"name %s back; one flipped cell must never "
|
|
580
681
|
"silently end a disagreement"
|
|
581
682
|
% (where, t, row["id"]))
|
|
683
|
+
kb_check_marks(root, marked, per_file_rows, errors)
|
|
582
684
|
return errors, warnings, notes
|
|
583
685
|
|
|
584
686
|
|
|
687
|
+
def kb_check_marks(root, marked, per_file_rows, errors):
|
|
688
|
+
"""F-080: each not-yet-honored mark names an ANALYSIS that exists in this
|
|
689
|
+
project and cites the claim id -- or the id of a row this claim succeeds,
|
|
690
|
+
since supersession copies the mark and edits no analysis. F-081: a Hybrid
|
|
691
|
+
mark names SHADOW_<doc_key>, and the newest shadow of that key is read."""
|
|
692
|
+
if not marked:
|
|
693
|
+
return
|
|
694
|
+
preds = {} # successor id -> ids it supersedes
|
|
695
|
+
for _, rows in per_file_rows:
|
|
696
|
+
for row in rows:
|
|
697
|
+
m = re.match(r"^SUPERSEDED ([0-9a-f, ]+)$", row["state"])
|
|
698
|
+
if row["id"] and m:
|
|
699
|
+
for t in [x.strip() for x in m.group(1).split(",") if x.strip()]:
|
|
700
|
+
preds.setdefault(t, set()).add(row["id"])
|
|
701
|
+
texts = {}
|
|
702
|
+
for where, row, targets in marked:
|
|
703
|
+
ids, todo = set(), [row["id"]] if row["id"] else []
|
|
704
|
+
while todo:
|
|
705
|
+
cid = todo.pop()
|
|
706
|
+
if cid not in ids:
|
|
707
|
+
ids.add(cid)
|
|
708
|
+
todo.extend(preds.get(cid, ()))
|
|
709
|
+
for target in targets:
|
|
710
|
+
if target not in texts:
|
|
711
|
+
texts[target] = _kb_mark_target(root, target)
|
|
712
|
+
rel, text, stray = texts[target]
|
|
713
|
+
if text is None and target.startswith("SHADOW_"):
|
|
714
|
+
errors.append("%s: %s -> %s: no %s in this project%s. Export the "
|
|
715
|
+
"shadow from the accepted devPNT version, or remove "
|
|
716
|
+
"the marker if it did not come from this project -- "
|
|
717
|
+
"it describes its origin's code"
|
|
718
|
+
% (where, KB_NOT_HONORED, target, rel,
|
|
719
|
+
" (%s is not named _vX.Y)" % stray if stray else ""))
|
|
720
|
+
elif text is None:
|
|
721
|
+
errors.append("%s: %s -> %s: no %s in this project. Remove the "
|
|
722
|
+
"marker if it did not come from this project -- it "
|
|
723
|
+
"describes its origin's code" % (where, KB_NOT_HONORED,
|
|
724
|
+
target, rel))
|
|
725
|
+
elif ids and not any(re.search(r"(?<![0-9a-f])%s(?![0-9a-f])" % cid, text)
|
|
726
|
+
for cid in ids):
|
|
727
|
+
errors.append("%s: %s does not cite claim %s -- the analysis or "
|
|
728
|
+
"design that records a deferred fix cites the id it "
|
|
729
|
+
"defers"
|
|
730
|
+
% (where, rel, row["id"]))
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _kb_mark_target(root, target):
|
|
734
|
+
"""(rel, text or None, stray name or None) for a marker target. ANALYSIS_x
|
|
735
|
+
is solutions/x.md; SHADOW_k is the newest solutions/SHADOW_k_vX.Y.md, and
|
|
736
|
+
when none exists a file of the same exact key with another suffix is
|
|
737
|
+
reported as the stray that was not read (F-081)."""
|
|
738
|
+
if not target.startswith("SHADOW_"):
|
|
739
|
+
rel = "solutions/%s.md" % target
|
|
740
|
+
path = sdlc_core.confine_under(root, rel)
|
|
741
|
+
return rel, (sdlc_core.read_text(path)
|
|
742
|
+
if path is not None and path.is_file() else None), None
|
|
743
|
+
key = target[len("SHADOW_"):]
|
|
744
|
+
sol = sdlc_core.confine_under(root, "solutions")
|
|
745
|
+
path = sdlc_core.newest_shadow(sol, key) if sol is not None else None
|
|
746
|
+
if path is not None and sdlc_core.confine_under(root, "solutions/" + path.name):
|
|
747
|
+
return "solutions/" + path.name, sdlc_core.read_text(path), None
|
|
748
|
+
stray = sdlc_core.stray_shadow(sol, key) if sol is not None else None
|
|
749
|
+
return "solutions/SHADOW_%s_v<X.Y>.md" % key, None, stray
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def kb_drop_marks(text):
|
|
753
|
+
"""(text, dropped): the topic text with every not-yet-honored part removed
|
|
754
|
+
from claim rows' `about` cells. Every other byte is kept -- other cells keep
|
|
755
|
+
their spacing, other lines are untouched (F-080: a bundle describes no code)."""
|
|
756
|
+
rows, _ = kb_parse_claims(text)
|
|
757
|
+
lines = text.split("\n")
|
|
758
|
+
col = 1 + CLAIM_COLUMNS.index("about")
|
|
759
|
+
dropped = False
|
|
760
|
+
for row in rows:
|
|
761
|
+
parts = kb_about_parts(row["about"])
|
|
762
|
+
keep = [p for p in parts if not KB_NOT_HONORED_RE.match(p)]
|
|
763
|
+
if len(keep) == len(parts):
|
|
764
|
+
continue
|
|
765
|
+
# The parser strips the pipes at both ends, so a row with no trailing
|
|
766
|
+
# pipe is legal: counted from the left, the about cell sits at `col`
|
|
767
|
+
# either way.
|
|
768
|
+
segs = lines[row["_line"] - 1].split("|")
|
|
769
|
+
if len(segs) not in (len(CLAIM_COLUMNS) + 1, len(CLAIM_COLUMNS) + 2):
|
|
770
|
+
continue
|
|
771
|
+
segs[col] = " %s " % ("; ".join(keep) or "-")
|
|
772
|
+
lines[row["_line"] - 1] = "|".join(segs)
|
|
773
|
+
dropped = True
|
|
774
|
+
return "\n".join(lines), dropped
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _kb_is_declared(row):
|
|
778
|
+
return (row.get("prov") or "").strip().upper() == "DECLARED"
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def kb_has_declarations(selected):
|
|
782
|
+
"""Whether any selected topic holds a DECLARED row -- a yes/no for export's
|
|
783
|
+
line, never a count (the same rule as F-080's dropped-marker line)."""
|
|
784
|
+
return any(_kb_is_declared(r) for _p, _t, rows in selected.values() for r in rows)
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def kb_drop_declarations(text):
|
|
788
|
+
"""The topic text without its DECLARED rows (F-084: a declaration is this
|
|
789
|
+
project's commitment to its recipient and stays here). Every other line is
|
|
790
|
+
kept byte for byte. Closed by construction: check refuses any pointer from
|
|
791
|
+
a non-DECLARED row at a DECLARED one, so nothing left behind points at a
|
|
792
|
+
dropped row."""
|
|
793
|
+
rows, _ = kb_parse_claims(text)
|
|
794
|
+
gone = set(r["_line"] for r in rows if _kb_is_declared(r))
|
|
795
|
+
if not gone:
|
|
796
|
+
return text
|
|
797
|
+
return "\n".join(ln for i, ln in enumerate(text.split("\n"), 1) if i not in gone)
|
|
798
|
+
|
|
799
|
+
|
|
585
800
|
def kb_check_locator(target, loc, where, errors, cache=None):
|
|
586
801
|
"""A locator must address an existing span of the stored bytes.
|
|
587
802
|
|
|
@@ -912,6 +1127,10 @@ def kb_build_corpus_index(root):
|
|
|
912
1127
|
for meta_p in sorted(given.glob("*.meta.md")):
|
|
913
1128
|
meta = sdlc_core.load_frontmatter(sdlc_core.read_text(meta_p).splitlines()) or {}
|
|
914
1129
|
orig = meta_p.name[:-len(".meta.md")]
|
|
1130
|
+
if kb_is_tombstone(meta):
|
|
1131
|
+
lines.append("- `%s` — %s — withdrawn" % (
|
|
1132
|
+
orig, (meta.get("date") or "undated")))
|
|
1133
|
+
continue
|
|
915
1134
|
sup = kb_unquote(meta.get("supersedes"))
|
|
916
1135
|
lines.append("- `%s` — %s%s — %s" % (
|
|
917
1136
|
orig, (meta.get("date") or "undated"),
|
|
@@ -1056,6 +1275,21 @@ def kb_corpus_check(root):
|
|
|
1056
1275
|
meta = sdlc_core.load_frontmatter(sdlc_core.read_text(meta_p).splitlines()) or {}
|
|
1057
1276
|
orig = given / meta_p.name[:-len(".meta.md")]
|
|
1058
1277
|
rel = "corpus/given/" + orig.name
|
|
1278
|
+
if kb_is_tombstone(meta):
|
|
1279
|
+
# F-077: a withdrawn artifact keeps its record, never its bytes
|
|
1280
|
+
# nor a digest of them -- a digest of the secret-bearing bytes
|
|
1281
|
+
# beside a redacted successor tests password guesses offline.
|
|
1282
|
+
kept = [k for k in ("sha256", "original_sha256")
|
|
1283
|
+
if (meta.get(k) or "").strip()]
|
|
1284
|
+
if kept:
|
|
1285
|
+
errors.append("%s: withdrawn, yet its sidecar keeps %s -- "
|
|
1286
|
+
"a tombstone keeps no digest of the withdrawn "
|
|
1287
|
+
"bytes (distillation.md par.1)"
|
|
1288
|
+
% (rel, ", ".join(k + ":" for k in kept)))
|
|
1289
|
+
if orig.is_file():
|
|
1290
|
+
errors.append("%s: withdrawn, yet its bytes are still here -- "
|
|
1291
|
+
"delete them" % rel)
|
|
1292
|
+
continue
|
|
1059
1293
|
if not orig.is_file():
|
|
1060
1294
|
errors.append("%s: sidecar exists but the original is gone" % rel)
|
|
1061
1295
|
continue
|
|
@@ -1082,17 +1316,36 @@ def kb_corpus_check(root):
|
|
|
1082
1316
|
"extraction is intact, the pointer to the original is "
|
|
1083
1317
|
"not. A warning and not an error: a bundle carries "
|
|
1084
1318
|
"artifacts and sidecars, never the originals, so after "
|
|
1085
|
-
"an import this dangles legitimately"
|
|
1086
|
-
% (rel, op,
|
|
1319
|
+
"an import this dangles legitimately%s"
|
|
1320
|
+
% (rel, _kb_mask_userinfo(op),
|
|
1321
|
+
_kb_mask_userinfo(", ".join(tried)),
|
|
1322
|
+
" -- a non-local identifier belongs in original_ref:, "
|
|
1323
|
+
"which is recorded and never resolved"
|
|
1324
|
+
if re.match(r"^[A-Za-z][A-Za-z0-9+.\-]+://", op) else ""))
|
|
1087
1325
|
sup = kb_unquote(meta.get("supersedes"))
|
|
1088
1326
|
if sup:
|
|
1089
1327
|
superseded.add(sup)
|
|
1090
|
-
|
|
1328
|
+
sup_side = given / (sup + ".meta.md")
|
|
1329
|
+
tomb = sup_side.is_file() and kb_is_tombstone(
|
|
1330
|
+
sdlc_core.load_frontmatter(sdlc_core.read_text(sup_side).splitlines()))
|
|
1331
|
+
if not (given / sup).is_file() and not tomb:
|
|
1091
1332
|
warnings.append("%s: supersedes %r which is not in given/"
|
|
1092
1333
|
% (rel, sup))
|
|
1093
1334
|
kb_check_coverage(rel, orig,
|
|
1094
1335
|
(meta.get("extracted_through") or "").strip(),
|
|
1095
1336
|
_kb_cited_for(cited, orig.name), errors, warnings)
|
|
1337
|
+
# F-077: a file nobody declared -- or bytes a withdrawal left behind. A
|
|
1338
|
+
# stored extraction `<stem>.txt` beside a declared `<stem>.<ext>` is the
|
|
1339
|
+
# declared artifact's own (distillation.md par.1), not a stray.
|
|
1340
|
+
declared_stems = set(Path(m.name[:-len(".meta.md")]).stem
|
|
1341
|
+
for m in given.glob("*.meta.md"))
|
|
1342
|
+
for p in sorted(given.iterdir()):
|
|
1343
|
+
if (p.is_file() and not p.name.endswith(".meta.md")
|
|
1344
|
+
and not (given / (p.name + ".meta.md")).is_file()
|
|
1345
|
+
and not (p.suffix == ".txt" and p.stem in declared_stems)):
|
|
1346
|
+
warnings.append("corpus/given/%s: no sidecar -- an artifact nobody "
|
|
1347
|
+
"declared, or bytes a withdrawal left behind" % p.name)
|
|
1348
|
+
warnings.extend(_kb_same_text_warnings(root))
|
|
1096
1349
|
notes = corpus / "notes"
|
|
1097
1350
|
if notes.is_dir():
|
|
1098
1351
|
for p in sorted(notes.glob("*.md")):
|
|
@@ -1110,6 +1363,226 @@ def kb_corpus_check(root):
|
|
|
1110
1363
|
return errors, warnings
|
|
1111
1364
|
|
|
1112
1365
|
|
|
1366
|
+
# ------------------------------------------------- credentials (F-077)
|
|
1367
|
+
# A credential stored in given/ cannot be fixed in place: the digest check
|
|
1368
|
+
# refuses the edit and git keeps the bytes. These shapes are written from the
|
|
1369
|
+
# formats' own syntax (key=value lines, PEM headers, URL userinfo, bearer
|
|
1370
|
+
# headers); no vendor key format is matched (distillation.md par.1 lists the
|
|
1371
|
+
# declared false positives and false negatives). Findings carry line numbers
|
|
1372
|
+
# and kinds, NEVER the value: the validator's output lands in logs and CI.
|
|
1373
|
+
|
|
1374
|
+
KB_CRED_PAIRS = {("api", "key"): "api-key", ("access", "key"): "access-key",
|
|
1375
|
+
("private", "key"): "private-key",
|
|
1376
|
+
("client", "secret"): "client-secret", ("secret", "key"): "secret"}
|
|
1377
|
+
KB_CRED_LAST = {"password": "password", "passwd": "password",
|
|
1378
|
+
"passphrase": "password", "secret": "secret", "token": "token",
|
|
1379
|
+
"credential": "credentials", "credentials": "credentials",
|
|
1380
|
+
"apikey": "api-key", "accesskey": "access-key"}
|
|
1381
|
+
KB_CRED_ASSIGN_RE = re.compile(
|
|
1382
|
+
r"""^\s*(?:export\s+)?(["']?)([A-Za-z0-9_.\-]+)\1\s*[=:]\s*(.*)$""")
|
|
1383
|
+
KB_CRED_SEGMENT_RE = re.compile(r"[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])")
|
|
1384
|
+
KB_CRED_PEM_RE = re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----")
|
|
1385
|
+
KB_CRED_URL_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.\-]*://[^/\s:@]*:([^@\s/]+)@")
|
|
1386
|
+
KB_CRED_QUERY_RE = re.compile(r"[?&;](?:password|pwd|passwd)=([^&\s;#]+)", re.I)
|
|
1387
|
+
KB_CRED_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9\-._~+/]{16,}")
|
|
1388
|
+
KB_CRED_PLACEHOLDER_RE = re.compile(
|
|
1389
|
+
r"<.*>|\$\{.*\}|\{\{.*\}\}|%\(.*\)s|ENC\(.*\)|[*xX]+|[|>][-+]?\d*")
|
|
1390
|
+
KB_CRED_LINE_MAX = 4096
|
|
1391
|
+
KB_REVIEWED_RE = re.compile(r"^reviewed\s*[-—–:]\s*\S")
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def _kb_cred_real(value):
|
|
1395
|
+
"""True when a value looks like an actual secret, not a placeholder."""
|
|
1396
|
+
v = re.split(r"\s+(?:#|//)", value.strip(), maxsplit=1)[0].strip()
|
|
1397
|
+
if v.endswith(","):
|
|
1398
|
+
v = v[:-1].rstrip()
|
|
1399
|
+
if len(v) >= 2 and v[0] in "\"'" and v[-1] == v[0]:
|
|
1400
|
+
v = v[1:-1]
|
|
1401
|
+
if not v or re.search(r"\s", v):
|
|
1402
|
+
return False
|
|
1403
|
+
if v.lower() in ("true", "false", "null", "none", "yes", "no"):
|
|
1404
|
+
return False
|
|
1405
|
+
if v.startswith("[REDACTED") or KB_CRED_PLACEHOLDER_RE.fullmatch(v):
|
|
1406
|
+
return False
|
|
1407
|
+
return True
|
|
1408
|
+
|
|
1409
|
+
|
|
1410
|
+
def _kb_cred_key_kind(key):
|
|
1411
|
+
segs = [s.lower() for part in re.split(r"[_.\-]", key)
|
|
1412
|
+
for s in KB_CRED_SEGMENT_RE.findall(part)]
|
|
1413
|
+
if len(segs) >= 2 and tuple(segs[-2:]) in KB_CRED_PAIRS:
|
|
1414
|
+
return KB_CRED_PAIRS[tuple(segs[-2:])]
|
|
1415
|
+
return KB_CRED_LAST.get(segs[-1]) if segs else None
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def kb_credential_line_kind(line):
|
|
1419
|
+
"""The credential kind one line carries, or None. Never returns the value."""
|
|
1420
|
+
line = line[:KB_CRED_LINE_MAX]
|
|
1421
|
+
if KB_CRED_PEM_RE.search(line):
|
|
1422
|
+
return "private-key"
|
|
1423
|
+
m = KB_CRED_ASSIGN_RE.match(line)
|
|
1424
|
+
if m:
|
|
1425
|
+
kind = _kb_cred_key_kind(m.group(2))
|
|
1426
|
+
if kind and _kb_cred_real(m.group(3)):
|
|
1427
|
+
return kind
|
|
1428
|
+
m = KB_CRED_URL_RE.search(line)
|
|
1429
|
+
if m and _kb_cred_real(m.group(1)):
|
|
1430
|
+
return "connection-string"
|
|
1431
|
+
m = KB_CRED_QUERY_RE.search(line)
|
|
1432
|
+
if m and _kb_cred_real(m.group(1)):
|
|
1433
|
+
return "connection-string"
|
|
1434
|
+
if KB_CRED_BEARER_RE.search(line):
|
|
1435
|
+
return "bearer-token"
|
|
1436
|
+
return None
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
def kb_credential_findings(path):
|
|
1440
|
+
"""[(line_no, kind)] for one file; [] for a binary or unreadable one.
|
|
1441
|
+
|
|
1442
|
+
A NUL in the first 8 KB means binary, unless the file opens with a UTF-16
|
|
1443
|
+
byte-order mark (then it is decoded whole). Text is streamed line by line,
|
|
1444
|
+
each inspected on its first KB_CRED_LINE_MAX characters."""
|
|
1445
|
+
out = []
|
|
1446
|
+
try:
|
|
1447
|
+
with open(path, "rb") as f:
|
|
1448
|
+
head = f.read(8192)
|
|
1449
|
+
if head[:2] in (b"\xff\xfe", b"\xfe\xff"):
|
|
1450
|
+
lines = Path(path).read_bytes().decode("utf-16", errors="replace").splitlines()
|
|
1451
|
+
for n, line in enumerate(lines, 1):
|
|
1452
|
+
kind = kb_credential_line_kind(line)
|
|
1453
|
+
if kind:
|
|
1454
|
+
out.append((n, kind))
|
|
1455
|
+
return out
|
|
1456
|
+
if b"\x00" in head:
|
|
1457
|
+
return []
|
|
1458
|
+
with open(path, encoding="utf-8-sig", errors="replace") as f:
|
|
1459
|
+
for n, line in enumerate(f, 1):
|
|
1460
|
+
kind = kb_credential_line_kind(line.rstrip("\r\n"))
|
|
1461
|
+
if kind:
|
|
1462
|
+
out.append((n, kind))
|
|
1463
|
+
except OSError:
|
|
1464
|
+
return []
|
|
1465
|
+
return out
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
def _kb_mask_userinfo(text):
|
|
1469
|
+
"""A password inside a URL never reaches the validator's output -- also once
|
|
1470
|
+
a candidate path has turned its `//` into `\\` (Windows join)."""
|
|
1471
|
+
return re.sub(r"([/\\][^/\\\s:@]*:)\S*@", r"\1***@", text)
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
def _kb_reviewed(meta):
|
|
1475
|
+
return bool(KB_REVIEWED_RE.match(kb_unquote((meta or {}).get("credential_scan"))))
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
def _kb_cred_message(rel, found, fix):
|
|
1479
|
+
nums = [str(n) for n, _ in found[:10]] + (["..."] if len(found) > 10 else [])
|
|
1480
|
+
kinds = sorted(set(k for _, k in found))
|
|
1481
|
+
return "%s: %s %s look like a credential (%s) -- %s" % (
|
|
1482
|
+
rel, "line" if len(found) == 1 else "lines", ", ".join(nums),
|
|
1483
|
+
", ".join(kinds), fix)
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def kb_is_tombstone(meta):
|
|
1487
|
+
return bool(kb_unquote((meta or {}).get("withdrawn")))
|
|
1488
|
+
|
|
1489
|
+
|
|
1490
|
+
def kb_credential_errors(root, docs):
|
|
1491
|
+
"""FS2 errors over given/ artifacts and document-guide snapshots, escapes
|
|
1492
|
+
applied. One carrier per command calls this (check, corpus); validate never."""
|
|
1493
|
+
errors = []
|
|
1494
|
+
given = docs / "corpus" / "given"
|
|
1495
|
+
if given.is_dir():
|
|
1496
|
+
# A stored extraction `<stem>.txt` has no sidecar of its own: it is the
|
|
1497
|
+
# declared `<stem>.<ext>`'s (distillation.md par.1), so its escape lives there.
|
|
1498
|
+
by_stem = {}
|
|
1499
|
+
for m in sorted(given.glob("*.meta.md")):
|
|
1500
|
+
by_stem.setdefault(Path(m.name[:-len(".meta.md")]).stem, m)
|
|
1501
|
+
for p in sorted(given.iterdir()):
|
|
1502
|
+
if not p.is_file() or p.name.endswith(".meta.md"):
|
|
1503
|
+
continue
|
|
1504
|
+
side = given / (p.name + ".meta.md")
|
|
1505
|
+
if not side.is_file() and p.suffix == ".txt" and p.stem in by_stem:
|
|
1506
|
+
side = by_stem[p.stem]
|
|
1507
|
+
meta = (sdlc_core.load_frontmatter(sdlc_core.read_text(side).splitlines())
|
|
1508
|
+
or {}) if side.is_file() else {}
|
|
1509
|
+
if _kb_reviewed(meta):
|
|
1510
|
+
continue
|
|
1511
|
+
found = kb_credential_findings(p)
|
|
1512
|
+
if found:
|
|
1513
|
+
errors.append(_kb_cred_message(
|
|
1514
|
+
"corpus/given/" + p.name, found,
|
|
1515
|
+
"given/ is never edited: redact into a successor and withdraw "
|
|
1516
|
+
"this one (distillation.md par.1), or declare "
|
|
1517
|
+
"`credential_scan: reviewed -- <reason>` in its sidecar"))
|
|
1518
|
+
for rel, _p, meta, _t in sdlc_core.list_guides(root):
|
|
1519
|
+
if (meta.get("source_kind") or "document").strip() != "document":
|
|
1520
|
+
continue
|
|
1521
|
+
df = (meta.get("distilled_from") or "").strip()
|
|
1522
|
+
src = sdlc_core.confine_under(root, df) if df else None
|
|
1523
|
+
if src is None or not src.is_file() or _kb_reviewed(meta):
|
|
1524
|
+
continue
|
|
1525
|
+
try:
|
|
1526
|
+
src.resolve().relative_to(given.resolve())
|
|
1527
|
+
continue # scanned above, once
|
|
1528
|
+
except (OSError, ValueError):
|
|
1529
|
+
pass
|
|
1530
|
+
found = kb_credential_findings(src)
|
|
1531
|
+
if found:
|
|
1532
|
+
errors.append(_kb_cred_message(
|
|
1533
|
+
df, found,
|
|
1534
|
+
"redact into a new snapshot (its name changes) and update %s's "
|
|
1535
|
+
"distilled_from, source_hash and markers, or declare "
|
|
1536
|
+
"`credential_scan: reviewed -- <reason>` in the guide" % rel))
|
|
1537
|
+
return errors
|
|
1538
|
+
|
|
1539
|
+
|
|
1540
|
+
def _kb_print_credentials(root, docs):
|
|
1541
|
+
errs = kb_credential_errors(root, docs)
|
|
1542
|
+
for e in errs:
|
|
1543
|
+
print("[ERROR] %s" % e)
|
|
1544
|
+
return 1 if errs else 0
|
|
1545
|
+
|
|
1546
|
+
|
|
1547
|
+
def _kb_same_text_warnings(docs):
|
|
1548
|
+
"""FS6: a .sources/ file and a given/ artifact holding the same text after
|
|
1549
|
+
line-ending normalization. Snapshots are hashed once; a given/ artifact is
|
|
1550
|
+
hashed only when its size is within a factor of two of some snapshot's (CRLF
|
|
1551
|
+
at most doubles a text), so a large extraction corpus is not read for this."""
|
|
1552
|
+
src = docs / "reference" / ".sources"
|
|
1553
|
+
given = docs / "corpus" / "given"
|
|
1554
|
+
if not (src.is_dir() and given.is_dir()):
|
|
1555
|
+
return []
|
|
1556
|
+
snaps = {}
|
|
1557
|
+
sizes = []
|
|
1558
|
+
for p in sorted(src.rglob("*")):
|
|
1559
|
+
if p.is_file():
|
|
1560
|
+
try:
|
|
1561
|
+
snaps.setdefault(sdlc_core.sha256_file(p), []).append(p)
|
|
1562
|
+
sizes.append(p.stat().st_size)
|
|
1563
|
+
except OSError:
|
|
1564
|
+
continue
|
|
1565
|
+
if not snaps:
|
|
1566
|
+
return []
|
|
1567
|
+
lo, hi = min(sizes) // 2, max(sizes) * 2
|
|
1568
|
+
out = []
|
|
1569
|
+
for g in sorted(given.iterdir()):
|
|
1570
|
+
if not g.is_file() or g.name.endswith(".meta.md"):
|
|
1571
|
+
continue
|
|
1572
|
+
try:
|
|
1573
|
+
if not lo <= g.stat().st_size <= hi:
|
|
1574
|
+
continue
|
|
1575
|
+
h = sdlc_core.sha256_file(g)
|
|
1576
|
+
except OSError:
|
|
1577
|
+
continue
|
|
1578
|
+
for s in snaps.get(h, []):
|
|
1579
|
+
out.append("reference/.sources/%s and corpus/given/%s hold the same text "
|
|
1580
|
+
"-- one home per source text: re-point the guide to "
|
|
1581
|
+
"corpus/given/ (portability.md par.6, kb-agentic)"
|
|
1582
|
+
% (s.relative_to(src).as_posix(), g.name))
|
|
1583
|
+
return out
|
|
1584
|
+
|
|
1585
|
+
|
|
1113
1586
|
def kb_sha256_bytes(path):
|
|
1114
1587
|
"""RAW-byte digest for binaries. The spine's sha256_file normalizes CRLF->LF
|
|
1115
1588
|
(right for text snapshots, wrong for binaries, where a hostile 0D0A/0A pair
|
|
@@ -1394,21 +1867,29 @@ def kb_export_closure(docs, slugs):
|
|
|
1394
1867
|
selected = dict((s, all_topics[s]) for s in slugs)
|
|
1395
1868
|
# id -> slug, over the WHOLE graph, so a conflict partner is findable
|
|
1396
1869
|
owner_of = {}
|
|
1870
|
+
declared_ids = set()
|
|
1397
1871
|
for slug, (_p, _t, rows) in all_topics.items():
|
|
1398
1872
|
for r in rows:
|
|
1399
1873
|
if r["id"]:
|
|
1400
1874
|
owner_of[r["id"]] = slug
|
|
1875
|
+
if _kb_is_declared(r):
|
|
1876
|
+
declared_ids.add(r["id"])
|
|
1401
1877
|
added = []
|
|
1402
1878
|
pending = list(selected)
|
|
1403
1879
|
while pending:
|
|
1404
1880
|
slug = pending.pop()
|
|
1405
1881
|
for r in selected[slug][2]:
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
for ref in
|
|
1882
|
+
if _kb_is_declared(r):
|
|
1883
|
+
continue # F-084: a declaration stays in this project
|
|
1884
|
+
_kind, refs = _kb_states(r["state"])
|
|
1885
|
+
for ref in refs:
|
|
1410
1886
|
other = owner_of.get(ref)
|
|
1411
|
-
if
|
|
1887
|
+
if ref in declared_ids:
|
|
1888
|
+
errors.append("claim %s in topic '%s' points at %s, a DECLARED row: "
|
|
1889
|
+
"declarations are not exported and a fact never "
|
|
1890
|
+
"points at one -- run check" % (r["id"] or "(no id)",
|
|
1891
|
+
slug, ref))
|
|
1892
|
+
elif other is None:
|
|
1412
1893
|
errors.append("claim %s in topic '%s' points at id %s, which "
|
|
1413
1894
|
"no topic owns: export would carry a broken set"
|
|
1414
1895
|
% (r["id"] or "(no id)", slug, ref))
|
|
@@ -1420,11 +1901,17 @@ def kb_export_closure(docs, slugs):
|
|
|
1420
1901
|
seen = set()
|
|
1421
1902
|
for slug, (_p, _t, rows) in sorted(selected.items()):
|
|
1422
1903
|
for r in rows:
|
|
1904
|
+
if _kb_is_declared(r):
|
|
1905
|
+
continue # its artifact stays with the declaration
|
|
1423
1906
|
for rel in kb_claim_sources(r):
|
|
1424
1907
|
for cand in (rel, rel + ".meta.md",
|
|
1425
1908
|
rel[:-len(Path(rel).suffix)] + ".txt" if Path(rel).suffix else rel):
|
|
1426
1909
|
if cand in seen:
|
|
1427
1910
|
continue
|
|
1911
|
+
if cand.endswith(".meta.md") and (docs / cand).is_file() and \
|
|
1912
|
+
kb_is_tombstone(sdlc_core.load_frontmatter(
|
|
1913
|
+
sdlc_core.read_text(docs / cand).splitlines())):
|
|
1914
|
+
continue # F-077: a tombstone never travels
|
|
1428
1915
|
if (docs / cand).is_file():
|
|
1429
1916
|
seen.add(cand)
|
|
1430
1917
|
artifacts.append(cand)
|
|
@@ -1432,10 +1919,16 @@ def kb_export_closure(docs, slugs):
|
|
|
1432
1919
|
|
|
1433
1920
|
|
|
1434
1921
|
def kb_bundle_write(docs, out, selected, artifacts, project):
|
|
1435
|
-
"""Write the bundle. Mirrors the docs-root layout; no path is rewritten.
|
|
1922
|
+
"""Write the bundle. Mirrors the docs-root layout; no path is rewritten.
|
|
1923
|
+
Returns whether a not-yet-honored mark was dropped (F-080) -- a yes/no,
|
|
1924
|
+
never a count: a count of dropped marks is a count of claims by the mark."""
|
|
1436
1925
|
out.mkdir(parents=True, exist_ok=True)
|
|
1437
1926
|
(out / "topics").mkdir(exist_ok=True)
|
|
1927
|
+
dropped = False
|
|
1438
1928
|
for slug, (p, text, _rows) in sorted(selected.items()):
|
|
1929
|
+
text, d = kb_drop_marks(text)
|
|
1930
|
+
dropped = dropped or d
|
|
1931
|
+
text = kb_drop_declarations(text)
|
|
1439
1932
|
(out / "topics" / p.name).write_text(text, encoding="utf-8")
|
|
1440
1933
|
for rel in artifacts:
|
|
1441
1934
|
dst = out / rel
|
|
@@ -1450,12 +1943,341 @@ def kb_bundle_write(docs, out, selected, artifacts, project):
|
|
|
1450
1943
|
for rel in artifacts:
|
|
1451
1944
|
lines.append("- `%s` sha256:%s" % (rel, kb_sha256_bytes(docs / rel)))
|
|
1452
1945
|
(out / BUNDLE_MANIFEST).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
1946
|
+
return dropped
|
|
1947
|
+
|
|
1948
|
+
|
|
1949
|
+
def _kb_replace_cells(line, updates):
|
|
1950
|
+
"""One claim-table row with the cells at the given column indexes replaced;
|
|
1951
|
+
everything else -- indentation, outer pipes, a CRLF tail -- verbatim."""
|
|
1952
|
+
end = "\r" if line.endswith("\r") else ""
|
|
1953
|
+
body = line[:-1] if end else line
|
|
1954
|
+
lead = body[:len(body) - len(body.lstrip())]
|
|
1955
|
+
s = body.strip()
|
|
1956
|
+
opens, closes = s.startswith("|"), len(s) > 1 and s.endswith("|")
|
|
1957
|
+
cells = s[1 if opens else 0:len(s) - 1 if closes else len(s)].split("|")
|
|
1958
|
+
for i, v in updates.items():
|
|
1959
|
+
cells[i] = " %s " % v
|
|
1960
|
+
return "%s%s%s%s%s" % (lead, "|" if opens else "", "|".join(cells),
|
|
1961
|
+
"|" if closes else "", end)
|
|
1962
|
+
|
|
1963
|
+
|
|
1964
|
+
def _kb_states(state):
|
|
1965
|
+
"""'CONTESTED a, b' / 'SUPERSEDED a' / 'DIVERGES a' -> (kind, [ids]);
|
|
1966
|
+
OK/other -> (None, []). The one parser of the full state grammar."""
|
|
1967
|
+
m = re.match(r"^(CONTESTED|SUPERSEDED|DIVERGES) ([0-9a-f, ]+)$", (state or "").strip())
|
|
1968
|
+
if not m:
|
|
1969
|
+
return None, []
|
|
1970
|
+
return m.group(1), [t.strip() for t in m.group(2).split(",") if t.strip()]
|
|
1971
|
+
|
|
1972
|
+
|
|
1973
|
+
def _kb_bom_text(data):
|
|
1974
|
+
"""(bom, text): a UTF-8 BOM (Windows authoring) is kept apart and given back
|
|
1975
|
+
on write, as sdlc_core.read_text reads past it."""
|
|
1976
|
+
bom = codecs.BOM_UTF8 if data.startswith(codecs.BOM_UTF8) else b""
|
|
1977
|
+
return bom, data[len(bom):].decode("utf-8")
|
|
1978
|
+
|
|
1979
|
+
|
|
1980
|
+
def _kb_foreign_note(data, project, rel):
|
|
1981
|
+
"""F-083: the bytes of a ruling note as it lands in another project, and the
|
|
1982
|
+
number of lines inserted. Lines go right after the opening '---' with the
|
|
1983
|
+
note's own line ending; an existing origin line is rewritten in place, so it
|
|
1984
|
+
moves nothing. None when the note opens no frontmatter."""
|
|
1985
|
+
bom, text = _kb_bom_text(data)
|
|
1986
|
+
nl = "\r\n" if "\r\n" in text else "\n"
|
|
1987
|
+
lines = text.split(nl)
|
|
1988
|
+
if not lines or lines[0].strip() != "---":
|
|
1989
|
+
return None
|
|
1990
|
+
end = next((i for i in range(1, min(len(lines), 60)) if lines[i].strip() == "---"),
|
|
1991
|
+
None)
|
|
1992
|
+
if end is None:
|
|
1993
|
+
return None
|
|
1994
|
+
inserted = ["imported_from: %s — %s" % (project, rel)]
|
|
1995
|
+
origin = [i for i in range(1, end) if re.match(r"^origin:", lines[i], re.I)]
|
|
1996
|
+
for i in origin:
|
|
1997
|
+
lines[i] = "origin: imported"
|
|
1998
|
+
if not origin:
|
|
1999
|
+
inserted.append("origin: imported")
|
|
2000
|
+
lines[1:1] = inserted
|
|
2001
|
+
return bom + nl.join(lines).encode("utf-8"), len(inserted)
|
|
2002
|
+
|
|
2003
|
+
|
|
2004
|
+
def _kb_shift_loc(loc, k):
|
|
2005
|
+
a, b = (int(x) for x in LOC_LINE_RE.match(loc).groups())
|
|
2006
|
+
if a >= 2:
|
|
2007
|
+
return "L%d-%d" % (a + k, b + k)
|
|
2008
|
+
return "L1-%d" % (b + k if b >= 2 else 1)
|
|
2009
|
+
|
|
2010
|
+
|
|
2011
|
+
def _kb_span(data, loc):
|
|
2012
|
+
"""The lines a line locator addresses, minus the two lines an import may
|
|
2013
|
+
insert or rewrite -- what a moved locator must still cite."""
|
|
2014
|
+
a, b = (int(x) for x in LOC_LINE_RE.match(loc).groups())
|
|
2015
|
+
return [x for x in _kb_bom_text(data)[1].splitlines()[a - 1:b]
|
|
2016
|
+
if not re.match(r"^(imported_from|origin):", x, re.I)]
|
|
2017
|
+
|
|
2018
|
+
|
|
2019
|
+
def _kb_rewrite_derived_from(data, renames):
|
|
2020
|
+
"""A note whose derived_from names a renamed ruling note names the copy. The
|
|
2021
|
+
edit stays on the derived_from line, so no line moves."""
|
|
2022
|
+
bom, text = _kb_bom_text(data)
|
|
2023
|
+
nl = "\r\n" if "\r\n" in text else "\n"
|
|
2024
|
+
lines = text.split(nl)
|
|
2025
|
+
if not lines or lines[0].strip() != "---":
|
|
2026
|
+
return data
|
|
2027
|
+
for i in range(1, min(len(lines), 60)):
|
|
2028
|
+
if lines[i].strip() == "---":
|
|
2029
|
+
break
|
|
2030
|
+
m = re.match(r"^(derived_from:\s*)(.*)$", lines[i], re.I)
|
|
2031
|
+
if not m:
|
|
2032
|
+
continue
|
|
2033
|
+
val = m.group(2).strip()
|
|
2034
|
+
listed = val.startswith("[") and val.endswith("]")
|
|
2035
|
+
toks = val[1:-1].split(",") if listed else [val]
|
|
2036
|
+
out = []
|
|
2037
|
+
for tok in toks:
|
|
2038
|
+
e = kb_unquote(tok)
|
|
2039
|
+
hit = next((new for old, new in renames.items()
|
|
2040
|
+
if e in (old, old[len("corpus/notes/"):])), None)
|
|
2041
|
+
out.append((" " if listed and out else "") + hit if hit else tok)
|
|
2042
|
+
new = ("[%s]" % ",".join(out)) if listed else out[0]
|
|
2043
|
+
lines[i] = m.group(1) + new
|
|
2044
|
+
return bom + nl.join(lines).encode("utf-8")
|
|
2045
|
+
|
|
2046
|
+
|
|
2047
|
+
def kb_import_foreign_rulings(bundle, docs, topic_rels, written, project):
|
|
2048
|
+
"""F-083 -- knowledge crosses a project boundary, authority does not
|
|
2049
|
+
(portability.md §4, owner rulings 2026-08-03 and R3 of F-083).
|
|
2050
|
+
|
|
2051
|
+
-> (content, drop, report, errors). content: {rel: bytes} to write instead of
|
|
2052
|
+
the bundle's bytes; drop: rels never to write (a ruling note under its own
|
|
2053
|
+
name); report: {"converted": n, "notes": [renamed notes of skipped topics]}.
|
|
2054
|
+
|
|
2055
|
+
A RULING row arrives IMPORTED, resting on a renamed copy of its note that
|
|
2056
|
+
names the origin; the rows it superseded and the ruling form one CONTESTED
|
|
2057
|
+
set again. The note is never edited in place: a line inserted into it moves
|
|
2058
|
+
the line every locator cites, and check verifies line bounds, not text."""
|
|
2059
|
+
errors, content, drop = [], {}, set()
|
|
2060
|
+
report = {"converted": 0, "notes": []}
|
|
2061
|
+
slug = re.sub(r"[^a-z0-9]", "-", (project or "unknown").lower())
|
|
2062
|
+
parsed = {}
|
|
2063
|
+
for rel in topic_rels:
|
|
2064
|
+
text = (bundle / rel).read_bytes().decode("utf-8")
|
|
2065
|
+
parsed[rel] = (text, kb_parse_claims(text)[0])
|
|
2066
|
+
rulings = [(rel, r) for rel in topic_rels for r in parsed[rel][1]
|
|
2067
|
+
if r["prov"].strip().upper() == "RULING"]
|
|
2068
|
+
# Which notes land foreign. A note cited by a RULING row is a ruling note;
|
|
2069
|
+
# so is any bundle note declaring a basis: (reconciliation.md) -- one can
|
|
2070
|
+
# travel without its row, cited by another row, and land first under its
|
|
2071
|
+
# own name, ready to carry a local-looking RULING later (closure review B1).
|
|
2072
|
+
users = {} # note -> [(topic rel, row)] RULING rows on it
|
|
2073
|
+
for rel, r in rulings:
|
|
2074
|
+
src = r["source"].split(";")[0].strip()
|
|
2075
|
+
path, _, loc = src.rpartition("#")
|
|
2076
|
+
if not (re.match(r"^corpus/notes/[^/]+\.md$", path) and LOC_LINE_RE.match(loc)
|
|
2077
|
+
and (bundle / path).is_file()):
|
|
2078
|
+
if rel in written:
|
|
2079
|
+
errors.append("claim %s in %s is a RULING resting on %r, not on a "
|
|
2080
|
+
"note in corpus/notes/ cited by line (a ruling is a "
|
|
2081
|
+
"note, reconciliation.md) -- carry it by hand"
|
|
2082
|
+
% (r["id"] or "(no id)", rel, src))
|
|
2083
|
+
continue
|
|
2084
|
+
users.setdefault(path, []).append((rel, r))
|
|
2085
|
+
ndir = bundle / "corpus" / "notes"
|
|
2086
|
+
for p in (sorted(ndir.glob("*.md")) if ndir.is_dir() else []):
|
|
2087
|
+
meta = sdlc_core.load_frontmatter(_kb_bom_text(p.read_bytes())[1].splitlines()) or {}
|
|
2088
|
+
if meta.get("basis") and not meta.get("imported_from"):
|
|
2089
|
+
users.setdefault("corpus/notes/" + p.name, [])
|
|
2090
|
+
renames = {} # original note -> (copy, bytes, k)
|
|
2091
|
+
for path in sorted(users):
|
|
2092
|
+
data = (bundle / path).read_bytes()
|
|
2093
|
+
meta = sdlc_core.load_frontmatter(_kb_bom_text(data)[1].splitlines()) or {}
|
|
2094
|
+
ruled = [(rel, r) for rel, r in users[path] if rel in written]
|
|
2095
|
+
where = ("claim %s in %s" % (ruled[0][1]["id"] or "(no id)", ruled[0][0])
|
|
2096
|
+
if ruled else path)
|
|
2097
|
+
if meta.get("imported_from"):
|
|
2098
|
+
if ruled:
|
|
2099
|
+
errors.append("%s is a RULING on %s, which already names a foreign "
|
|
2100
|
+
"origin -- the source project relabelled an imported "
|
|
2101
|
+
"decision; fix it there" % (where, path))
|
|
2102
|
+
continue # already foreign: it travels as it is
|
|
2103
|
+
home = docs / path
|
|
2104
|
+
if not ruled and home.is_file() and kb_sha256_bytes(home) == hashlib.sha256(data).hexdigest():
|
|
2105
|
+
continue # this project holds the note, no topic here rules on it
|
|
2106
|
+
foreign = _kb_foreign_note(data, project, path)
|
|
2107
|
+
if foreign is None:
|
|
2108
|
+
if ruled:
|
|
2109
|
+
errors.append("%s rests on %s, which opens no frontmatter"
|
|
2110
|
+
% (where, path))
|
|
2111
|
+
continue
|
|
2112
|
+
copy = "corpus/notes/imported_%s_%s" % (slug, path[len("corpus/notes/"):])
|
|
2113
|
+
if sdlc_core.confine_under(docs, copy) is None:
|
|
2114
|
+
errors.append("renamed note %r escapes the docs root" % copy)
|
|
2115
|
+
continue
|
|
2116
|
+
renames[path] = (copy, foreign[0], foreign[1])
|
|
2117
|
+
if errors:
|
|
2118
|
+
return content, drop, report, errors
|
|
2119
|
+
for old, (copy, data, _k) in renames.items():
|
|
2120
|
+
drop.add(old)
|
|
2121
|
+
content[copy] = data
|
|
2122
|
+
for p in sorted((bundle / "corpus" / "notes").glob("*.md")) \
|
|
2123
|
+
if (bundle / "corpus" / "notes").is_dir() else []:
|
|
2124
|
+
rel = "corpus/notes/" + p.name
|
|
2125
|
+
if rel in drop:
|
|
2126
|
+
continue
|
|
2127
|
+
new = _kb_rewrite_derived_from(p.read_bytes(),
|
|
2128
|
+
dict((o, c) for o, (c, _d, _k) in renames.items()))
|
|
2129
|
+
if new != p.read_bytes():
|
|
2130
|
+
content[rel] = new
|
|
2131
|
+
for copy in [c for c in list(content) if c in [v[0] for v in renames.values()]]:
|
|
2132
|
+
content[copy] = _kb_rewrite_derived_from(
|
|
2133
|
+
content[copy], dict((o, c) for o, (c, _d, _k) in renames.items()))
|
|
2134
|
+
|
|
2135
|
+
# rows of the topics this import writes: move sources, re-mint ids
|
|
2136
|
+
ren, rows_w, prov_of = {}, [], {}
|
|
2137
|
+
for rel in topic_rels:
|
|
2138
|
+
if rel not in written:
|
|
2139
|
+
continue
|
|
2140
|
+
for r in parsed[rel][1]:
|
|
2141
|
+
rows_w.append((rel, r))
|
|
2142
|
+
for rel, r in rows_w:
|
|
2143
|
+
cells = [s.strip() for s in r["source"].split(";") if s.strip()]
|
|
2144
|
+
moved = []
|
|
2145
|
+
for n, s in enumerate(cells):
|
|
2146
|
+
path, _, loc = s.rpartition("#")
|
|
2147
|
+
if path in renames and LOC_LINE_RE.match(loc):
|
|
2148
|
+
copy, data, k = renames[path]
|
|
2149
|
+
nloc = _kb_shift_loc(loc, k)
|
|
2150
|
+
if _kb_span((bundle / path).read_bytes(), loc) != _kb_span(data, nloc):
|
|
2151
|
+
errors.append("claim %s in %s: %s#%s would not cite the same text "
|
|
2152
|
+
"in %s#%s -- nothing was converted"
|
|
2153
|
+
% (r["id"], rel, path, loc, copy, nloc))
|
|
2154
|
+
cells[n] = "%s#%s" % (copy, nloc)
|
|
2155
|
+
moved.append(n)
|
|
2156
|
+
elif path in renames:
|
|
2157
|
+
errors.append("claim %s in %s cites %s by %r, not by line: the "
|
|
2158
|
+
"renamed copy would move it" % (r["id"], rel, path, loc))
|
|
2159
|
+
r["_source"] = "; ".join(cells) if moved else r["source"]
|
|
2160
|
+
r["_id"] = r["id"]
|
|
2161
|
+
if 0 in moved and r["id"]:
|
|
2162
|
+
path, _, loc = cells[0].rpartition("#")
|
|
2163
|
+
try:
|
|
2164
|
+
r["_id"] = kb_claim_id(path, loc, kb_qty_key(r["qty"]))
|
|
2165
|
+
except ValueError:
|
|
2166
|
+
pass
|
|
2167
|
+
ren[r["id"]] = r["_id"]
|
|
2168
|
+
conv = r["prov"].strip().upper() == "RULING" and \
|
|
2169
|
+
r["source"].split(";")[0].strip().rpartition("#")[0] in renames
|
|
2170
|
+
r["_prov"] = "IMPORTED" if conv else r["prov"]
|
|
2171
|
+
r["_conv"] = conv
|
|
2172
|
+
if errors:
|
|
2173
|
+
return content, drop, report, errors
|
|
2174
|
+
converted = set(r["id"] for _rel, r in rows_w if r["_conv"])
|
|
2175
|
+
report["converted"] = len(converted)
|
|
2176
|
+
members = {}
|
|
2177
|
+
for rel, r in rows_w:
|
|
2178
|
+
kind, targets = _kb_states(r["state"])
|
|
2179
|
+
hit = [t for t in targets if t in converted]
|
|
2180
|
+
if kind != "SUPERSEDED" or not hit:
|
|
2181
|
+
continue
|
|
2182
|
+
if len(hit) != len(targets):
|
|
2183
|
+
errors.append("claim %s in %s is SUPERSEDED by the ruling %s and by %s, "
|
|
2184
|
+
"which is not a ruling: its set cannot reopen without "
|
|
2185
|
+
"deciding which successor stands -- re-rule it in the "
|
|
2186
|
+
"source project" % (r["id"], rel, ", ".join(hit),
|
|
2187
|
+
", ".join(t for t in targets if t not in hit)))
|
|
2188
|
+
continue
|
|
2189
|
+
for t in hit:
|
|
2190
|
+
members.setdefault(t, set()).add(r["id"])
|
|
2191
|
+
partners = {}
|
|
2192
|
+
for rel, r in rows_w:
|
|
2193
|
+
if r["_conv"] and members.get(r["id"]):
|
|
2194
|
+
kind, targets = _kb_states(r["state"])
|
|
2195
|
+
if kind == "SUPERSEDED" and any(t not in converted for t in targets):
|
|
2196
|
+
errors.append("ruling %s in %s settled rows but was itself superseded "
|
|
2197
|
+
"by %s, which is not a ruling: its set cannot reopen "
|
|
2198
|
+
"cleanly -- re-rule it in the source project"
|
|
2199
|
+
% (r["id"], rel, ", ".join(targets)))
|
|
2200
|
+
group = members[r["id"]] | {r["id"]}
|
|
2201
|
+
for x in group:
|
|
2202
|
+
partners.setdefault(x, set()).update(group - {x})
|
|
2203
|
+
if errors:
|
|
2204
|
+
return content, drop, report, errors
|
|
2205
|
+
for rel, r in rows_w:
|
|
2206
|
+
kind, targets = _kb_states(r["state"])
|
|
2207
|
+
if r["id"] in partners:
|
|
2208
|
+
own = set(targets) if kind == "CONTESTED" else set()
|
|
2209
|
+
r["_state"] = "CONTESTED " + ", ".join(
|
|
2210
|
+
sorted(ren.get(t, t) for t in own | partners[r["id"]]))
|
|
2211
|
+
elif kind:
|
|
2212
|
+
r["_state"] = "%s %s" % (kind, ", ".join(ren.get(t, t) for t in targets))
|
|
2213
|
+
else:
|
|
2214
|
+
r["_state"] = r["state"]
|
|
2215
|
+
prov_of[r["_id"]] = r["_prov"].strip().upper()
|
|
2216
|
+
|
|
2217
|
+
# boundary: every pointer must resolve after the import, never onto a foreign row
|
|
2218
|
+
held = {}
|
|
2219
|
+
for _slug, (_p, _t, trows) in kb_collect_topics(docs).items():
|
|
2220
|
+
for t in trows:
|
|
2221
|
+
if t["id"]:
|
|
2222
|
+
held[t["id"]] = (t["prov"].strip().upper(), t["state"])
|
|
2223
|
+
for rel, r in rows_w:
|
|
2224
|
+
kind, targets = _kb_states(r["_state"])
|
|
2225
|
+
for t in targets:
|
|
2226
|
+
prov = prov_of.get(t) or (held.get(t) or (None,))[0]
|
|
2227
|
+
if t not in prov_of and t in held and kind == "CONTESTED":
|
|
2228
|
+
hkind, hts = _kb_states(held[t][1])
|
|
2229
|
+
if hkind != "CONTESTED" or r["_id"] not in hts:
|
|
2230
|
+
errors.append("claim %s in %s is CONTESTED with %s, which this "
|
|
2231
|
+
"project holds without naming it back: the import "
|
|
2232
|
+
"never edits an existing topic -- add the pointer "
|
|
2233
|
+
"there first, or carry the row by hand"
|
|
2234
|
+
% (r["_id"], rel, t))
|
|
2235
|
+
continue
|
|
2236
|
+
if prov is None:
|
|
2237
|
+
errors.append("claim %s in %s points at %s, which this import does not "
|
|
2238
|
+
"write and this project does not hold: its set would "
|
|
2239
|
+
"arrive incomplete -- import the topics together"
|
|
2240
|
+
% (r["_id"], rel, t))
|
|
2241
|
+
elif kind == "SUPERSEDED" and prov == "IMPORTED":
|
|
2242
|
+
errors.append("claim %s in %s is SUPERSEDED by %s, which is IMPORTED "
|
|
2243
|
+
"here: a foreign decision cannot settle it -- import the "
|
|
2244
|
+
"topics together, or reopen the set by hand"
|
|
2245
|
+
% (r["_id"], rel, t))
|
|
2246
|
+
if errors:
|
|
2247
|
+
return content, drop, report, errors
|
|
1453
2248
|
|
|
1454
|
-
|
|
1455
|
-
|
|
2249
|
+
for rel in topic_rels:
|
|
2250
|
+
if rel not in written:
|
|
2251
|
+
continue
|
|
2252
|
+
text, rows = parsed[rel]
|
|
2253
|
+
lines = text.split("\n")
|
|
2254
|
+
for r in rows:
|
|
2255
|
+
up = {}
|
|
2256
|
+
for col, key in (("id", "_id"), ("source", "_source"), ("prov", "_prov"),
|
|
2257
|
+
("state", "_state")):
|
|
2258
|
+
if r[key] != r[col]:
|
|
2259
|
+
up[CLAIM_COLUMNS.index(col)] = r[key]
|
|
2260
|
+
if up:
|
|
2261
|
+
lines[r["_line"] - 1] = _kb_replace_cells(lines[r["_line"] - 1], up)
|
|
2262
|
+
new = "\n".join(lines).encode("utf-8")
|
|
2263
|
+
if new != (bundle / rel).read_bytes():
|
|
2264
|
+
content[rel] = new
|
|
2265
|
+
for rel, r in rulings:
|
|
2266
|
+
path = r["source"].split(";")[0].strip().rpartition("#")[0]
|
|
2267
|
+
if rel not in written and path in renames:
|
|
2268
|
+
note = renames[path][0]
|
|
2269
|
+
if note not in report["notes"]:
|
|
2270
|
+
report["notes"].append(note)
|
|
2271
|
+
return content, drop, report, errors
|
|
2272
|
+
|
|
2273
|
+
|
|
2274
|
+
def kb_import_plan(bundle, docs, report=None):
|
|
1456
2275
|
"""(writes, skipped_topics, dedup, errors) -- computed BEFORE anything is
|
|
1457
2276
|
written, because an import that half-applies leaves a tree whose checks fail
|
|
1458
|
-
and whose owner cannot tell what landed.
|
|
2277
|
+
and whose owner cannot tell what landed. Each write is (rel, bytes, dst):
|
|
2278
|
+
the bytes are the bundle's, except where F-083 makes a foreign ruling land
|
|
2279
|
+
foreign (kb_import_foreign_rulings). `report`, when a dict, receives what
|
|
2280
|
+
that conversion did."""
|
|
1459
2281
|
errors, writes, skipped, dedup = [], [], [], []
|
|
1460
2282
|
man = bundle / BUNDLE_MANIFEST
|
|
1461
2283
|
if not man.is_file():
|
|
@@ -1471,6 +2293,7 @@ def kb_import_plan(bundle, docs):
|
|
|
1471
2293
|
if r["id"]:
|
|
1472
2294
|
known_ids.add(r["id"])
|
|
1473
2295
|
|
|
2296
|
+
entries, topic_rels, written = [], [], set()
|
|
1474
2297
|
for p in sorted(bundle.rglob("*")):
|
|
1475
2298
|
if not p.is_file() or p.name == BUNDLE_MANIFEST:
|
|
1476
2299
|
continue
|
|
@@ -1480,22 +2303,48 @@ def kb_import_plan(bundle, docs):
|
|
|
1480
2303
|
errors.append("bundle entry %r escapes the docs root — refusing the "
|
|
1481
2304
|
"whole import, not just this file" % rel)
|
|
1482
2305
|
continue
|
|
2306
|
+
if rel.startswith("topics/"):
|
|
2307
|
+
topic_rels.append(rel)
|
|
2308
|
+
if dst.is_file():
|
|
2309
|
+
skipped.append(rel)
|
|
2310
|
+
continue
|
|
2311
|
+
written.add(rel)
|
|
2312
|
+
entries.append((rel, p, dst))
|
|
2313
|
+
for rel, p, _dst in entries:
|
|
2314
|
+
if rel.startswith("topics/"):
|
|
2315
|
+
for r in kb_parse_claims(sdlc_core.read_text(p))[0]:
|
|
2316
|
+
if _kb_is_declared(r):
|
|
2317
|
+
errors.append("%s: claim %s is DECLARED -- a declaration stays in "
|
|
2318
|
+
"the project that made it; export never ships one, "
|
|
2319
|
+
"so this bundle was edited by hand"
|
|
2320
|
+
% (rel, r["id"] or "(no id)"))
|
|
2321
|
+
if errors:
|
|
2322
|
+
return [], skipped, [], errors
|
|
2323
|
+
|
|
2324
|
+
content, drop, rep, errors = kb_import_foreign_rulings(
|
|
2325
|
+
bundle, docs, topic_rels, written, str(meta.get("source_project", "")).strip())
|
|
2326
|
+
if report is not None:
|
|
2327
|
+
report.update(rep)
|
|
2328
|
+
if errors:
|
|
2329
|
+
return [], skipped, [], errors
|
|
2330
|
+
planned = [(rel, p, dst) for rel, p, dst in entries if rel not in drop]
|
|
2331
|
+
for rel in sorted(set(content) - set(rel for rel, _p, _d in entries)):
|
|
2332
|
+
planned.append((rel, None, sdlc_core.confine_under(docs, rel)))
|
|
2333
|
+
for rel, p, dst in planned:
|
|
2334
|
+
data = content[rel] if rel in content else p.read_bytes()
|
|
1483
2335
|
if dst.is_file() and not rel.startswith("topics/"):
|
|
1484
|
-
if kb_sha256_bytes(dst) !=
|
|
2336
|
+
if kb_sha256_bytes(dst) != hashlib.sha256(data).hexdigest():
|
|
1485
2337
|
errors.append("%s exists with different bytes (target %s… vs "
|
|
1486
2338
|
"bundle %s…): content-addressed names must mean "
|
|
1487
2339
|
"equal content" % (rel, kb_sha256_bytes(dst)[:8],
|
|
1488
|
-
|
|
1489
|
-
continue
|
|
1490
|
-
if rel.startswith("topics/") and dst.is_file():
|
|
1491
|
-
skipped.append(rel)
|
|
2340
|
+
hashlib.sha256(data).hexdigest()[:8]))
|
|
1492
2341
|
continue
|
|
1493
|
-
writes.append((rel,
|
|
2342
|
+
writes.append((rel, data, dst))
|
|
1494
2343
|
|
|
1495
|
-
for rel,
|
|
2344
|
+
for rel, data, _dst in writes:
|
|
1496
2345
|
if not rel.startswith("topics/"):
|
|
1497
2346
|
continue
|
|
1498
|
-
rows, _ = kb_parse_claims(
|
|
2347
|
+
rows, _ = kb_parse_claims(data.decode("utf-8"))
|
|
1499
2348
|
for r in rows:
|
|
1500
2349
|
if r["id"] and r["id"] in known_ids:
|
|
1501
2350
|
dedup.append(r["id"])
|
|
@@ -1622,15 +2471,20 @@ def kb_refresh_indexes(docs):
|
|
|
1622
2471
|
|
|
1623
2472
|
|
|
1624
2473
|
def kb_validate_surface(docs, full=False):
|
|
1625
|
-
"""Additive seam for marketing: preserve its own validator and report.
|
|
2474
|
+
"""Additive seam for marketing: preserve its own validator and report.
|
|
2475
|
+
`full` is marketing's check: the credential scan (F-077) runs there outside
|
|
2476
|
+
the topics/corpus guard, since document-guide snapshots need no corpus."""
|
|
1626
2477
|
rc = max(memory_validate(docs), _kb_extra_validate(docs))
|
|
2478
|
+
if full:
|
|
2479
|
+
rc = max(rc, _kb_print_credentials(docs.parent, docs))
|
|
1627
2480
|
if full and ((docs / "topics").is_dir() or (docs / "corpus").is_dir()):
|
|
1628
2481
|
cycle = kb_time_cycle(docs)
|
|
1629
2482
|
rc = max(rc, kb_cmd_graph(docs, cycle), kb_cmd_corpus(docs, cycle))
|
|
1630
2483
|
return rc
|
|
1631
2484
|
|
|
1632
2485
|
INTERCEPTED = {"index", "validate", "check", "graph", "corpus", "claim-id",
|
|
1633
|
-
"anchor", "export", "import", "stale", "recall"
|
|
2486
|
+
"anchor", "export", "import", "stale", "recall", "adopt",
|
|
2487
|
+
"inventory"}
|
|
1634
2488
|
|
|
1635
2489
|
|
|
1636
2490
|
def _kb_root(args):
|
|
@@ -1670,8 +2524,9 @@ def kb_cmd_graph(docs, cycle=None):
|
|
|
1670
2524
|
return 0
|
|
1671
2525
|
|
|
1672
2526
|
|
|
1673
|
-
def kb_cmd_corpus(docs, cycle=None):
|
|
2527
|
+
def kb_cmd_corpus(docs, cycle=None, extra_errors=()):
|
|
1674
2528
|
errors, warnings = kb_corpus_check(docs)
|
|
2529
|
+
errors = list(extra_errors) + errors
|
|
1675
2530
|
if cycle is None:
|
|
1676
2531
|
cycle = kb_time_cycle(docs)
|
|
1677
2532
|
_tc_cited, tc_chain, tc_dang = kb_time_cycle_warns(cycle)
|
|
@@ -1699,6 +2554,9 @@ def _kb_extra_validate(docs):
|
|
|
1699
2554
|
"""The kb additions to validate: generated-index freshness for the overlay's
|
|
1700
2555
|
two indexes. Prints nothing on a tree without them (TS-K7)."""
|
|
1701
2556
|
rc = 0
|
|
2557
|
+
for e in kb_exclusions(docs)[1]:
|
|
2558
|
+
print("[ERROR] %s" % e)
|
|
2559
|
+
rc = 1
|
|
1702
2560
|
checks = (("topics", kb_build_topic_index), ("corpus", kb_build_corpus_index))
|
|
1703
2561
|
for dirname, builder in checks:
|
|
1704
2562
|
idx = docs / dirname / "INDEX.md"
|
|
@@ -1728,10 +2586,454 @@ def kb_cmd_check(root, docs, strict=False, hybrid=False):
|
|
|
1728
2586
|
("corpus", lambda: kb_cmd_corpus(docs, cycle)))
|
|
1729
2587
|
return sdlc_core.cmd_check(
|
|
1730
2588
|
root, strict=strict, hybrid=hybrid,
|
|
1731
|
-
extra_validate=lambda: max(_kb_extra_validate(docs), memory_validate(docs)
|
|
2589
|
+
extra_validate=lambda: max(_kb_extra_validate(docs), memory_validate(docs),
|
|
2590
|
+
_kb_print_credentials(root, docs)),
|
|
1732
2591
|
extra_sections=sections)
|
|
1733
2592
|
|
|
1734
2593
|
|
|
2594
|
+
def kb_cmd_adopt(root, docs):
|
|
2595
|
+
"""F-077: the dry-run plan of adopting kb-agentic on an existing docs root --
|
|
2596
|
+
which document guides rest on a snapshot, and what moving each into
|
|
2597
|
+
corpus/given/ involves. Reads only; there is no --apply (portability.md
|
|
2598
|
+
par.6: the choice of source and the redaction are judgements)."""
|
|
2599
|
+
if not docs.is_dir():
|
|
2600
|
+
print("[ERROR] no docs root at %s" % docs)
|
|
2601
|
+
return 1
|
|
2602
|
+
given = docs / "corpus" / "given"
|
|
2603
|
+
twins = {}
|
|
2604
|
+
if given.is_dir():
|
|
2605
|
+
for g in sorted(given.iterdir()):
|
|
2606
|
+
if g.is_file() and not g.name.endswith(".meta.md"):
|
|
2607
|
+
try:
|
|
2608
|
+
twins.setdefault(sdlc_core.sha256_file(g), g.name)
|
|
2609
|
+
except OSError:
|
|
2610
|
+
pass
|
|
2611
|
+
print("=== adopt: document guides -> corpus/given/ (dry run) ===")
|
|
2612
|
+
code_n = 0
|
|
2613
|
+
for rel, _p, meta, _t in sdlc_core.list_guides(root):
|
|
2614
|
+
if (meta.get("source_kind") or "document").strip() != "document":
|
|
2615
|
+
code_n += 1
|
|
2616
|
+
continue
|
|
2617
|
+
df = (meta.get("distilled_from") or "").strip()
|
|
2618
|
+
print("%s" % rel)
|
|
2619
|
+
print(" distilled_from: %s" % (df or "(none)"))
|
|
2620
|
+
src = sdlc_core.confine_under(root, df) if df else None
|
|
2621
|
+
if df and src is None:
|
|
2622
|
+
print(" source: outside the project root -- not read")
|
|
2623
|
+
continue
|
|
2624
|
+
if src is None or not src.is_file():
|
|
2625
|
+
print(" source: missing")
|
|
2626
|
+
continue
|
|
2627
|
+
try:
|
|
2628
|
+
where = "/" + src.relative_to(root.resolve()).as_posix()
|
|
2629
|
+
except ValueError:
|
|
2630
|
+
where = ""
|
|
2631
|
+
if "/corpus/given/" in where:
|
|
2632
|
+
print(" source: already in corpus/given/")
|
|
2633
|
+
continue
|
|
2634
|
+
print(" source: %s" % (".sources/ snapshot" if "/reference/.sources/" in where
|
|
2635
|
+
else "elsewhere"))
|
|
2636
|
+
try:
|
|
2637
|
+
twin = twins.get(sdlc_core.sha256_file(src))
|
|
2638
|
+
except OSError:
|
|
2639
|
+
twin = None
|
|
2640
|
+
if twin:
|
|
2641
|
+
print(" same text already in corpus/given/%s" % twin)
|
|
2642
|
+
found = kb_credential_findings(src)
|
|
2643
|
+
if found:
|
|
2644
|
+
print(" " + _kb_cred_message("credential-shaped", found,
|
|
2645
|
+
"redact before storing (distillation.md par.1)")
|
|
2646
|
+
.split(": ", 1)[1])
|
|
2647
|
+
print("%d code guide(s) stay on .sources/." % code_n)
|
|
2648
|
+
print("Nothing was written. Procedure: portability.md par.6 (kb-agentic).")
|
|
2649
|
+
return 0
|
|
2650
|
+
|
|
2651
|
+
|
|
2652
|
+
# --------------------------------------------------------------------------
|
|
2653
|
+
# F-084: the corpus's reach over the project that holds it. A file left out on
|
|
2654
|
+
# purpose and a file nobody opened used to look the same; these classify every
|
|
2655
|
+
# project file against the sidecars, by content, and name the residue.
|
|
2656
|
+
|
|
2657
|
+
KB_REGISTRY = "audit/INVENTORY_exclusions.md"
|
|
2658
|
+
KB_REACH_DEADLINE = 1.0 # orient's walk; inventory has none (on demand)
|
|
2659
|
+
_kb_clock = time.monotonic # injectable for the deadline test
|
|
2660
|
+
_KB_DATE_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})\b")
|
|
2661
|
+
# Windows: FILE_ATTRIBUTE_OFFLINE, RECALL_ON_OPEN, RECALL_ON_DATA_ACCESS;
|
|
2662
|
+
# macOS: SF_DATALESS. A file whose bytes are not on the disk is never read --
|
|
2663
|
+
# reading it would download it.
|
|
2664
|
+
_KB_CLOUD_ATTRS = 0x1000 | 0x40000 | 0x400000
|
|
2665
|
+
_KB_SF_DATALESS = 0x40000000
|
|
2666
|
+
_KB_REPARSE_POINT = 0x400
|
|
2667
|
+
|
|
2668
|
+
|
|
2669
|
+
class _KbDeadline(Exception):
|
|
2670
|
+
pass
|
|
2671
|
+
|
|
2672
|
+
|
|
2673
|
+
def _kb_glob_regex(pattern):
|
|
2674
|
+
"""FS2's gitignore subset -> (compiled regex, dir_only)."""
|
|
2675
|
+
dir_only = pattern.endswith("/")
|
|
2676
|
+
pat = pattern.rstrip("/")
|
|
2677
|
+
out, i = [], 0
|
|
2678
|
+
while i < len(pat):
|
|
2679
|
+
c = pat[i]
|
|
2680
|
+
if pat.startswith("**/", i):
|
|
2681
|
+
out.append("(?:[^/]+/)*")
|
|
2682
|
+
i += 3
|
|
2683
|
+
elif pat.startswith("**", i):
|
|
2684
|
+
out.append(".*")
|
|
2685
|
+
i += 2
|
|
2686
|
+
elif c == "*":
|
|
2687
|
+
out.append("[^/]*")
|
|
2688
|
+
i += 1
|
|
2689
|
+
elif c == "?":
|
|
2690
|
+
out.append("[^/]")
|
|
2691
|
+
i += 1
|
|
2692
|
+
elif c == "[":
|
|
2693
|
+
j = pat.find("]", i + 1)
|
|
2694
|
+
if j < 0:
|
|
2695
|
+
out.append(re.escape(c))
|
|
2696
|
+
i += 1
|
|
2697
|
+
else:
|
|
2698
|
+
body = pat[i + 1:j]
|
|
2699
|
+
if body.startswith("!"):
|
|
2700
|
+
body = "^" + body[1:]
|
|
2701
|
+
out.append("[%s]" % body.replace("\\", "\\\\"))
|
|
2702
|
+
i = j + 1
|
|
2703
|
+
else:
|
|
2704
|
+
out.append(re.escape(c))
|
|
2705
|
+
i += 1
|
|
2706
|
+
rx = "".join(out)
|
|
2707
|
+
rx = ("^" + rx + "$") if "/" in pat else ("^(?:.*/)?" + rx + "$")
|
|
2708
|
+
return re.compile(rx, re.IGNORECASE if os.name == "nt" else 0), dir_only
|
|
2709
|
+
|
|
2710
|
+
|
|
2711
|
+
def kb_exclusions(docs):
|
|
2712
|
+
"""(rows, errors) of the exclusion registry. rows: (pattern, regex,
|
|
2713
|
+
dir_only). An absent registry excludes nothing and is silent."""
|
|
2714
|
+
p = docs / KB_REGISTRY
|
|
2715
|
+
rows, errors = [], []
|
|
2716
|
+
if not p.is_file():
|
|
2717
|
+
return rows, errors
|
|
2718
|
+
import datetime as _dt
|
|
2719
|
+
for n, line in enumerate(sdlc_core.read_text(p).splitlines(), 1):
|
|
2720
|
+
s = line.strip()
|
|
2721
|
+
if not s.startswith("|"):
|
|
2722
|
+
continue
|
|
2723
|
+
cells = [c.strip() for c in s.strip("|").split("|")]
|
|
2724
|
+
joined = "".join(cells)
|
|
2725
|
+
if "-" in joined and set(joined) <= set("-: "):
|
|
2726
|
+
continue # separator
|
|
2727
|
+
if cells and cells[0].lower() == "pattern":
|
|
2728
|
+
continue # header
|
|
2729
|
+
where = "%s:%d" % (KB_REGISTRY, n)
|
|
2730
|
+
if len(cells) != 3:
|
|
2731
|
+
errors.append("%s: a row has three cells: pattern | reason | approved" % where)
|
|
2732
|
+
continue
|
|
2733
|
+
pattern, reason, approved = cells
|
|
2734
|
+
bad = []
|
|
2735
|
+
if not pattern:
|
|
2736
|
+
bad.append("empty pattern")
|
|
2737
|
+
elif pattern.startswith("/") or ".." in pattern.split("/"):
|
|
2738
|
+
bad.append("pattern %r leaves the project root" % pattern)
|
|
2739
|
+
if not reason:
|
|
2740
|
+
bad.append("empty reason -- say why it never enters the corpus")
|
|
2741
|
+
m = _KB_DATE_RE.match(approved)
|
|
2742
|
+
ok_date = False
|
|
2743
|
+
if m:
|
|
2744
|
+
try:
|
|
2745
|
+
_dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
2746
|
+
ok_date = True
|
|
2747
|
+
except ValueError:
|
|
2748
|
+
pass
|
|
2749
|
+
if not ok_date:
|
|
2750
|
+
bad.append("approved %r is not the day of the practitioner's approval "
|
|
2751
|
+
"(YYYY-MM-DD)" % approved)
|
|
2752
|
+
rx = dir_only = None
|
|
2753
|
+
if not bad:
|
|
2754
|
+
try:
|
|
2755
|
+
rx, dir_only = _kb_glob_regex(pattern)
|
|
2756
|
+
except re.error as e:
|
|
2757
|
+
bad.append("pattern %r is not a valid glob (%s)" % (pattern, e))
|
|
2758
|
+
if bad:
|
|
2759
|
+
errors.append("%s: %s" % (where, "; ".join(bad)))
|
|
2760
|
+
continue
|
|
2761
|
+
rows.append((pattern, rx, dir_only))
|
|
2762
|
+
return rows, errors
|
|
2763
|
+
|
|
2764
|
+
|
|
2765
|
+
def kb_excluded(rel, is_dir, rows):
|
|
2766
|
+
"""The registry pattern matching project-relative `rel`, or None."""
|
|
2767
|
+
for pattern, rx, dir_only in rows:
|
|
2768
|
+
if dir_only and not is_dir:
|
|
2769
|
+
continue
|
|
2770
|
+
if rx.match(rel):
|
|
2771
|
+
return pattern
|
|
2772
|
+
return None
|
|
2773
|
+
|
|
2774
|
+
|
|
2775
|
+
def kb_unreadable(entry):
|
|
2776
|
+
"""'link' / 'cloud-only' for a DirEntry whose bytes are never read; None
|
|
2777
|
+
otherwise. Links include Windows junctions (the `_memory_safe` idiom)."""
|
|
2778
|
+
try:
|
|
2779
|
+
if entry.is_symlink():
|
|
2780
|
+
return "link"
|
|
2781
|
+
is_junction = getattr(entry, "is_junction", None)
|
|
2782
|
+
if is_junction is not None and is_junction():
|
|
2783
|
+
return "link"
|
|
2784
|
+
st = entry.stat(follow_symlinks=False)
|
|
2785
|
+
except OSError:
|
|
2786
|
+
return "unreadable"
|
|
2787
|
+
attrs = getattr(st, "st_file_attributes", 0) or 0
|
|
2788
|
+
if attrs & _KB_REPARSE_POINT and entry.is_dir(follow_symlinks=False):
|
|
2789
|
+
return "link" # a junction on a runtime without is_junction
|
|
2790
|
+
if attrs & _KB_CLOUD_ATTRS or (getattr(st, "st_flags", 0) or 0) & _KB_SF_DATALESS:
|
|
2791
|
+
return "cloud-only"
|
|
2792
|
+
return None
|
|
2793
|
+
|
|
2794
|
+
|
|
2795
|
+
def _kb_path_key(p):
|
|
2796
|
+
return os.path.normcase(os.path.abspath(str(p)))
|
|
2797
|
+
|
|
2798
|
+
|
|
2799
|
+
def _kb_reach_index(docs):
|
|
2800
|
+
"""What the sidecars say about originals: digest -> sidecar, named path ->
|
|
2801
|
+
(sidecar, original_sha256 or None), the artifacts' sizes. Tombstones count
|
|
2802
|
+
for nothing, their `original_path:` included."""
|
|
2803
|
+
by_digest, by_path, sizes = {}, {}, set()
|
|
2804
|
+
given = docs / "corpus" / "given"
|
|
2805
|
+
if not given.is_dir():
|
|
2806
|
+
return by_digest, by_path, sizes
|
|
2807
|
+
for meta_p in sorted(given.glob("*.meta.md")):
|
|
2808
|
+
meta = sdlc_core.load_frontmatter(sdlc_core.read_text(meta_p).splitlines()) or {}
|
|
2809
|
+
if kb_is_tombstone(meta):
|
|
2810
|
+
continue
|
|
2811
|
+
rel = "corpus/given/" + meta_p.name
|
|
2812
|
+
try:
|
|
2813
|
+
sizes.add((given / meta_p.name[:-len(".meta.md")]).stat().st_size)
|
|
2814
|
+
except OSError:
|
|
2815
|
+
pass
|
|
2816
|
+
for k in ("sha256", "original_sha256"):
|
|
2817
|
+
v = (meta.get(k) or "").strip().lower()
|
|
2818
|
+
if v:
|
|
2819
|
+
by_digest.setdefault(v, rel)
|
|
2820
|
+
op = kb_unquote(meta.get("original_path"))
|
|
2821
|
+
if op:
|
|
2822
|
+
osha = (meta.get("original_sha256") or "").strip().lower() or None
|
|
2823
|
+
for c in _kb_original_candidates(op, docs):
|
|
2824
|
+
try:
|
|
2825
|
+
present = c.is_symlink() or c.exists()
|
|
2826
|
+
except OSError:
|
|
2827
|
+
present = False
|
|
2828
|
+
if present:
|
|
2829
|
+
by_path.setdefault(_kb_path_key(c), (rel, osha))
|
|
2830
|
+
break
|
|
2831
|
+
return by_digest, by_path, sizes
|
|
2832
|
+
|
|
2833
|
+
|
|
2834
|
+
def kb_corpus_reach(docs, rel=".", exact=True, deadline=None):
|
|
2835
|
+
"""FS1/FS3: classify the project files under `rel` (project-relative).
|
|
2836
|
+
|
|
2837
|
+
exact=True (inventory): every candidate is hashed -- comparison by content,
|
|
2838
|
+
wherever the file now lives. exact=False (orient's bound): only files whose
|
|
2839
|
+
size equals some given/ artifact's are hashed, and every file an
|
|
2840
|
+
original_path names counts as examined, so the NOT_EXAMINED count never
|
|
2841
|
+
undercounts. Returns {"entries": [(rel, STATE, basis)], "not_examined": n}.
|
|
2842
|
+
Raises _KbDeadline when `deadline` seconds of walking pass.
|
|
2843
|
+
"""
|
|
2844
|
+
project = docs.parent
|
|
2845
|
+
rows, _errors = kb_exclusions(docs)
|
|
2846
|
+
by_digest, by_path, sizes = _kb_reach_index(docs)
|
|
2847
|
+
docs_key = _kb_path_key(docs)
|
|
2848
|
+
entries, counter = [], [0]
|
|
2849
|
+
start = _kb_clock()
|
|
2850
|
+
|
|
2851
|
+
def add(r, state, basis=""):
|
|
2852
|
+
entries.append((r, state, basis))
|
|
2853
|
+
if state == "NOT_EXAMINED":
|
|
2854
|
+
counter[0] += 1
|
|
2855
|
+
|
|
2856
|
+
def visit(e, erel):
|
|
2857
|
+
if deadline is not None and _kb_clock() - start > deadline:
|
|
2858
|
+
raise _KbDeadline()
|
|
2859
|
+
kind = kb_unreadable(e)
|
|
2860
|
+
try:
|
|
2861
|
+
# A link's target is never looked up (a dead network target would
|
|
2862
|
+
# stall orient): a link is matched by its NAME, as if it were a
|
|
2863
|
+
# directory, for the skip rule and dir-only exclusions.
|
|
2864
|
+
dir_like = kind == "link" or e.is_dir(follow_symlinks=False)
|
|
2865
|
+
except OSError:
|
|
2866
|
+
dir_like = False
|
|
2867
|
+
if dir_like and _kb_path_key(e.path) == docs_key:
|
|
2868
|
+
return
|
|
2869
|
+
if dir_like and sdlc_core.skip_dir(e.name):
|
|
2870
|
+
add(erel + "/", "SKIPPED", "built-in rule")
|
|
2871
|
+
return
|
|
2872
|
+
pat = kb_excluded(erel, dir_like, rows)
|
|
2873
|
+
if pat:
|
|
2874
|
+
add(erel + "/" if dir_like else erel, "EXCLUDED", pat)
|
|
2875
|
+
return
|
|
2876
|
+
if kind is None and dir_like:
|
|
2877
|
+
walk(e.path, erel)
|
|
2878
|
+
return
|
|
2879
|
+
if kind == "link":
|
|
2880
|
+
dir_like = False
|
|
2881
|
+
named = by_path.get(_kb_path_key(e.path))
|
|
2882
|
+
if kind:
|
|
2883
|
+
if named:
|
|
2884
|
+
add(erel, "INGESTED", "%s, by path — not read" % named[0])
|
|
2885
|
+
else:
|
|
2886
|
+
add(erel, "NOT_EXAMINED", {"link": "link — not followed",
|
|
2887
|
+
"cloud-only": "cloud-only — not read"}
|
|
2888
|
+
.get(kind, "unreadable — not read"))
|
|
2889
|
+
return
|
|
2890
|
+
if not e.is_file(follow_symlinks=False):
|
|
2891
|
+
return
|
|
2892
|
+
digest = None
|
|
2893
|
+
try:
|
|
2894
|
+
if exact or e.stat(follow_symlinks=False).st_size in sizes:
|
|
2895
|
+
digest = kb_sha256_bytes(Path(e.path))
|
|
2896
|
+
except OSError:
|
|
2897
|
+
digest = None
|
|
2898
|
+
if digest and digest in by_digest:
|
|
2899
|
+
add(erel, "INGESTED", by_digest[digest])
|
|
2900
|
+
elif named:
|
|
2901
|
+
srel, osha = named
|
|
2902
|
+
if osha is None:
|
|
2903
|
+
add(erel, "INGESTED", "%s, by path — digest not kept" % srel)
|
|
2904
|
+
elif not exact:
|
|
2905
|
+
add(erel, "INGESTED", srel) # examined; exactness is inventory's
|
|
2906
|
+
else:
|
|
2907
|
+
add(erel, "INGESTED_EARLIER", "%s, original changed since ingest" % srel)
|
|
2908
|
+
else:
|
|
2909
|
+
add(erel, "NOT_EXAMINED")
|
|
2910
|
+
|
|
2911
|
+
def walk(d, drel):
|
|
2912
|
+
try:
|
|
2913
|
+
with os.scandir(d) as it:
|
|
2914
|
+
items = sorted(it, key=lambda x: x.name)
|
|
2915
|
+
except OSError:
|
|
2916
|
+
add((drel + "/") if drel else "./", "NOT_EXAMINED", "unreadable — not read")
|
|
2917
|
+
return
|
|
2918
|
+
for e in items:
|
|
2919
|
+
visit(e, (drel + "/" + e.name) if drel else e.name)
|
|
2920
|
+
|
|
2921
|
+
rel = (rel or ".").replace("\\", "/").strip("/")
|
|
2922
|
+
if rel in ("", "."):
|
|
2923
|
+
walk(str(project), "")
|
|
2924
|
+
else:
|
|
2925
|
+
target = project / rel
|
|
2926
|
+
with os.scandir(str(target.parent)) as it:
|
|
2927
|
+
for e in it:
|
|
2928
|
+
if os.path.normcase(e.name) == os.path.normcase(target.name):
|
|
2929
|
+
visit(e, rel)
|
|
2930
|
+
entries.sort(key=lambda t: t[0])
|
|
2931
|
+
return {"entries": entries, "not_examined": counter[0]}
|
|
2932
|
+
|
|
2933
|
+
|
|
2934
|
+
def kb_cmd_inventory(root, docs, path):
|
|
2935
|
+
"""FS3: one line per project file under `path`, in path order, with its
|
|
2936
|
+
state. No totals, no filter, no sort by state (r37). Writes nothing."""
|
|
2937
|
+
if not docs.is_dir():
|
|
2938
|
+
print("[ERROR] no docs root")
|
|
2939
|
+
return 1
|
|
2940
|
+
project = docs.parent
|
|
2941
|
+
rel = (path or "").replace("\\", "/").strip()
|
|
2942
|
+
parts = [x for x in rel.split("/") if x not in ("", ".")]
|
|
2943
|
+
if rel.startswith("/") or ntpath.isabs(rel) or ".." in parts:
|
|
2944
|
+
print("[ERROR] %s is outside the project root" % path)
|
|
2945
|
+
return 2
|
|
2946
|
+
# Lexical, never resolved: a link on the way is not followed (FS1). Each
|
|
2947
|
+
# component is matched to the name on disk, case-insensitively where the
|
|
2948
|
+
# filesystem is, so a differently-cased path is never an empty listing.
|
|
2949
|
+
real = []
|
|
2950
|
+
for name in parts:
|
|
2951
|
+
here = project.joinpath(*real)
|
|
2952
|
+
try:
|
|
2953
|
+
with os.scandir(str(here)) as it:
|
|
2954
|
+
found = [e.name for e in it
|
|
2955
|
+
if os.path.normcase(e.name) == os.path.normcase(name)]
|
|
2956
|
+
except OSError:
|
|
2957
|
+
found = []
|
|
2958
|
+
if not found:
|
|
2959
|
+
print("[ERROR] %s does not exist" % path)
|
|
2960
|
+
return 2
|
|
2961
|
+
real.append(name if name in found else found[0])
|
|
2962
|
+
if len(real) < len(parts) and (
|
|
2963
|
+
os.path.islink(project.joinpath(*real))
|
|
2964
|
+
or getattr(os.path, "isjunction", lambda _p: False)(project.joinpath(*real))):
|
|
2965
|
+
print("[ERROR] %s lies behind %s, a link the inventory never follows"
|
|
2966
|
+
% (path, "/".join(real)))
|
|
2967
|
+
return 2
|
|
2968
|
+
parts = real
|
|
2969
|
+
rows, _e = kb_exclusions(docs)
|
|
2970
|
+
for i, name in enumerate(parts):
|
|
2971
|
+
prefix = "/".join(parts[:i + 1])
|
|
2972
|
+
here = project.joinpath(*parts[:i + 1])
|
|
2973
|
+
if _kb_path_key(here) == _kb_path_key(docs):
|
|
2974
|
+
print("[ERROR] %s is the docs root: the corpus is not a project-file area" % path)
|
|
2975
|
+
return 2
|
|
2976
|
+
dir_like = os.path.islink(here) or os.path.isdir(here)
|
|
2977
|
+
if dir_like and sdlc_core.skip_dir(name):
|
|
2978
|
+
print("[ERROR] %s lies in %s/, which the validator never treats as "
|
|
2979
|
+
"project files (built-in rule)" % (path, prefix))
|
|
2980
|
+
return 2
|
|
2981
|
+
if dir_like and kb_excluded(prefix, True, rows):
|
|
2982
|
+
print("[ERROR] %s lies in %s/, excluded by %s" % (path, prefix, KB_REGISTRY))
|
|
2983
|
+
return 2
|
|
2984
|
+
reach = kb_corpus_reach(docs, "/".join(parts) or ".", exact=True)
|
|
2985
|
+
for r, state, basis in reach["entries"]:
|
|
2986
|
+
print(("%-16s %s %s" % (state, r, basis)).rstrip())
|
|
2987
|
+
print("Nothing was written.")
|
|
2988
|
+
return 0
|
|
2989
|
+
|
|
2990
|
+
|
|
2991
|
+
def kb_changed_originals(docs):
|
|
2992
|
+
"""FS5: [(artifact rel, original_path, n citing claims)] for every live,
|
|
2993
|
+
non-superseded sidecar whose local original's bytes differ from the
|
|
2994
|
+
`original_sha256:` it recorded at ingest."""
|
|
2995
|
+
given = docs / "corpus" / "given"
|
|
2996
|
+
if not given.is_dir():
|
|
2997
|
+
return []
|
|
2998
|
+
metas = []
|
|
2999
|
+
superseded = set()
|
|
3000
|
+
for meta_p in sorted(given.glob("*.meta.md")):
|
|
3001
|
+
meta = sdlc_core.load_frontmatter(sdlc_core.read_text(meta_p).splitlines()) or {}
|
|
3002
|
+
metas.append((meta_p, meta))
|
|
3003
|
+
sup = kb_unquote(meta.get("supersedes"))
|
|
3004
|
+
if sup:
|
|
3005
|
+
superseded.add(sup)
|
|
3006
|
+
cites = {}
|
|
3007
|
+
for _slug, (_p, _t, rows) in kb_collect_topics(docs).items():
|
|
3008
|
+
for r in rows:
|
|
3009
|
+
for src in set(kb_claim_sources(r)):
|
|
3010
|
+
cites[src] = cites.get(src, 0) + 1
|
|
3011
|
+
out = []
|
|
3012
|
+
for meta_p, meta in metas:
|
|
3013
|
+
art = meta_p.name[:-len(".meta.md")]
|
|
3014
|
+
if kb_is_tombstone(meta) or art in superseded:
|
|
3015
|
+
continue
|
|
3016
|
+
op = kb_unquote(meta.get("original_path"))
|
|
3017
|
+
osha = (meta.get("original_sha256") or "").strip().lower()
|
|
3018
|
+
if not (op and osha):
|
|
3019
|
+
continue
|
|
3020
|
+
for c in _kb_original_candidates(op, docs):
|
|
3021
|
+
try:
|
|
3022
|
+
if c.is_symlink() or not c.is_file():
|
|
3023
|
+
continue
|
|
3024
|
+
st = os.lstat(str(c)) # attributes without opening the file
|
|
3025
|
+
if (getattr(st, "st_file_attributes", 0) or 0) & _KB_CLOUD_ATTRS or \
|
|
3026
|
+
(getattr(st, "st_flags", 0) or 0) & _KB_SF_DATALESS:
|
|
3027
|
+
break
|
|
3028
|
+
if kb_sha256_bytes(c) != osha:
|
|
3029
|
+
rel = "corpus/given/" + art
|
|
3030
|
+
out.append((rel, op, cites.get(rel, 0)))
|
|
3031
|
+
except OSError:
|
|
3032
|
+
pass
|
|
3033
|
+
break
|
|
3034
|
+
return out
|
|
3035
|
+
|
|
3036
|
+
|
|
1735
3037
|
def kb_cmd_stale(root, docs, hybrid=False):
|
|
1736
3038
|
"""F-044: the consolidation view. The spine's output and exit semantics
|
|
1737
3039
|
come first (--hybrid passed through untouched), then the `## claims`
|
|
@@ -1747,6 +3049,10 @@ def kb_cmd_stale(root, docs, hybrid=False):
|
|
|
1747
3049
|
% (topic_rel, rid, until,
|
|
1748
3050
|
("cited by: " + ", ".join(by)) if by else "uncited"))
|
|
1749
3051
|
lines += chain_w
|
|
3052
|
+
for rel, op, n in kb_changed_originals(docs):
|
|
3053
|
+
lines.append("%s: original %s changed since ingest -- %d claim(s) cite it; "
|
|
3054
|
+
"ingest the new version as a successor (supersedes:)"
|
|
3055
|
+
% (rel, op, n))
|
|
1750
3056
|
if lines:
|
|
1751
3057
|
print("")
|
|
1752
3058
|
print("## claims")
|
|
@@ -1831,9 +3137,14 @@ def kb_cmd_export(args):
|
|
|
1831
3137
|
if errors:
|
|
1832
3138
|
return 1
|
|
1833
3139
|
out = Path(args.out)
|
|
1834
|
-
kb_bundle_write(docs, out, selected, artifacts, root.name)
|
|
3140
|
+
dropped = kb_bundle_write(docs, out, selected, artifacts, root.name)
|
|
1835
3141
|
print("[ok] bundle written: %s" % out)
|
|
1836
3142
|
print(" topics: %d, artifacts: %d" % (len(selected), len(artifacts)))
|
|
3143
|
+
if dropped:
|
|
3144
|
+
print(" not-yet-honored markers dropped: they describe this "
|
|
3145
|
+
"project's code")
|
|
3146
|
+
if kb_has_declarations(selected):
|
|
3147
|
+
print(" declarations stay in this project: not exported")
|
|
1837
3148
|
if added:
|
|
1838
3149
|
# never silent: a set that grew is a fact about the export, and the
|
|
1839
3150
|
# alternative -- dropping the partner rows -- ships a broken tree.
|
|
@@ -1848,7 +3159,8 @@ def kb_cmd_import(args):
|
|
|
1848
3159
|
if not bundle.is_dir():
|
|
1849
3160
|
print("[ERROR] no such bundle directory: %s" % bundle)
|
|
1850
3161
|
return 2
|
|
1851
|
-
|
|
3162
|
+
report = {}
|
|
3163
|
+
writes, skipped, dedup, errors = kb_import_plan(bundle, docs, report)
|
|
1852
3164
|
for e in errors:
|
|
1853
3165
|
print("[ERROR] %s" % e)
|
|
1854
3166
|
if errors:
|
|
@@ -1858,10 +3170,18 @@ def kb_cmd_import(args):
|
|
|
1858
3170
|
if args.dry_run:
|
|
1859
3171
|
print("[ok] dry run: %d file(s) would be written" % len(writes))
|
|
1860
3172
|
else:
|
|
1861
|
-
for _rel,
|
|
3173
|
+
for _rel, data, dst in writes:
|
|
1862
3174
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
1863
|
-
dst.write_bytes(
|
|
3175
|
+
dst.write_bytes(data)
|
|
1864
3176
|
print("[ok] imported %d file(s) into %s" % (len(writes), docs))
|
|
3177
|
+
if report.get("converted"):
|
|
3178
|
+
# portability.md §0 point 4: the agent must be able to tell the user
|
|
3179
|
+
print(" %d ruling(s) arrive as IMPORTED: each rests on a renamed copy of "
|
|
3180
|
+
"its note (imported_<project>_...) and settles nothing until you "
|
|
3181
|
+
"ratify it (portability.md §4)" % report["converted"])
|
|
3182
|
+
for note in report.get("notes", []):
|
|
3183
|
+
print("[note] %s: a ruling of a skipped topic lands only as this foreign "
|
|
3184
|
+
"copy -- a row merged by hand cites it as IMPORTED" % note)
|
|
1865
3185
|
if dedup:
|
|
1866
3186
|
print(" %d claim(s) already present, by id — the same artifact cited "
|
|
1867
3187
|
"at the same span mints the same id in any project" % len(dedup))
|
|
@@ -1870,7 +3190,8 @@ def kb_cmd_import(args):
|
|
|
1870
3190
|
"(taxonomy.md) and merge by hand — an import never decides that."
|
|
1871
3191
|
% rel)
|
|
1872
3192
|
print("[note] re-run 'sdlc_check.py check' now: the import is additive, and "
|
|
1873
|
-
"an imported
|
|
3193
|
+
"an imported ruling stays IMPORTED until you ratify it with a note of "
|
|
3194
|
+
"your own.")
|
|
1874
3195
|
return 0
|
|
1875
3196
|
|
|
1876
3197
|
|
|
@@ -1886,11 +3207,50 @@ def kb_cmd_orient(argv):
|
|
|
1886
3207
|
needs to locate topics/, and ignores everything else.
|
|
1887
3208
|
|
|
1888
3209
|
The append fails open: a broken graph read must never break orient itself.
|
|
3210
|
+
It is also bounded (F-075): the spine leaves ORIENT_APPEND_RESERVE of the
|
|
3211
|
+
client's hook cap free, and the append is cut to it -- the topic router
|
|
3212
|
+
first, since it is the part that grows -- so the whole output stays under
|
|
3213
|
+
the cap.
|
|
1889
3214
|
"""
|
|
1890
3215
|
try:
|
|
1891
3216
|
rc = sdlc_core.main(argv)
|
|
1892
3217
|
except SystemExit as e:
|
|
1893
3218
|
rc = e.code if isinstance(e.code, int) else 1
|
|
3219
|
+
buf = io.StringIO()
|
|
3220
|
+
try:
|
|
3221
|
+
with contextlib.redirect_stdout(buf):
|
|
3222
|
+
_kb_orient_append(argv)
|
|
3223
|
+
except (Exception, SystemExit):
|
|
3224
|
+
pass
|
|
3225
|
+
text = buf.getvalue()
|
|
3226
|
+
if text:
|
|
3227
|
+
print(_fit_orient_append(text, sdlc_core.ORIENT_APPEND_RESERVE), end="")
|
|
3228
|
+
return rc
|
|
3229
|
+
|
|
3230
|
+
|
|
3231
|
+
def _fit_orient_append(text, budget):
|
|
3232
|
+
"""`text` within `budget` characters: shrink the topic-router block first,
|
|
3233
|
+
then, if the rest alone is too wide, cut the whole append. Never raises."""
|
|
3234
|
+
if len(text) <= budget:
|
|
3235
|
+
return text
|
|
3236
|
+
lines = text.rstrip("\n").split("\n")
|
|
3237
|
+
try:
|
|
3238
|
+
i = lines.index("## Topic router")
|
|
3239
|
+
j = next((k for k in range(i + 1, len(lines)) if not lines[k].strip()), len(lines))
|
|
3240
|
+
block, others = lines[i + 1:j], lines[:i + 1] + lines[j:]
|
|
3241
|
+
room = budget - len("\n".join(others)) - 2
|
|
3242
|
+
if room > 0:
|
|
3243
|
+
fitted = sdlc_core.orient_fit(block, room, "topics/INDEX.md")
|
|
3244
|
+
out = "\n".join(lines[:i + 1] + fitted + lines[j:]) + "\n"
|
|
3245
|
+
if len(out) <= budget:
|
|
3246
|
+
return out
|
|
3247
|
+
except ValueError:
|
|
3248
|
+
pass
|
|
3249
|
+
return "\n".join(sdlc_core.orient_fit(lines, budget - 1, "the knowledge layer's files")) + "\n"
|
|
3250
|
+
|
|
3251
|
+
|
|
3252
|
+
def _kb_orient_append(argv):
|
|
3253
|
+
"""The lines kb_cmd_orient appends after the spine's orientation."""
|
|
1894
3254
|
# Pre-bind so a throw INSIDE the router print (after root was validly
|
|
1895
3255
|
# bound) does not erase it -- the recency block below must survive a
|
|
1896
3256
|
# router failure with the bound root intact (closure R2 finding).
|
|
@@ -1944,7 +3304,7 @@ def kb_cmd_orient(argv):
|
|
|
1944
3304
|
# skill's own worktree hygiene makes that routine), which is why the
|
|
1945
3305
|
# frontmatter date is primary.
|
|
1946
3306
|
if root is None:
|
|
1947
|
-
return
|
|
3307
|
+
return
|
|
1948
3308
|
notes_dir = root / name / "corpus" / "notes"
|
|
1949
3309
|
if notes_dir.is_dir():
|
|
1950
3310
|
import datetime as _dt
|
|
@@ -2017,7 +3377,23 @@ def kb_cmd_orient(argv):
|
|
|
2017
3377
|
% (n_docs, m_exp, k_chain))
|
|
2018
3378
|
except (Exception, SystemExit):
|
|
2019
3379
|
pass
|
|
2020
|
-
|
|
3380
|
+
try:
|
|
3381
|
+
# F-084: the corpus's reach, an upper bound of the project files never
|
|
3382
|
+
# examined (R5). Its own try: a failing walk prints nothing and takes
|
|
3383
|
+
# nothing else with it.
|
|
3384
|
+
if root is not None and (root / name / "corpus").is_dir():
|
|
3385
|
+
try:
|
|
3386
|
+
n = kb_corpus_reach(root / name, ".", exact=False,
|
|
3387
|
+
deadline=KB_REACH_DEADLINE)["not_examined"]
|
|
3388
|
+
except _KbDeadline:
|
|
3389
|
+
print("\nCorpus reach: not counted within 1s -- "
|
|
3390
|
+
"sdlc_check.py inventory <path>")
|
|
3391
|
+
else:
|
|
3392
|
+
if n:
|
|
3393
|
+
print("\nCorpus reach: <= %d project files not examined -- "
|
|
3394
|
+
"exact: sdlc_check.py inventory <path>" % n)
|
|
3395
|
+
except (Exception, SystemExit):
|
|
3396
|
+
pass
|
|
2021
3397
|
|
|
2022
3398
|
|
|
2023
3399
|
def kb_cmd_help():
|
|
@@ -2044,6 +3420,13 @@ shared project memory -- also available:
|
|
|
2044
3420
|
anchor <path> <phrase> resolve a phrase to a verified locator span
|
|
2045
3421
|
export --out <dir> bundle a subgraph WITH the bytes its claims cite
|
|
2046
3422
|
import <dir> merge a bundle in additively (never overwrites)
|
|
3423
|
+
adopt dry run: document guides resting on .sources/
|
|
3424
|
+
snapshots, and what moving them into corpus/given/
|
|
3425
|
+
involves (writes nothing; portability.md par.6)
|
|
3426
|
+
inventory <path> the project files under <path> (. = all), each with
|
|
3427
|
+
its state against the corpus: INGESTED,
|
|
3428
|
+
INGESTED_EARLIER, EXCLUDED, NOT_EXAMINED (or
|
|
3429
|
+
SKIPPED); no totals, writes nothing
|
|
2047
3430
|
remind one-line per-turn reminder (UserPromptSubmit hook,
|
|
2048
3431
|
opt-in -- ENFORCEMENT par.4; constant, zero-read)
|
|
2049
3432
|
|
|
@@ -2111,6 +3494,13 @@ def main(argv=None):
|
|
|
2111
3494
|
p.add_argument("--dry-run", action="store_true")
|
|
2112
3495
|
p.add_argument("--root")
|
|
2113
3496
|
p.add_argument("--docs-dir")
|
|
3497
|
+
p = sub.add_parser("adopt")
|
|
3498
|
+
p.add_argument("--root")
|
|
3499
|
+
p.add_argument("--docs-dir")
|
|
3500
|
+
p = sub.add_parser("inventory")
|
|
3501
|
+
p.add_argument("path")
|
|
3502
|
+
p.add_argument("--root")
|
|
3503
|
+
p.add_argument("--docs-dir")
|
|
2114
3504
|
args = ap.parse_args(argv)
|
|
2115
3505
|
if args.cmd == "claim-id":
|
|
2116
3506
|
return kb_cmd_claim_id(args)
|
|
@@ -2135,7 +3525,11 @@ def main(argv=None):
|
|
|
2135
3525
|
if args.cmd == "graph":
|
|
2136
3526
|
return kb_cmd_graph(docs)
|
|
2137
3527
|
if args.cmd == "corpus":
|
|
2138
|
-
return kb_cmd_corpus(docs)
|
|
3528
|
+
return kb_cmd_corpus(docs, extra_errors=kb_credential_errors(root, docs))
|
|
3529
|
+
if args.cmd == "adopt":
|
|
3530
|
+
return kb_cmd_adopt(root, docs)
|
|
3531
|
+
if args.cmd == "inventory":
|
|
3532
|
+
return kb_cmd_inventory(root, docs, args.path)
|
|
2139
3533
|
if args.cmd == "validate":
|
|
2140
3534
|
return kb_cmd_validate(root, docs, strict=args.strict, hybrid=args.hybrid)
|
|
2141
3535
|
return kb_cmd_check(root, docs, strict=args.strict, hybrid=args.hybrid)
|