kibi-core 0.26.1 → 0.27.0

package/src/kb.pl

@@ -27,6 +27,7 @@
     kb_indexed_sources/1,
     kb_query_entities/8,
     kb_search_entities/6,
+    kb_relationship_edges/3,
     kb_list_search_candidates/5,
     kb_list_search_candidates/6,
     kb_entity_ids/1,
@@ -34,6 +35,7 @@
     kb_entity/3,
     kb_entities_by_source/2,
     kb_assert_relationship/4,
+    kb_assert_relationships_reporting/2,
     kb_query_proof_contracts/4,
     kb_assert_relationship_no_audit/4,
     kb_log_relationship_upsert/4,
@@ -88,6 +90,8 @@
 :- use_module(library(aggregate), [aggregate_all/3]).
 :- use_module(library(lists), [sum_list/2, subtract/3, intersection/3]).
 :- use_module(library(ordsets)).
+:- use_module(library(assoc), [list_to_assoc/2, get_assoc/3]).
+:- use_module(library(pairs), [group_pairs_by_key/2, pairs_values/2]).
 :- use_module(library(solution_sequences), [distinct/1, limit/2, offset/2, order_by/2]).
 :- use_module('../schema/entities.pl', [entity_type/1, entity_property/3, required_property/2]).
 :- use_module('../schema/relationships.pl', [relationship_type/1, valid_relationship/3]).
@@ -140,6 +144,17 @@ kb_uri('urn-kibi:').
 :- dynamic kb_entity_props_memo/2.
 :- dynamic kb_entity_props_memo_key/1.
 :- dynamic kb_decision_signature_memo/2.
+:- dynamic kb_req_predicate_memo/2.
+:- dynamic kb_req_subject_memo/2.
+:- dynamic kb_contradiction_witness_memo/2.
+:- dynamic kb_consistency_memo/3.
+:- dynamic kb_predicate_schema_memo/2.
+:- dynamic kb_requirement_proof_memo/3.
+:- dynamic kb_requirement_ladder_memo/3.
+:- dynamic kb_status_source_index_memo/2.
+:- dynamic kb_scenario_feasibility_memo/3.
+:- dynamic kb_search_ids_memo/4.
+:- dynamic kb_query_ids_memo/3.
 :- dynamic kb_index_verified_key/1.
 :- dynamic entity/4.  % Support legacy .pl file format (Type, Id, Title, Props)
 
@@ -367,6 +382,17 @@ kb_detach :-
             retractall(kb_entity_props_memo(_, _)),
             retractall(kb_entity_props_memo_key(_)),
             retractall(kb_decision_signature_memo(_, _)),
+            retractall(kb_req_predicate_memo(_, _)),
+            retractall(kb_req_subject_memo(_, _)),
+            retractall(kb_contradiction_witness_memo(_, _)),
+            retractall(kb_consistency_memo(_, _, _)),
+            retractall(kb_predicate_schema_memo(_, _)),
+            retractall(kb_requirement_proof_memo(_, _, _)),
+            retractall(kb_requirement_ladder_memo(_, _, _)),
+            retractall(kb_status_source_index_memo(_, _)),
+            retractall(kb_scenario_feasibility_memo(_, _, _)),
+            retractall(kb_search_ids_memo(_, _, _, _)),
+            retractall(kb_query_ids_memo(_, _, _)),
             retractall(kb_index_entity(_, _)),
             retractall(kb_index_type(_, _)),
             retractall(kb_index_tag(_, _)),
@@ -1698,10 +1724,7 @@ kb_query_entities(TypeFilter, IdFilter, Tags, SourceFilter, Limit, Offset, Rows,
     Offset >= 0,
     is_list(Tags),
     kb_ensure_indexes,
-    findall(Id,
-            indexed_entity_match(TypeFilter, IdFilter, Tags, SourceFilter, Id),
-            RawIds),
-    sort(RawIds, Ids),
+    query_index_ids(TypeFilter, IdFilter, Tags, SourceFilter, Ids),
     length(Ids, Count),
     drop_index_ids(Offset, Ids, Remaining),
     take_index_ids(Limit, Remaining, PageIds),
@@ -1710,6 +1733,25 @@ kb_query_entities(TypeFilter, IdFilter, Tags, SourceFilter, Limit, Offset, Rows,
               kb_entity(Id, Type, Props) ),
             Rows).
 
+% Clients read a large result in bounded pages, each page a separate query,
+% so the ids matched for the last filter are kept for its next pages instead
+% of matching and sorting the whole index again per page. Keyed like the
+% search memo, so any write starts a fresh match.
+% The caller ensured the indexes, so the entity count is known.
+query_index_ids(TypeFilter, IdFilter, Tags, SourceFilter, Ids) :-
+    kb_index_entity_count(IndexCount),
+    index_verification_key(IndexCount, Key),
+    Filters = filters(TypeFilter, IdFilter, Tags, SourceFilter),
+    (   kb_query_ids_memo(Key, Filters, Cached)
+    ->  Ids = Cached
+    ;   findall(Id,
+                indexed_entity_match(TypeFilter, IdFilter, Tags, SourceFilter, Id),
+                RawIds),
+        sort(RawIds, Ids),
+        retractall(kb_query_ids_memo(_, _, _)),
+        assertz(kb_query_ids_memo(Key, Filters, Ids))
+    ).
+
 %% kb_query_proof_contracts(+Id, +Limit, +Offset, -Rows)
 % Page proof-campaign candidates while reading only contract and binding RDF
 % properties. In particular, do not call kb_entity/3 here: that materializes
@@ -1848,19 +1890,37 @@ kb_search_entities(TypeFilter, Query, Limit, Offset, Rows, Count) :-
     kb_ensure_indexes,
     index_text_tokens(Query, QueryTokens),
     QueryTokens \= [],
-    findall(Id,
-            ( kb_index_entity(Id, _),
-              (TypeFilter == none -> true ; kb_index_type(TypeFilter, Id)),
-              ( indexed_markdown_source(Id)
-              ; forall(member(QueryToken, QueryTokens),
-                       indexed_token_match(QueryToken, Id)) ) ),
-            RawIds),
-    sort(RawIds, Ids),
+    search_index_ids(TypeFilter, QueryTokens, Ids),
     length(Ids, Count),
     drop_index_ids(Offset, Ids, Remaining),
     take_index_ids(Limit, Remaining, PageIds),
     search_candidate_rows(PageIds, Rows).
 
+% Callers read a large candidate set page by page, each page a separate
+% query, so the ids matched for the last query are kept for its next pages
+% instead of scanning the token index again. The memo is keyed like the index
+% verification (graph, RDF generation and index size), so any write starts a
+% fresh scan.
+search_index_ids(TypeFilter, QueryTokens, Ids) :-
+    (   kb_index_entity_count(IndexCount)
+    ->  true
+    ;   IndexCount = none
+    ),
+    index_verification_key(IndexCount, Key),
+    (   kb_search_ids_memo(Key, TypeFilter, QueryTokens, Cached)
+    ->  Ids = Cached
+    ;   findall(Id,
+                ( kb_index_entity(Id, _),
+                  (TypeFilter == none -> true ; kb_index_type(TypeFilter, Id)),
+                  ( indexed_markdown_source(Id)
+                  ; forall(member(QueryToken, QueryTokens),
+                           indexed_token_match(QueryToken, Id)) ) ),
+                RawIds),
+        sort(RawIds, Ids),
+        retractall(kb_search_ids_memo(_, _, _, _)),
+        assertz(kb_search_ids_memo(Key, TypeFilter, QueryTokens, Ids))
+    ).
+
 %% kb_list_search_candidates(+Type, +Limit, +Offset, -Rows, -Count)
 % Page every indexed entity (optionally of one type) as projected search
 % candidate rows, ordered by id. Semantic discovery uses it when no lexical
@@ -1978,6 +2038,25 @@ kb_assert_relationship(RelType, FromId, ToId, Metadata) :-
         kb_log_relationship_upsert(RelType, FromId, ToId, Metadata)
     ).
 
+%% kb_assert_relationships_reporting(+Relationships, -FailedPositions)
+% Assert each rel(Type, From, To) of Relationships with kb_assert_relationship/4
+% (empty metadata), in order, keeping every one that succeeds. FailedPositions
+% lists the 1-based positions of those that failed or raised; their writes are
+% rolled back with their own transaction. Sync uses it after a batch
+% transaction fails, so one dangling link no longer sends the whole batch
+% through one round trip per relationship.
+kb_assert_relationships_reporting(Relationships, FailedPositions) :-
+    foldl(assert_relationship_reporting, Relationships, 1-[], _-Failed0),
+    reverse(Failed0, FailedPositions).
+
+assert_relationship_reporting(rel(RelType, FromId, ToId), Position-Failed0, Next-Failed) :-
+    Next is Position + 1,
+    % mutalog-disable-next-line literal: an unbound metadata entry is ignored like an empty list
+    (   catch(kb_assert_relationship(RelType, FromId, ToId, []), _, fail)
+    ->  Failed = Failed0
+    ;   Failed = [Position|Failed0]
+    ).
+
 %% kb_assert_relationship_no_audit(+Type, +From, +To, +Metadata)
 % Assert a relationship RDF payload without recording audit side effects.
 % Used by write-gated MCP transactions so failed contradiction checks do not
@@ -2181,6 +2260,37 @@ kb_relationship_with_reluri(RelURI, Graph, FromId, ToId) :-
     entity_uri_to_id(FromURI, FromId),
     entity_uri_to_id(ToURI, ToId).
 
+%% kb_relationship_edges(+Types, +Ids, -Edges)
+% implements REQ-mcp-search-discovery
+% Every [Type, From, To] relationship of one of Types that starts or ends at
+% one of Ids (a set), in the order and with the multiplicity of
+%   findall([T,F,O], (member(T, Types), kb_relationship(T, F, O),
+%                     (member(F, Ids) ; member(O, Ids))), Edges)
+% Search reads the graph around its results this way. Ends are compared as
+% URIs, so only the matching edges are converted back to ids rather than
+% every edge of every type.
+kb_relationship_edges(Types, Ids, Edges) :-
+    kb_graph(Graph),
+    kb_uri(BaseURI),
+    atom_concat(BaseURI, 'entity/', Prefix),
+    findall(URI,
+            ( member(Id, Ids),
+              (   atom_concat(Prefix, Id, URI)
+              ;   atom_concat('kb:entity/', Id, URI)
+              ) ),
+            URIs0),
+    sort(URIs0, URIs),
+    findall([Type, From, To],
+            ( member(Type, Types),
+              atom_concat(BaseURI, Type, RelURI),
+              rdf(FromURI, RelURI, ToURI, Graph),
+              (   ord_memberchk(FromURI, URIs)
+              ;   ord_memberchk(ToURI, URIs)
+              ),
+              entity_uri_to_id(FromURI, From),
+              entity_uri_to_id(ToURI, To) ),
+            Edges).
+
 % Helper predicates
 
 %% entity_id_to_uri(+Id, -URI)
@@ -2639,13 +2749,7 @@ contradicting_reqs(ReqA, ReqB, Reason) :-
 % contradiction.  This remains deliberately syntactic: differently shaped
 % terms are never treated as equivalent merely because their prose is similar.
 req_conflict_witness(ReqA, ReqB, Witness) :-
-    current_req(ReqA),
-    current_req(ReqB),
-    ReqA @< ReqB,
-    kb_relationship(constrains, ReqA, SubjectFactA),
-    kb_relationship(constrains, ReqB, SubjectFactB),
-    fact_subject_key(SubjectFactA, SubjectKey),
-    fact_subject_key(SubjectFactB, SubjectKey),
+    subject_conflict_pair(ReqA, ReqB, SubjectKey),
     effective_req_property_fact(ReqA, SubjectKey, FactA, PropertyKey, OpA, ValTypeA, ValA, UnitA, ScopeA, PolarityA, ValidFromA, ValidToA),
     effective_req_property_fact(ReqB, SubjectKey, FactB, PropertyKey, OpB, ValTypeB, ValB, UnitB, ScopeB, PolarityB, ValidFromB, ValidToB),
     FactA \= FactB,
@@ -2673,11 +2777,7 @@ req_conflict_witness(ReqA, ReqB, Witness) :-
         right: Right
     }.
 req_conflict_witness(ReqA, ReqB, Witness) :-
-    current_req(ReqA),
-    current_req(ReqB),
-    ReqA @< ReqB,
-    effective_req_predicate(ReqA, FactA, Namespace, Name, Args, PolarityA),
-    effective_req_predicate(ReqB, FactB, Namespace, Name, Args, PolarityB),
+    predicate_conflict_pair(ReqA, ReqB, FactA, FactB, Namespace, Name, Args, PolarityA, PolarityB),
     FactA \= FactB,
     opposite_predicate_polarity(PolarityA, PolarityB),
     atomic_list_concat(Args, ',', ArgsText),
@@ -2757,6 +2857,143 @@ req_conflict_witness(ReqA, ReqB, Witness) :-
         right: Right
     }.
 
+% Cost: the search answer layer and the upsert contradiction check ask for
+% the witnesses of one requirement with the other side unbound, and kb_check
+% asks for every pair. Enumerating every current requirement as the partner
+% and only then reading either side's facts cost a full requirement scan per
+% requirement (quadratic for kb_check). The pairings below return the same
+% solutions in the same order as that enumeration: partners come from an
+% index of the current requirements constraining each subject or stating
+% each predicate, visited in the order current_req/1 enumerates them, so a
+% requirement without such facts has no partners at all.
+subject_conflict_pair(ReqA, ReqB, SubjectKey) :-
+    % The mode tests only pick the cheapest route: every route gives the
+    % same solutions in the same order for the arguments it is given.
+    % mutalog-disable-next-line negate,goal_true,goal_fail: routing only; the later branches give the same pairs
+    (   nonvar(ReqA), nonvar(ReqB)
+    ->  current_req(ReqA),
+        current_req(ReqB),
+        ReqA @< ReqB
+    % mutalog-disable-next-line negate,goal_true,goal_fail: routing only; the last branch gives the same pairs
+    ;   nonvar(ReqB)
+    ->  current_req(ReqB),
+        req_subject_partners(ReqB, before, Partners),
+        member(ReqA, Partners)
+    ;   current_req(ReqA),
+        req_subject_partners(ReqA, after, Partners),
+        member(ReqB, Partners)
+    ),
+    kb_relationship(constrains, ReqA, SubjectFactA),
+    kb_relationship(constrains, ReqB, SubjectFactB),
+    fact_subject_key(SubjectFactA, SubjectKey),
+    fact_subject_key(SubjectFactB, SubjectKey).
+
+%% req_subject_partners(+ReqId, +Side, -Partners)
+% Current requirements constraining a subject that ReqId constrains, in
+% current_req/1 order: those after ReqId in standard order (Side = after) or
+% before it (Side = before).
+req_subject_partners(ReqId, Side, Partners) :-
+    req_subject_index(Index),
+    findall(Position-Partner,
+            ( req_subject_fact_key(ReqId, SubjectKey),
+              get_assoc(SubjectKey, Index, Entries),
+              member(Position-Partner, Entries),
+              subject_partner_side(Side, ReqId, Partner) ),
+            Partners0),
+    sort(Partners0, Partners1),
+    pairs_values(Partners1, Partners).
+
+% mutalog-disable-next-line cut_insert: the comparison has at most one solution
+subject_partner_side(after, ReqId, Partner) :- ReqId @< Partner.
+subject_partner_side(before, ReqId, Partner) :- Partner @< ReqId.
+
+req_subject_fact_key(ReqId, SubjectKey) :-
+    % mutalog-disable-next-line goal_true: extra partners are dropped by the pair's own constrains check
+    kb_relationship(constrains, ReqId, SubjectFactId),
+    fact_subject_key(SubjectFactId, SubjectKey).
+
+% Memoized per KB generation, like the predicate index below.
+req_subject_index(Index) :-
+    index_verification_key(req_subjects, Key),
+    (   kb_req_subject_memo(Key, Cached)
+    ->  Index = Cached
+    ;   findall(ReqId, current_req(ReqId), ReqIds),
+        findall(SubjectKey-(Position-ReqId),
+                ( nth1(Position, ReqIds, ReqId),
+                  req_subject_fact_key(ReqId, SubjectKey) ),
+                Pairs0),
+        sort(Pairs0, Pairs),
+        group_pairs_by_key(Pairs, Grouped),
+        list_to_assoc(Grouped, Computed),
+        retractall(kb_req_subject_memo(_, _)),
+        assertz(kb_req_subject_memo(Key, Computed)),
+        Index = Computed
+    ).
+
+predicate_conflict_pair(ReqA, ReqB, FactA, FactB, Namespace, Name, Args, PolarityA, PolarityB) :-
+    (   nonvar(ReqA), var(ReqB)
+    ->  current_req(ReqA),
+        findall(Position-Partner,
+                ( effective_req_predicate(ReqA, _, PartnerNs, PartnerName, PartnerArgs, _),
+                  current_req_predicate(PartnerNs-PartnerName-PartnerArgs, Position, Partner),
+                  ReqA @< Partner ),
+                Partners0),
+        sort(Partners0, Partners),
+        member(_-ReqB, Partners)
+    ;   nonvar(ReqB), var(ReqA)
+    ->  current_req(ReqB),
+        findall(Position-Partner,
+                ( effective_req_predicate(ReqB, _, PartnerNs, PartnerName, PartnerArgs, _),
+                  current_req_predicate(PartnerNs-PartnerName-PartnerArgs, Position, Partner),
+                  Partner @< ReqB ),
+                Partners0),
+        sort(Partners0, Partners),
+        member(_-ReqA, Partners)
+    % mutalog-disable-next-line negate,goal_true,goal_fail: routing only; the last branch gives the same pairs
+    ;   nonvar(ReqA), nonvar(ReqB)
+    ->  current_req(ReqA),
+        current_req(ReqB),
+        ReqA @< ReqB
+    ;   current_req(ReqA),
+        findall(Position-Partner,
+                ( effective_req_predicate(ReqA, _, PartnerNs, PartnerName, PartnerArgs, _),
+                  current_req_predicate(PartnerNs-PartnerName-PartnerArgs, Position, Partner),
+                  ReqA @< Partner ),
+                Partners0),
+        sort(Partners0, Partners),
+        member(_-ReqB, Partners)
+    ),
+    effective_req_predicate(ReqA, FactA, Namespace, Name, Args, PolarityA),
+    effective_req_predicate(ReqB, FactB, Namespace, Name, Args, PolarityB).
+
+%% current_req_predicate(+Signature, -Position, -ReqId)
+% A current requirement stating a predicate fact with Signature
+% (Namespace-Name-Args), and its position in current_req/1 order.
+current_req_predicate(Signature, Position, ReqId) :-
+    req_predicate_index(Index),
+    get_assoc(Signature, Index, Entries),
+    member(Position-ReqId, Entries).
+
+% Memoized per KB generation, like the decision signatures: reading a
+% predicate fact decodes its entity, and every lookup would otherwise read
+% the predicate facts of every requirement.
+req_predicate_index(Index) :-
+    index_verification_key(req_predicates, Key),
+    (   kb_req_predicate_memo(Key, Cached)
+    ->  Index = Cached
+    ;   findall(ReqId, current_req(ReqId), ReqIds),
+        findall((Namespace-Name-Args)-(Position-ReqId),
+                ( nth1(Position, ReqIds, ReqId),
+                  effective_req_predicate(ReqId, _, Namespace, Name, Args, _) ),
+                Pairs0),
+        sort(Pairs0, Pairs),
+        group_pairs_by_key(Pairs, Grouped),
+        list_to_assoc(Grouped, Computed),
+        retractall(kb_req_predicate_memo(_, _)),
+        assertz(kb_req_predicate_memo(Key, Computed)),
+        Index = Computed
+    ).
+
 %% kb_domain_contradiction_pairs_examined(-Count)
 % implements REQ-search-answer-hygiene
 % How many distinct pairs of requirement facts the domain-contradictions
@@ -2772,22 +3009,12 @@ kb_domain_contradiction_pairs_examined(Count) :-
     length(Pairs, Count).
 
 domain_contradiction_candidate_pair(FactA, FactB) :-
-    current_req(ReqA),
-    current_req(ReqB),
-    ReqA @< ReqB,
-    kb_relationship(constrains, ReqA, SubjectFactA),
-    kb_relationship(constrains, ReqB, SubjectFactB),
-    fact_subject_key(SubjectFactA, SubjectKey),
-    fact_subject_key(SubjectFactB, SubjectKey),
+    subject_conflict_pair(ReqA, ReqB, SubjectKey),
     effective_req_property_fact(ReqA, SubjectKey, FactA, PropertyKey, _, _, _, _, _, _, _, _),
     effective_req_property_fact(ReqB, SubjectKey, FactB, PropertyKey, _, _, _, _, _, _, _, _),
     FactA \= FactB.
 domain_contradiction_candidate_pair(FactA, FactB) :-
-    current_req(ReqA),
-    current_req(ReqB),
-    ReqA @< ReqB,
-    effective_req_predicate(ReqA, FactA, Namespace, Name, Args, _),
-    effective_req_predicate(ReqB, FactB, Namespace, Name, Args, _),
+    predicate_conflict_pair(_ReqA, _ReqB, FactA, FactB, _Namespace, _Name, _Args, _, _),
     FactA \= FactB.
 domain_contradiction_candidate_pair(FactA, FactB) :-
     decision_signatures(Signatures),
@@ -3023,7 +3250,32 @@ comparison_quantity(ValType, Value, Unit, Type, CanonValue, CanonUnit) :-
 
 %% predicate_schema(+FactId, -Namespace, -Name, -Arity, -ArgumentNames, -ArgumentTypes)
 % Read one project-local ontology predicate schema fact.
+% Finding the schemas of a predicate scans every fact, and contradiction
+% witnesses, decision signatures and key lookups ask for them per candidate,
+% so the schema rows are memoized per KB generation (outside RDF
+% transactions), in the order the scan finds them.
 predicate_schema(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes) :-
+    var(FactId),
+    \+ rdf_active_transaction(_),
+    !,
+    predicate_schema_rows(Rows),
+    member(schema(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes), Rows).
+predicate_schema(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes) :-
+    predicate_schema_scan(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes).
+
+predicate_schema_rows(Rows) :-
+    index_verification_key(predicate_schemas, Key),
+    (   kb_predicate_schema_memo(Key, Cached)
+    ->  Rows = Cached
+    ;   findall(schema(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes),
+                predicate_schema_scan(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes),
+                Computed),
+        retractall(kb_predicate_schema_memo(_, _)),
+        assertz(kb_predicate_schema_memo(Key, Computed)),
+        Rows = Computed
+    ).
+
+predicate_schema_scan(FactId, Namespace, Name, Arity, ArgumentNames, ArgumentTypes) :-
     kb_entity(FactId, fact, Props),
     memberchk(fact_kind=KindRaw, Props),
     normalize_term_atom(KindRaw, predicate_schema),

package/src/kb_consistency.pl

@@ -58,7 +58,22 @@ check_kb_consistency_unresolved(Violations) :-
         Unsorted),
     sort(Unsorted, Violations).
 
+% Both rules read the same verification, so it is produced once per KB
+% generation (recomputed inside an RDF transaction).
 consistency_verification(Bundle, Report) :-
+    (   \+ rdf_db:rdf_active_transaction(_),
+        kb:index_verification_key(kb_consistency, Key)
+    ->  (   kb:kb_consistency_memo(Key, Bundle0, Report0)
+        ->  Bundle = Bundle0,
+            Report = Report0
+        ;   compute_consistency_verification(Bundle, Report),
+            retractall(kb:kb_consistency_memo(_, _, _)),
+            assertz(kb:kb_consistency_memo(Key, Bundle, Report))
+        )
+    ;   compute_consistency_verification(Bundle, Report)
+    ).
+
+compute_consistency_verification(Bundle, Report) :-
     kb_certificates(Produced),
     with_output_to(string(Json), json_write_dict(current_output, Produced, [width(0)])),
     open_string(Json, Stream),