okf 1.12.0 → 2.0.0

data/lib/okf/bundle/reader.rb

@@ -10,12 +10,19 @@ module OKF
     # index.md/log.md is kept as raw text (its structure is validated as text), and
     # a file the reader cannot use — frontmatter that does not parse, or a file it
     # cannot open at all — is retained as an unparseable entry (carrying the
-    # ParseError message or the errno, so §9.1 can report it) rather than dropped
-    # or raised. That tolerance is the whole §9 best-effort promise: one bad file
-    # never breaks the rest, and this is the read every verb shares. Every read
-    # goes through Path.join_under! so a symlinked or crafted path cannot escape
-    # the bundle root — that guard still raises, because a path leaving the root
-    # is not a bad file, it is a bundle lying about its shape.
+    # ParseError message or the errno, so §11 condition 1 can report it) rather than dropped
+    # or raised. That tolerance is the whole §11 best-effort promise: one bad file
+    # never breaks the rest, and this is the read every verb shares.
+    #
+    # Containment is enforced twice, because the two ways out of the root are
+    # different. A crafted *path* (`..`, an absolute string) is caught lexically
+    # by Path.join_under!. A *symlink* whose name sits inside the root but whose
+    # target does not cannot be seen lexically — File.expand_path does not
+    # resolve links — so each file is also realpath-resolved and its real
+    # location checked against the real root before a byte is read. An escaping
+    # file joins the unparseable bucket rather than raising: a planted symlink is
+    # one bad file, and letting it take down the whole bundle read would hand any
+    # writer of a served directory a denial of service. §11 condition 1 then names it.
     class Reader
       def self.read(dir)
         new(dir).read
@@ -32,9 +39,22 @@ module OKF
         reserved = []
         unparseable = []
 
-        markdown_paths.each do |path|
+        paths = markdown_paths
+        # Resolved once for the whole loop, but never at the cost of the
+        # best-effort promise: if the root itself has become unreadable since
+        # the glob, this stays nil and each file's own SafeRead call raises
+        # inside the per-file rescue below — one bad bundle degrades to
+        # unparseable entries, it does not crash the read every verb shares.
+        real_root = begin
+          File.realpath(@root) unless paths.empty?
+        rescue SystemCallError
+          nil
+        end
+
+        paths.each do |path|
           begin
-            content = File.read(Path.join_under!(@root, path), encoding: "UTF-8")
+            absolute = Path.join_under!(@root, path)
+            content = SafeRead.read!(@root, absolute, real_root: real_root)
             if Concept.reserved?(path)
               reserved << Entry.new(path: path, content: content)
             else
@@ -43,18 +63,21 @@ module OKF
             end
           rescue Markdown::Frontmatter::ParseError => e
             unparseable << Entry.new(path: path, content: content, error: e.message)
-          rescue SystemCallError => e
-            # A file that cannot be opened is one unusable file, not a broken
-            # bundle. Letting the errno out of here breaks "one bad file never
-            # breaks the rest" for every verb at once — the read is the one path
-            # they all share — and it breaks it in the worst way: a backtrace,
-            # under an exit code that claims the bundle is non-conformant. So it
-            # joins the same bucket a bad frontmatter block does, and §9.1 reports
-            # it naming the file and the errno.
+          rescue Path::Error, SystemCallError => e
+            # A file we cannot safely read is one unusable file, not a broken
+            # bundle: an errno on open, or a path that leaves the root — lexically
+            # (`..`, an absolute string) or through a symlink whose target escapes
+            # it. Letting either out of here breaks "one bad file never breaks the
+            # rest" for every verb at once — the read is the one path they all
+            # share — and in the worst way: a backtrace under an exit code that
+            # claims non-conformance, or a served bundle taken down by one planted
+            # symlink. So it joins the same bucket a bad frontmatter block does,
+            # and §11 condition 1 reports it naming the file and the reason.
             #
             # Its content is "" rather than nil: unknown, but every analyzer reads
-            # it as text, and empty is the honest shape of a file we never saw —
-            # no links to resolve, no encoding to be invalid, nothing claimed.
+            # it as text, and empty is the honest shape of a file we never read —
+            # no links to resolve, nothing claimed, and for a symlink escape, none
+            # of the target's bytes.
             unparseable << Entry.new(path: path, content: "", error: e.message)
           end
         end
@@ -64,6 +87,12 @@ module OKF
 
       private
 
+      # Dir.glob's default (no FNM_DOTMATCH) excludes hidden files and every
+      # path under a hidden directory — kept deliberately, as the Unix
+      # convention it is: a bundle read from a project root must not pull in
+      # an installed skill under .claude/ or templates under .github/ as
+      # concepts. §3's taxonomy is read as covering the visible tree; the
+      # exclusion is documented in authoring.md and pinned by a reader test.
       def markdown_paths
         return [] unless Dir.exist?(@root)
 

data/lib/okf/bundle/references.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "set"
+
+module OKF
+  class Bundle
+    # The §6.3 inventory, pure: every file under `references/` — handed in by
+    # the shell as +files+, because the reader models concepts and this model
+    # performs no disk access — with which concepts cite each one through the
+    # §6.2 path-valued fields (resource, sources[].resource, computation,
+    # executor.resource, attester.resource), plus the pointers into
+    # `references/` that resolve to nothing.
+    #
+    # The bare-path trap is what the dangling list exists to surface: §6.2
+    # resolves a bare path relative to the concept, so `references/x` works
+    # from the bundle root and silently misses one directory down. When the
+    # missed spelling would have hit with a leading slash, the entry says so.
+    #
+    # Scope is the folder, by design. §6.3 is "a naming convention, not a
+    # requirement", and a computation stored beside its concept is legal —
+    # and deliberately not this inventory's to list.
+    class References
+      def self.build(bundle, files:)
+        new(bundle, files: files)
+      end
+
+      # Files under references/, sorted by path, each
+      # { path:, dir:, kind:, referenced_by: [ { id:, field: } ] } — kind is
+      # "concept" when the file is a parsed concept (§6.3 allows both), "file"
+      # otherwise. referenced_by is ordered by citing concept id.
+      attr_reader :entries
+
+      # Pointers into references/ that resolve to no file, ordered by concept
+      # id then field position, each { id:, field:, raw:, resolved:, hint: } —
+      # hint names the leading-slash fix when the bare spelling exists at the
+      # root, nil otherwise.
+      attr_reader :dangling
+
+      def initialize(bundle, files:)
+        @bundle = bundle
+        @files = files.sort
+        @existing = @files.to_set
+        build!
+      end
+
+      private
+
+      def build!
+        concept_paths = @bundle.concepts.to_set(&:path)
+        cited = {}
+        @dangling = []
+
+        @bundle.concepts.sort_by(&:id).each do |concept|
+          path_fields(concept).each do |field, raw|
+            target = Markdown::Links.resolve_path(raw, from: concept.path, bundle: @bundle.root)
+            next if target.nil? # external, empty, or a directory — not a file pointer
+
+            if @existing.include?(target)
+              (cited[target] ||= []) << { id: concept.id, field: field }
+            elsif under_references?(target) || bare_references?(raw)
+              @dangling << dangling_row(concept, field, raw, target)
+            end
+          end
+        end
+
+        @entries = @files.map do |path|
+          { path: path, dir: File.dirname(path),
+            kind: concept_paths.include?(path) ? "concept" : "file",
+            referenced_by: cited[path] || [] }
+        end
+      end
+
+      # Every §6.2 path-valued field this concept carries, as
+      # [ field-label, raw ] pairs. sources includes §13.1's lifted Citations —
+      # when the fallback is a concept's provenance, its pointers are too.
+      def path_fields(concept)
+        fields = [
+          [ "resource", concept.frontmatter["resource"] ],
+          [ "computation", concept.computation ],
+          [ "executor.resource", concept.executor && concept.executor["resource"] ],
+          [ "attester.resource", concept.attester && concept.attester["resource"] ]
+        ]
+        concept.sources.each_with_index do |source, index|
+          fields << [ "sources[#{index}].resource", source["resource"] ]
+        end
+        fields.reject { |_field, raw| OKF.blank?(raw) }
+      end
+
+      def under_references?(path)
+        path.to_s.start_with?("references/")
+      end
+
+      # The bare spelling of the trap: a raw that *names* references/ without
+      # the leading slash, which §6.2 resolves relative to the concept. From a
+      # subdirectory the resolved path leaves references/, so the target check
+      # above no longer sees it — the raw is what still does.
+      def bare_references?(raw)
+        raw.to_s.split("#", 2).first.to_s.start_with?("references/")
+      end
+
+      def dangling_row(concept, field, raw, target)
+        bare = raw.to_s.split("#", 2).first.to_s
+        hint = nil
+        if !bare.start_with?("/") && bare != target && @existing.include?(bare)
+          hint = "/#{bare} exists — missing leading slash?"
+        end
+        { id: concept.id, field: field, raw: raw.to_s, resolved: target, hint: hint }
+      end
+    end
+  end
+end

data/lib/okf/bundle/row_filter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module OKF
+  class Bundle
+    # The one catalog-row predicate every narrowing surface shares — the CLI's
+    # filter flags, the MCP shell's catalog filters, and its search narrowing
+    # select rows by these rules and no other copy of them. The seam earned the
+    # module: the predicate was hand-spelled in three shells with only the leaf
+    # folds shared, and the copies diverged three recorded times (a raw status
+    # compare answering the opposite of `--status stable`, "" meaning no-filter
+    # on one tool and match-nothing on the next, the root's spellings).
+    #
+    # Callers hand it *normalized* wants — nil means "no filter", and `dir`
+    # arrives as a folded base path ("." for the bundle root) — because
+    # argument spelling is each shell's own: the CLI folds the `root` alias
+    # against the bundle's real directories, the MCP shell maps ""/"/" onto
+    # ".", and neither belongs here. Pure, like everything beside it.
+    module RowFilter
+      module_function
+
+      def matches?(row, type: nil, dir: nil, tag: nil, status: nil, trust: nil)
+        (type.nil? || fold(row[:type]) == fold(type)) &&
+          (dir.nil? || under_dir?(row[:dir], dir)) &&
+          (tag.nil? || Array(row[:tags]).any? { |value| fold(value) == fold(tag) }) &&
+          (status.nil? || Concept.effective_status(row[:status]) == Concept.fold_status(status)) &&
+          (trust.nil? || fold(row[:trust]) == Concept.fold_tier(trust))
+      end
+
+      # The one rule `--dir` is built on: a dir names itself and everything
+      # beneath it; "." is a prefix of nothing, so the root selects only what
+      # lives directly in it.
+      def under_dir?(entry_dir, base)
+        entry = fold(entry_dir)
+        path = fold(base)
+        entry == path || entry.start_with?("#{path}/")
+      end
+
+      def fold(value)
+        value.to_s.downcase
+      end
+
+      # The row-shaped spelling of Concept.shows_trust? — "is this row's tier one
+      # a surface should claim?". It lives beside #matches? because narrowing is
+      # one of its three consumers: the trust facet's gate, its counts and its
+      # narrowing all read this, so the facet describes exactly the rows wearing a
+      # tier. `generated` is the catalog's raw declared-key boolean, which is what
+      # tells a concept that opted into §5 from one that said nothing at all.
+      def shows_trust?(row)
+        Concept.shows_trust?(row[:trust], row[:generated])
+      end
+    end
+  end
+end

data/lib/okf/bundle/search.rb

@@ -99,14 +99,29 @@ module OKF
         "tags" => 3,
         "type" => 2,
         "description" => 2,
+        # A regression fix rather than a feature: in v0.1 a citation's text
+        # lived in the body and was searchable at weight 1. After a bundle
+        # migrates it lives in frontmatter, so without this a migrated bundle
+        # silently loses the hit entirely. What this restores is *recall* at
+        # the same weight — not an identical total. A v0.1 concept keeps
+        # matching `body` too, because the text really is body prose there, so
+        # it scores one higher than its migrated twin; that divergence is
+        # deliberate and pinned by cli_twins_test.rb ("migrating moves a
+        # source-only hit's snippet from body text to source text"). Suppressing
+        # it would mean lying about `--in body` on a bundle whose body does
+        # contain the words.
+        "sources" => 1,
         "body" => 1
       }.freeze
 
       FIELDS = WEIGHTS.keys.freeze
 
       # Fields whose match is only meaningful with surrounding context. The other
-      # fields already appear whole on the result row.
-      SNIPPET_FIELDS = %w[description body].freeze
+      # fields already appear whole on the result row. `sources` is here because
+      # indexed-but-un-snippeted would degrade a consumer's evidence line to a
+      # bare id list: after migration the snippet moves from body text to source
+      # text, it does not vanish.
+      SNIPPET_FIELDS = %w[description body sources].freeze
 
       # Characters of context kept on each side of the first matched term.
       SNIPPET_RADIUS = 44
@@ -195,6 +210,9 @@ module OKF
           "type" => concept.type.to_s,
           "description" => concept.description.to_s,
           "tags" => Array(concept.tags).join(" "),
+          # Titles and resources together: a source is findable by what it is
+          # called and by where it lives, which is how the body list read.
+          "sources" => concept.sources.flat_map { |source| [ source["title"], source["resource"] ] }.compact.join(" "),
           "body" => concept.body
         }
       end

data/lib/okf/bundle/validator/result.rb

@@ -3,7 +3,7 @@
 module OKF
   class Bundle
     class Validator
-      # The outcome of a §9 conformance check (see OKF::Bundle::Validator): hard `errors`,
+      # The outcome of a §11 conformance check (see OKF::Bundle::Validator): hard `errors`,
       # soft `warnings`, and file `counts`. Conformant iff there are no errors.
       class Result
         attr_reader :errors, :warnings, :counts
@@ -22,8 +22,11 @@ module OKF
           errors << { path: path, message: message }
         end
 
-        def add_warning(path, message)
-          warnings << { path: path, message: message }
+        # `check:` (a stable id) and `source:` (:spec | :convention) make a
+        # warning machine-readable; errors keep their exact two-key shape —
+        # consumers already read it, so new keys land on warnings only.
+        def add_warning(path, message, check: nil, source: nil)
+          warnings << { path: path, message: message, check: check, source: source }
         end
 
         def count(kind)