okf 1.11.0 → 1.13.0

data/lib/okf/bundle/reader.rb

@@ -12,10 +12,17 @@ module OKF
     # cannot open at all — is retained as an unparseable entry (carrying the
     # ParseError message or the errno, so §9.1 can report it) rather than dropped
     # or raised. That tolerance is the whole §9 best-effort promise: one bad file
-    # never breaks the rest, and this is the read every verb shares. Every read
-    # goes through Path.join_under! so a symlinked or crafted path cannot escape
-    # the bundle root — that guard still raises, because a path leaving the root
-    # is not a bad file, it is a bundle lying about its shape.
+    # never breaks the rest, and this is the read every verb shares.
+    #
+    # Containment is enforced twice, because the two ways out of the root are
+    # different. A crafted *path* (`..`, an absolute string) is caught lexically
+    # by Path.join_under!. A *symlink* whose name sits inside the root but whose
+    # target does not cannot be seen lexically — File.expand_path does not
+    # resolve links — so each file is also realpath-resolved and its real
+    # location checked against the real root before a byte is read. An escaping
+    # file joins the unparseable bucket rather than raising: a planted symlink is
+    # one bad file, and letting it take down the whole bundle read would hand any
+    # writer of a served directory a denial of service. §9.1 then names it.
     class Reader
       def self.read(dir)
         new(dir).read
@@ -32,9 +39,22 @@ module OKF
         reserved = []
         unparseable = []
 
-        markdown_paths.each do |path|
+        paths = markdown_paths
+        # Resolved once for the whole loop, but never at the cost of the
+        # best-effort promise: if the root itself has become unreadable since
+        # the glob, this stays nil and each file's own SafeRead call raises
+        # inside the per-file rescue below — one bad bundle degrades to
+        # unparseable entries, it does not crash the read every verb shares.
+        real_root = begin
+          File.realpath(@root) unless paths.empty?
+        rescue SystemCallError
+          nil
+        end
+
+        paths.each do |path|
           begin
-            content = File.read(Path.join_under!(@root, path), encoding: "UTF-8")
+            absolute = Path.join_under!(@root, path)
+            content = SafeRead.read!(@root, absolute, real_root: real_root)
             if Concept.reserved?(path)
               reserved << Entry.new(path: path, content: content)
             else
@@ -43,18 +63,21 @@ module OKF
             end
           rescue Markdown::Frontmatter::ParseError => e
             unparseable << Entry.new(path: path, content: content, error: e.message)
-          rescue SystemCallError => e
-            # A file that cannot be opened is one unusable file, not a broken
-            # bundle. Letting the errno out of here breaks "one bad file never
-            # breaks the rest" for every verb at once — the read is the one path
-            # they all share — and it breaks it in the worst way: a backtrace,
-            # under an exit code that claims the bundle is non-conformant. So it
-            # joins the same bucket a bad frontmatter block does, and §9.1 reports
-            # it naming the file and the errno.
+          rescue Path::Error, SystemCallError => e
+            # A file we cannot safely read is one unusable file, not a broken
+            # bundle: an errno on open, or a path that leaves the root — lexically
+            # (`..`, an absolute string) or through a symlink whose target escapes
+            # it. Letting either out of here breaks "one bad file never breaks the
+            # rest" for every verb at once — the read is the one path they all
+            # share — and in the worst way: a backtrace under an exit code that
+            # claims non-conformance, or a served bundle taken down by one planted
+            # symlink. So it joins the same bucket a bad frontmatter block does,
+            # and §9.1 reports it naming the file and the reason.
             #
             # Its content is "" rather than nil: unknown, but every analyzer reads
-            # it as text, and empty is the honest shape of a file we never saw —
-            # no links to resolve, no encoding to be invalid, nothing claimed.
+            # it as text, and empty is the honest shape of a file we never read —
+            # no links to resolve, nothing claimed, and for a symlink escape, none
+            # of the target's bytes.
             unparseable << Entry.new(path: path, content: "", error: e.message)
           end
         end

data/lib/okf/bundle/search.rb

@@ -279,7 +279,7 @@ module OKF
       end
 
       # Ranked match rows, catalog-style identity plus where the terms hit:
-      # [{ slug:, id:, title:, type:, dir:, area:, tags:, matched: [field, …], score:, snippet: }, …]
+      # [{ slug:, id:, title:, type:, dir:, top_dir:, tags:, matched: [field, …], score:, snippet: }, …]
       # ordered by score descending, then slug, then id. `slug` is present only
       # when searching across bundles. No terms means no matches.
       def results
@@ -367,7 +367,7 @@ module OKF
           title: (concept.title || concept.id).to_s,
           type: concept.type.to_s,
           dir: OKF.dir_of(concept.id),
-          area: area_of(concept.id),
+          top_dir: top_dir_of(concept.id),
           tags: Array(concept.tags).map(&:to_s),
           matched: matched,
           score: score.round(4),
@@ -421,9 +421,9 @@ module OKF
         end
       end
 
-      # A concept's top-level area, mirroring the catalog's definition. Deprecated
-      # in favour of OKF.dir_of, which keeps the levels this one throws away.
-      def area_of(id)
+      # A concept's top-level dir, mirroring the catalog's definition — the first
+      # path segment. OKF.dir_of keeps the levels this one rolls up.
+      def top_dir_of(id)
         id.include?("/") ? id.split("/").first : "(root)"
       end
     end

data/lib/okf/bundle/skeleton.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module OKF
+  class Bundle
+    # The graph reduced to what a reader can hold in their head. Pure — built from
+    # an OKF::Bundle::Graph's nodes and edges, does no I/O, and decides nothing
+    # about how any of it is drawn.
+    #
+    # A dense bundle is not dense in the way a hub-and-spoke picture is. Measured
+    # on a 47-concept bundle with 227 links: the top hub takes 13 inbound and the
+    # median takes 4, so there is no 80/20 to exploit — dropping two thirds of the
+    # concepts still leaves 53 edges. The density lives *between directories*:
+    # 173 of those 227 links (76%) cross a directory boundary, and they collapse
+    # into 50 directory-to-directory arcs of which the top ten carry half the mass.
+    # That is the reduction worth drawing, and it is why #arcs exists at all.
+    #
+    # ── the two things it produces, and who reads them ──
+    #
+    #   dirs + arcs   the reduction, as counts: one row per directory, one
+    #                 weighted arc per ordered pair. Printed by
+    #                 `okf graph --traffic`, with cohesion derived from them.
+    #   edges         every link with the cut it survives (`keep_at`). Nothing
+    #                 prints these — they are what lets the graph page lay a
+    #                 large bundle out on its strongest links first.
+    #
+    # Both are emitted *unthresholded*, and #suggested_cut names where to cut
+    # rather than cutting, so a caller narrows the picture without this class
+    # having to know what a picture is.
+    #
+    # Directories come off the concept *id* (OKF.dir_of), not the file path, so
+    # this agrees with #catalog, #hubs and the `--dir` filter. Bundle#directory_index
+    # groups by path instead — deliberately, since an index.md is a physical
+    # listing — so the two disagree for a concept whose frontmatter `id` moves it.
+    # This side follows the id because the edges do.
+    class Skeleton
+      attr_reader :dirs, :arcs, :edges
+
+      def self.build(bundle)
+        graph = bundle.graph(minimal: true)
+        ids = graph.nodes.map { |node| node[:id] }
+        pairs = graph.edges.map { |edge| [ edge[:source], edge[:target] ] }
+        dir_by_id = ids.map { |id| [ id, OKF.dir_of(id) ] }.to_h
+
+        new(
+          dirs: dirs_for(dir_by_id, pairs),
+          arcs: arcs_for(dir_by_id, pairs),
+          edges: edges_for(pairs, neighbours_for(ids, pairs))
+        )
+      end
+
+      # ── the cuts (see above) ──
+
+      def self.arcs_above(arcs, weight)
+        arcs.select { |arc| arc[:weight] >= weight }
+      end
+
+      def self.edges_within(edges, keep_at)
+        edges.select { |edge| edge[:keep_at] <= keep_at }
+      end
+
+      # ── the directories ──
+
+      # One row per directory that holds a concept, plus every ancestor up to the
+      # root, so the tree stays connected through a directory that holds nothing
+      # directly. `count` is direct and `subtree` is at-or-below — the same pair
+      # `okf dirs` prints, and for the same reason: a direct count alone cannot
+      # say where the mass is once the listing is cut off at a depth.
+      def self.dirs_for(dir_by_id, pairs)
+        direct = Hash.new(0)
+        dir_by_id.each_value { |dir| direct[dir] += 1 }
+        internal = Hash.new(0)
+        pairs.each do |source, target|
+          from = dir_by_id[source]
+          internal[from] += 1 if from == dir_by_id[target]
+        end
+
+        every_dir(direct.keys).map do |dir|
+          { dir: dir, parent: parent_of(dir), count: direct[dir],
+            subtree: direct.reduce(0) { |sum, (other, n)| under?(other, dir) ? sum + n : sum },
+            internal: internal[dir] }
+        end
+      end
+
+      # The cross-directory link mass, aggregated. Directed and never a self-arc:
+      # a directory's internal links are carried on its own row (`internal`),
+      # because an arc from a box to itself is a loop the eye has to untangle to
+      # learn a number the box could simply have carried.
+      def self.arcs_for(dir_by_id, pairs)
+        weights = Hash.new(0)
+        pairs.each do |source, target|
+          from = dir_by_id[source]
+          to = dir_by_id[target]
+          weights[[ from, to ]] += 1 unless from == to
+        end
+
+        weights.map { |(from, to), weight| { source: from, target: to, weight: weight } }
+               .sort_by { |arc| [ -arc[:weight], arc[:source], arc[:target] ] }
+      end
+
+      # ── where to cut ──
+
+      # The arc cut that leaves a readable picture, chosen from the *shape* of
+      # the bundle rather than fixed. A fixed cut cannot work: measured at
+      # weight 3 across ten bundles it left 2 arcs on one and 136 on another —
+      # too tight to be a picture at one end, no reduction at all at the other.
+      #
+      # What stays constant when a bundle grows is not the arc count but the
+      # arcs *per box*: a node-link diagram reads at roughly one to two edges
+      # per node regardless of size. So the target is 1.5 arcs per directory,
+      # and the cut is whatever weight delivers it — 22 arcs over 13 directories
+      # on one bundle, 191 over 95 on another, both about the same density.
+      #
+      # The floor of 8 is for the small end, where 1.5-per-box would cut a
+      # ten-arc bundle down to something that no longer shows how it is joined
+      # up. Ties are kept rather than broken, so the result is *at least* the
+      # target — the alternative is dropping one of two arcs that weigh the
+      # same, which is an arbitrary choice presented as a threshold.
+      def self.suggested_cut(arcs, dir_count)
+        target = [ (dir_count * 1.5).ceil, 8 ].max
+        return 1 if arcs.empty? || arcs.size <= target
+
+        arcs[target - 1][:weight]
+      end
+
+      # ── the edges, each with the cut it survives ──
+
+      # The local-degree sparsifier (Lindner et al.): every concept keeps its own
+      # most-connected neighbours, and an edge survives if *either* end kept it.
+      # The union is the point — it is what stops a sparsifier from stranding the
+      # quiet half of the bundle, which a global "drop the weakest edges" rule
+      # does immediately.
+      #
+      # Not the disparity filter, the usual name in this territory: that one reads
+      # an edge's weight against its endpoint's total, and every link here weighs
+      # exactly 1, which makes every proportion identical and the filter a coin
+      # toss. Weighted-graph tools do not transfer to an unweighted graph just
+      # because both are graphs.
+      #
+      # `keep_at` is the smallest cut (0–100) at which the edge appears, so the
+      # consumer's whole job is `keep_at <= n`. An edge to a node's single most
+      # connected neighbour keeps at 0 and is therefore never cut away: the
+      # skeleton always spans every linked concept.
+      def self.edges_for(pairs, neighbours)
+        order = ordered_neighbours(neighbours)
+        pairs.map do |source, target|
+          { source: source, target: target,
+            keep_at: [ keep_at(order, neighbours, source, target),
+                       keep_at(order, neighbours, target, source) ].min }
+        end
+      end
+
+      # Where `other` sits in `id`'s neighbours (1 = most connected), turned into
+      # the smallest cut that reaches it. Position 1 needs no budget at all; past
+      # that, a cut of n keeps ceil(degree ** n/100) neighbours, so the answer is
+      # the least n satisfying that — solved directly rather than searched, and
+      # pinned against the definition it came from in the unit test.
+      def self.keep_at(order, neighbours, id, other)
+        position = order[id].index(other) + 1
+        return 0 if position == 1
+
+        degree = neighbours[id].size
+        (100 * Math.log(position - 1) / Math.log(degree)).floor + 1
+      end
+
+      def self.ordered_neighbours(neighbours)
+        neighbours.each_with_object({}) do |(id, set), out|
+          out[id] = set.sort_by { |other| [ -neighbours[other].size, other ] }
+        end
+      end
+
+      # Undirected neighbour sets. A link is structure regardless of which end
+      # authored it, and a concept every page cites but that cites nothing back
+      # is as central as one that does the citing.
+      def self.neighbours_for(ids, pairs)
+        sets = ids.map { |id| [ id, Set.new ] }.to_h
+        pairs.each do |source, target|
+          sets[source] << target if sets.key?(source) && sets.key?(target)
+          sets[target] << source if sets.key?(source) && sets.key?(target)
+        end
+        sets
+      end
+
+      # ── path arithmetic (pure; the bundle root is ".") ──
+
+      def self.every_dir(dirs)
+        seen = {}
+        dirs.each do |dir|
+          current = dir
+          loop do
+            seen[current] = true
+            break if current == "."
+
+            current = parent_of(current)
+          end
+        end
+        seen.keys.sort_by { |dir| dir == "." ? "" : dir }
+      end
+
+      def self.parent_of(dir)
+        return nil if dir == "."
+
+        File.dirname(dir)
+      end
+
+      # At or below — the same rule `--dir` is answered against, so a row's
+      # `subtree` and what `--dir <that row>` returns can never disagree.
+      def self.under?(dir, ancestor)
+        return true if ancestor == "."
+
+        dir == ancestor || dir.start_with?("#{ancestor}/")
+      end
+
+      def initialize(dirs:, arcs:, edges:)
+        @dirs = dirs
+        @arcs = arcs
+        @edges = edges
+      end
+
+      def suggested_cut
+        @suggested_cut ||= self.class.suggested_cut(arcs, dirs.size)
+      end
+
+      # The `keep_at` of each edge in +edges+, in *its* order — so a caller
+      # holding an OKF::Bundle::Graph can line the two up. Matched on the pair
+      # rather than by index: both lists derive from the same graph and so are
+      # already parallel today, and an index-aligned read would go wrong in
+      # silence on the day one of them stops being built that way.
+      def cuts_for(graph_edges)
+        by_pair = edges.each_with_object({}) { |edge, out| out[[ edge[:source], edge[:target] ]] = edge[:keep_at] }
+        graph_edges.map { |edge| by_pair[[ edge[:source], edge[:target] ]] || 0 }
+      end
+
+      # What the drawn views read. `edges` stays out: it is per-link data whose
+      # only consumer needs it at boot, before any fetch could answer — so it
+      # rides inline with the graph instead (see OKF::Render::Graph#edge_cuts_json).
+      def to_h
+        { dirs: dirs, arcs: arcs, suggested_cut: suggested_cut }
+      end
+    end
+  end
+end

data/lib/okf/bundle.rb

@@ -95,6 +95,10 @@ module OKF
       Graph.build(self, minimal: minimal, body: body)
     end
 
+    def skeleton
+      Skeleton.build(self)
+    end
+
     # Rich per-concept metadata the catalog / files / stats consumers want but the
     # lean graph omits — the descriptive frontmatter fields plus in/out link degree
     # taken from the graph edges. Pure: derived from the concepts and their links,
@@ -119,28 +123,28 @@ module OKF
           status: concept.frontmatter["status"]&.to_s,
           backlog_ref: concept.frontmatter["backlog_ref"]&.to_s,
           dir: OKF.dir_of(id),
-          area: area_of(id),
+          top_dir: top_dir_of(id),
           links_out: out_degree[id],
           links_in: in_degree[id]
         }
       end.sort_by { |entry| entry[:id] }
     end
 
-    # Concepts ranked by inbound link degree, each with the areas its inbound
-    # links come from — the evidence for "is this hub well-homed?": a hub whose
-    # inbound majority is foreign to its own area is a move candidate, one with
-    # a single dominant foreign area already names its better home. Only
-    # concepts with at least one inbound link appear. Pure: derived from the
-    # graph edges. Shared by the `okf graph --hubs` view.
+    # Concepts ranked by inbound link degree, each with the top-level dirs its
+    # inbound links come from — the evidence for "is this hub well-homed?": a hub
+    # whose inbound majority is foreign to its own top-level dir is a move
+    # candidate, one with a single dominant foreign dir already names its better
+    # home. Only concepts with at least one inbound link appear. Pure: derived
+    # from the graph edges. Shared by the `okf graph --hubs` view.
     def hubs
       inbound = {}
       graph(minimal: true).edges.each do |edge|
-        (inbound[edge[:target]] ||= Hash.new(0))[area_of(edge[:source])] += 1
+        (inbound[edge[:target]] ||= Hash.new(0))[top_dir_of(edge[:source])] += 1
       end
 
       inbound.map do |id, sources|
-        by_area = sources.sort_by { |area, count| [ -count, area ] }.to_h
-        { id: id, area: area_of(id), inbound: by_area.values.reduce(0, :+), by_area: by_area }
+        by_top_dir = sources.sort_by { |top_dir, count| [ -count, top_dir ] }.to_h
+        { id: id, top_dir: top_dir_of(id), inbound: by_top_dir.values.reduce(0, :+), by_top_dir: by_top_dir }
       end.sort_by { |row| [ -row[:inbound], row[:id] ] }
     end
 
@@ -156,6 +160,18 @@ module OKF
     # `id` must not move a concept out of the directory it lives in. Pure: derived
     # from the concepts and the reserved index text, no disk. Shared by the
     # `okf index` view and the server's Index panel (/index).
+    # Every directory this bundle has — the same set #directory_index enumerates
+    # (concepts, an index.md or a log.md, plus every ancestor), without building
+    # the map. It is the answer to "does this bundle have a directory named X?",
+    # and the CLI needs exactly that to decide whether `--dir root` names a real
+    # directory or the bundle root. Reading it off #catalog instead is the same
+    # question asked of a smaller set, which is how the two views came to
+    # disagree about one bundle. Memoized: the model is immutable once read,
+    # and the resolvers above ask per invocation, not per bundle load.
+    def directories
+      @directories ||= directory_set(concepts.map { |concept| File.dirname(concept.path) }.uniq)
+    end
+
     def directory_index
       by_dir = concepts.group_by { |concept| File.dirname(concept.path) }
       dirs = directory_set(by_dir.keys)
@@ -189,17 +205,21 @@ module OKF
 
     private
 
-    # A concept's top-level area, derived from its id — the same derivation the
-    # catalog exposes, so every grouped view labels the bundle root "(root)".
-    def area_of(id)
+    # A concept's top-level dir, derived from its id — the first path segment, the
+    # same derivation the catalog exposes, so every grouped view labels the bundle
+    # root "(root)". OKF.dir_of keeps the levels this one rolls up.
+    def top_dir_of(id)
       id.include?("/") ? id.split("/").first : "(root)"
     end
 
-    # Every directory to show: those holding concepts or an index.md, plus each of
-    # their ancestors up to the root, so the subdir tree stays connected even when
-    # an intermediate directory holds nothing directly. Sorted with "." first.
+    # Every directory to show: those holding concepts, an index.md or a log.md,
+    # plus each of their ancestors up to the root, so the subdir tree stays
+    # connected even when an intermediate directory holds nothing directly. A
+    # scoped log counts because `okf log` reads it — a directory whose only file
+    # is its history still exists, and leaving it out is how the `root` alias
+    # beat a real `root/` for the second file kind in a row. Sorted "." first.
     def directory_set(concept_dirs)
-      seed = concept_dirs + index_files.map { |path| File.dirname(path) }
+      seed = concept_dirs + (index_files + log_files).map { |path| File.dirname(path) }
       dirs = {}
       seed.each do |dir|
         current = dir

data/lib/okf/cli/catalog.rb

@@ -2,8 +2,8 @@
 
 module OKF
   class CLI
-    # Every concept with its metadata, grouped by area. The widest of the read
-    # views, and the one the others narrow down from.
+    # Every concept with its metadata, grouped by top-level dir. The widest of the
+    # read views, and the one the others narrow down from.
     class Catalog < Command
       def self.id
         :catalog
@@ -15,7 +15,7 @@ module OKF
 
       def self.help_rows
         [
-          [ "catalog   <dir|@slug> [--json] [filters]", "list concepts with metadata, by area" ]
+          [ "catalog   <dir|@slug> [--json] [filters]", "list concepts with metadata, by top-level dir" ]
         ]
       end
 
@@ -33,7 +33,7 @@ module OKF
         folder = OKF::Bundle::Folder.load(dir)
         report_skipped(folder)
         entries = folder.catalog
-        selected = filter_entries(entries, options)
+        selected = filter_entries(entries, options, dir_scope(folder, options))
         return print_catalog_json(dir, selected, options) if options[:json]
 
         print_catalog(dir, selected, entries.size)
@@ -44,9 +44,9 @@ module OKF
 
       def print_catalog(dir, entries, total)
         @out.puts "Catalog — #{bundle_label(dir)} (#{counted(entries.size, total, "concept")})"
-        entries.group_by { |entry| entry[:area] }.sort_by(&:first).each do |area, group|
+        entries.group_by { |entry| entry[:top_dir] }.sort_by(&:first).each do |top_dir, group|
           @out.puts
-          @out.puts "  #{area == "(root)" ? "(root)" : "#{area}/"} (#{group.size})"
+          @out.puts "  #{top_dir == "(root)" ? "(root)" : "#{top_dir}/"} (#{group.size})"
           group.each do |entry|
             links = entry[:links_out] + entry[:links_in]
             meta = [ entry[:type], (links.positive? ? "↳#{links}" : nil), entry[:status] ].compact.join("  ·  ")