okf 1.8.0 → 1.10.0

data/lib/okf/bundle/search.rb

@@ -2,24 +2,97 @@
 
 module OKF
   class Bundle
-    # Deterministic text retrieval over an in-memory bundle — the browser page's
-    # search brought server-side and extended to bodies. Terms are ANDed: every
-    # term must hit at least one searched field, though not necessarily the same
-    # one. A term is a case-insensitive substring, or a Ruby regular expression
-    # with `regexp: true`. Matches rank by where they hit (a title hit outranks a
-    # body hit) and carry one bounded context snippet, so answering "which concept
-    # covers X?" costs a row, not a body read.
+    # Ranked text retrieval over one or more in-memory bundles. Terms are ANDed:
+    # every term must hit at least one searched field, though not necessarily the
+    # same one. Rows carry the fields each term hit, so a result stays explainable
+    # rather than being a bare relevance number.
     #
-    # Deliberately not fuzzy: the consuming agent is the fuzzy layer — synonyms
-    # and vocabulary drift are judgment over the index map, not string distance.
+    # This class is a *facade*. It owns everything that defines what a result is —
+    # the documents, the row and its key order, the snippet window, the final sort
+    # — and delegates only "which documents match, how well, and where" to an
+    # engine (Search::Index by default, Search::Scan for regexp). An engine that
+    # built its own rows could disagree about what a match is; this split makes
+    # that unrepresentable.
     #
-    # Pure — no disk, no stdio. The CLI's `okf search` and any embedding app share
-    # it: OKF::Bundle::Search.call(bundle, [ "dedup", "key" ]).
+    # Pure — no disk, no stdio. The CLI's `okf search` and any embedding app
+    # share it: OKF::Bundle::Search.call(bundle, [ "dedup", "key" ]).
     class Search
+      # Raised when the query needs something the engine cannot do — either the
+      # one that was named, or any that is available. Carries structured data
+      # rather than a finished sentence, because the shell says "--regexp" where
+      # the core says ":regexp"; the CLI formats it and exits 2.
+      class UnsupportedQuery < OKF::Error
+        attr_reader :missing, :engine
+
+        def initialize(missing, engine: nil)
+          @missing = missing
+          @engine = engine
+          super(build_message(missing, engine))
+        end
+
+        private
+
+        def build_message(missing, engine)
+          return "no search engine is available" if missing.empty?
+
+          offered = missing.map { |name| ":#{name}" }.join(", ")
+          engine.nil? ? "no available search engine offers #{offered}" : "engine #{engine} does not offer #{offered}"
+        end
+      end
+
+      # Raised when `--engine` names something that is not on offer. An engine
+      # registered but reporting `available? == false` is absent from the list for
+      # the same reason it is absent from routing: it cannot answer. A future
+      # addon whose native build failed will want a kinder message than this one.
+      class UnknownEngine < OKF::Error
+        attr_reader :name, :available
+
+        def initialize(name, available)
+          @name = name
+          @available = available
+          super("unknown search engine: #{name} (available: #{available.join(", ")})")
+        end
+      end
+
+      # The **declarable** vocabulary: what an engine may claim about itself.
+      # Frozen so an engine declaring `:regex` is refused at registration rather
+      # than silently never selected — a typo in an addon would otherwise present
+      # as "my engine is ignored".
+      #
+      # `:prefix` lives here and *not* in ROUTABLE on purpose. Nothing asks for
+      # its absence, so it selects nothing; what it does is document that this
+      # engine grows a term to the tokens it prefixes, which an FTS5 engine may
+      # not do by default. Declarative, and honest about being declarative.
+      CAPABILITIES = %i[regexp fuzzy prefix].freeze
+
+      # The **routable** subset: the capabilities a query can actually require,
+      # and therefore the only ones that pick an engine. Kept distinct from
+      # CAPABILITIES because a capability nothing selects on, filed among the ones
+      # that do, is documentation posing as code.
+      #
+      # Each entry is also the option name the facade hands an engine that
+      # declares it — see #engine_options, which is what keeps a meaningful
+      # option from reaching an engine that would quietly drop it.
+      ROUTABLE = %i[regexp fuzzy].freeze
+
+      # Chosen when the query requires nothing in particular, which is the
+      # overwhelming majority of searches.
+      #
+      # The scan, not the index, because a one-shot CLI builds an index, asks one
+      # question and exits — a build with a single query to amortize it over.
+      # Measured end to end: 3.00s vs 0.24s at 1,000 concepts, 0.83s vs 0.18s at
+      # 250, and the gap widens with the bundle. Raw-text matching also carries no
+      # tokenizer, so the terms that are glued to symbols and therefore
+      # unreachable by token (`minifts`, $OKF_HOME) stay findable by default.
+      #
+      # What it gives up is BM25+ ranking, reachable with `--engine index` — and
+      # that is also the engine the browser page runs, so the two rank alike only
+      # when the index is named. See .okf/design/search-engines.md.
+      DEFAULT_ENGINE = :scan
+
       # The searchable fields with their rank weight, strongest signal first.
-      # A concept's score sums the weights of the fields that matched; hitting a
-      # field twice does not stack. Tags match against the space-joined list,
-      # mirroring the server page's haystack.
+      # In the index engine these ride as MiniFTS per-field `boost`; the scan
+      # sums the weights of the fields that matched instead.
       WEIGHTS = {
         "title" => 5,
         "id" => 4,
@@ -38,53 +111,165 @@ module OKF
       # Characters of context kept on each side of the first matched term.
       SNIPPET_RADIUS = 44
 
-      def self.call(bundle, terms, fields: nil, regexp: false)
-        new(bundle, terms, fields: fields, regexp: regexp).results
+      # Edit distance as a fraction of term length, under `fuzzy: true` — the
+      # same 0.2 the browser page passes, so both forgive the same typos.
+      FUZZY_DISTANCE = 0.2
+
+      # The unique document key is "<slug>\0<id>": ids are only unique *within* a
+      # bundle, and a merge that collided two bundles' same-named concepts would
+      # silently drop one.
+      KEY_SEPARATOR = "\0"
+
+      # Append-only and idempotent by id: a second registration of an id already
+      # present is a no-op, so a double `require` cannot double the registry and
+      # an addon cannot quietly displace a built-in. Deliberately the same shape
+      # as the Linter's planned register hook — two extension points, one idiom.
+      def self.register(engine)
+        rogue = engine.capabilities - CAPABILITIES
+        raise ArgumentError, "unknown search capability: #{rogue.join(", ")}" unless rogue.empty?
+
+        @engines ||= []
+        @engines << engine unless @engines.any? { |registered| registered.id == engine.id }
+        engine
+      end
+
+      # A frozen snapshot in registration order. Frozen because the registry is
+      # only meant to grow through .register, where the vocabulary is checked.
+      def self.engines
+        (@engines ||= []).dup.freeze
+      end
+
+      # The router. Naming an engine is an override, not a hint: it is how a
+      # caller reaches semantics no capability flag asks for — `--engine scan`
+      # means "match raw text", which the flags cannot express because there is
+      # nothing to *require*. A named engine that cannot do what was also asked
+      # is an error rather than a silent fallback, since falling back would answer
+      # a different question than the one that was posed.
+      #
+      # Unnamed, the default engine leads, then registration order; the first
+      # available engine offering *every* required capability answers. Partition
+      # rather than sort_by, because sort_by is not stable and registration order
+      # is the tie-break.
+      def self.engine_for(required, engines: self.engines, name: nil)
+        available = engines.select(&:available?)
+        return named_engine(name, required, available) unless OKF.blank?(name)
+
+        default, rest = available.partition { |engine| engine.id == DEFAULT_ENGINE }
+        found = (default + rest).find { |engine| (required - engine.capabilities).empty? }
+        return found if found
+
+        raise UnsupportedQuery, required
+      end
+
+      def self.named_engine(name, required, available)
+        wanted = name.to_s.downcase
+        found = available.find { |engine| engine.id.to_s == wanted }
+        raise UnknownEngine.new(name, available.map(&:id)) if found.nil?
+
+        missing = required - found.capabilities
+        raise UnsupportedQuery.new(missing, engine: found.id) unless missing.empty?
+
+        found
+      end
+      private_class_method :named_engine
+
+      def self.call(bundle, terms, fields: nil, regexp: false, fuzzy: false, engine: nil, engines: nil)
+        new([ [ nil, bundle ] ], terms, fields: fields, regexp: regexp, fuzzy: fuzzy, engine: engine, engines: engines).results
+      end
+
+      # Several bundles as [ slug, bundle ] pairs, ranked into one list with every
+      # row labeled by its slug. They share **one** index on purpose: BM25 weighs a
+      # term by how rare it is in the corpus, so per-bundle indexes would score the
+      # same match differently depending on which bundle it came from. One index
+      # makes one corpus, and the merged ranking is comparable by construction.
+      def self.across(bundles, terms, fields: nil, regexp: false, fuzzy: false, engine: nil, engines: nil)
+        new(bundles, terms, fields: fields, regexp: regexp, fuzzy: fuzzy, engine: engine, engines: engines).results
       end
 
-      # Raises RegexpError on an invalid pattern with `regexp: true` — the caller
-      # owns turning that into a usage error.
-      def initialize(bundle, terms, fields: nil, regexp: false)
-        @bundle = bundle
-        raw = Array(terms).reject { |term| OKF.blank?(term) }
-        @matchers = raw.map { |term| regexp ? Regexp.new(term.to_s, Regexp::IGNORECASE) : term.to_s.downcase }
+      # Raises RegexpError on an invalid pattern with `regexp: true`, and
+      # UnsupportedQuery when no engine can answer — the caller owns turning
+      # either into a usage error. `engines:` overrides the registry, which is how
+      # the "nothing qualifies" path stays reachable without an addon installed.
+      def initialize(bundles, terms, fields: nil, regexp: false, fuzzy: false, engine: nil, engines: nil)
+        @bundles = bundles
+        @terms = Array(terms).reject { |term| OKF.blank?(term) }.map(&:to_s)
         @fields = fields.nil? || fields.empty? ? FIELDS : fields
+        @regexp = regexp
+        @fuzzy = fuzzy
+        @engine = engine
+        @engines = engines
+        @sources = {}
       end
 
       # Ranked match rows, catalog-style identity plus where the terms hit:
-      # [{ id:, title:, type:, area:, tags:, matched: [field, …], score:, snippet: }, …]
-      # ordered by score descending, then id. No terms means no matches.
+      # [{ slug:, id:, title:, type:, area:, tags:, matched: [field, …], score:, snippet: }, …]
+      # ordered by score descending, then slug, then id. `slug` is present only
+      # when searching across bundles. No terms means no matches.
       def results
-        return [] if @matchers.empty?
+        return [] if @terms.empty?
 
-        @bundle.concepts
-               .map { |concept| match(concept) }
-               .compact
-               .sort_by { |row| [ -row[:score], row[:id] ] }
+        chosen = engine
+        rows = chosen.call(documents, @terms, **engine_options(chosen)).map do |hit|
+          slug, concept = @sources[hit[:key]]
+          row(slug, concept, hit[:matched], hit[:score], hit[:terms])
+        end
+        rows.sort_by { |row| [ -row[:score], row[:slug].to_s, row[:id] ] }
       end
 
       private
 
-      def match(concept)
-        texts = searchable_texts(concept)
-        matched = matched_fields(texts)
-        return nil if matched.nil?
+      # The engine is chosen by what the query needs, not by a flag naming one.
+      # `-e` unambiguously means "regexp semantics", so it routes on its own and
+      # says nothing about it — there is no --engine flag to reconcile with.
+      def engine
+        Search.engine_for(required_capabilities, engines: @engines || Search.engines, name: @engine)
+      end
 
-        {
-          id: concept.id,
-          title: (concept.title || concept.id).to_s,
-          type: concept.type.to_s,
-          area: area_of(concept.id),
-          tags: Array(concept.tags).map(&:to_s),
-          matched: matched,
-          score: matched.map { |field| WEIGHTS[field] }.reduce(0, :+),
-          snippet: snippet(texts, matched)
-        }
+      # What the query requires, in the routable vocabulary. `:prefix` never
+      # appears: it is declarable, not routable — nothing asks for its absence.
+      def required_capabilities
+        ROUTABLE.select { |capability| requested[capability] }
+      end
+
+      # `fields:` always, plus exactly the routable options the chosen engine
+      # declared it understands.
+      #
+      # The facade used to hand every engine every option and trust it to ignore
+      # what it could not use. Routing makes that harmless in practice — a fuzzy
+      # query only ever reaches a :fuzzy engine — but "harmless because something
+      # else prevents it" is precisely how an option comes to be dropped in
+      # silence the day that something else changes. An engine now receives only
+      # what it can act on, so there is nothing left for it to ignore.
+      def engine_options(chosen)
+        options = { fields: @fields }
+        ROUTABLE.each do |capability|
+          options[capability] = requested[capability] if chosen.capabilities.include?(capability)
+        end
+        options
+      end
+
+      def requested
+        @requested ||= { regexp: @regexp, fuzzy: @fuzzy }
+      end
+
+      # Every concept as an indexable document, keyed uniquely across bundles.
+      # @sources keeps the way back, so the index stores no fields of its own.
+      def documents
+        docs = []
+        @bundles.each do |slug, bundle|
+          bundle.concepts.each do |concept|
+            key = "#{slug}#{KEY_SEPARATOR}#{concept.id}"
+            @sources[key] = [ slug, concept ]
+            docs << field_texts(concept).merge("key" => key)
+          end
+        end
+        docs
       end
 
-      # { field => original-case text } for the fields this search reads.
-      def searchable_texts(concept)
-        texts = {
+      # { field => original-case text } for every searchable field. The index reads
+      # all of them; `fields:` narrows the search, not the document.
+      def field_texts(concept)
+        {
           "id" => concept.id,
           "title" => concept.title.to_s,
           "type" => concept.type.to_s,
@@ -92,35 +277,46 @@ module OKF
           "tags" => Array(concept.tags).join(" "),
           "body" => concept.body
         }
-        texts.each_with_object({}) do |(field, text), acc|
-          acc[field] = text if @fields.include?(field)
-        end
-      end
-
-      # The union of fields any term hit, in WEIGHTS order — or nil when some term
-      # hit nothing (terms are ANDed).
-      def matched_fields(texts)
-        hits = @matchers.map do |matcher|
-          fields = texts.keys.select { |field| hit?(matcher, texts[field]) }
-          return nil if fields.empty?
-
-          fields
-        end
-        FIELDS.select { |field| hits.flatten.include?(field) }
       end
 
-      def hit?(matcher, text)
-        matcher.is_a?(Regexp) ? matcher.match?(text) : text.downcase.include?(matcher)
+      # `slug` leads the row so a merged result reads bundle-first, and drops
+      # entirely for a single bundle, which has no slug to carry.
+      def row(slug, concept, matched, score, terms)
+        texts = field_texts(concept)
+        built = {
+          slug: slug,
+          id: concept.id,
+          title: (concept.title || concept.id).to_s,
+          type: concept.type.to_s,
+          area: area_of(concept.id),
+          tags: Array(concept.tags).map(&:to_s),
+          matched: matched,
+          score: score.round(4),
+          snippet: snippet(texts, matched, terms)
+        }
+        built.delete(:slug) if slug.nil?
+        built
       end
 
       # One bounded context window around the first term that hit the strongest
       # snippet-worthy field; "" when the match needs no context (id/title/type/tags).
-      def snippet(texts, matched)
+      def snippet(texts, matched, terms)
         field = SNIPPET_FIELDS.find { |candidate| matched.include?(candidate) }
         return "" if field.nil?
 
-        matcher = @matchers.find { |candidate| hit?(candidate, texts[field]) }
-        context(texts[field], matcher)
+        matcher = snippet_matcher(texts[field], terms)
+        matcher.nil? ? "" : context(texts[field], matcher)
+      end
+
+      # What to point the window at: the first of the engine's reported matchers
+      # that this text actually contains. Engine-agnostic on purpose — the scan
+      # reports compiled patterns, the index reports lowercased document terms,
+      # and both are things `locate` can find again in the flattened text.
+      def snippet_matcher(text, terms)
+        down = text.downcase
+        Array(terms).find do |term|
+          term.is_a?(Regexp) ? term.match?(text) : down.include?(term)
+        end
       end
 
       def context(text, matcher)

data/lib/okf/bundle.rb

@@ -111,7 +111,7 @@ module OKF
         id = concept.id
         {
           id: id,
-          title: (concept.title || id).to_s,
+          title: OKF.blank?(concept.title) ? File.basename(id) : concept.title.to_s,
           type: concept.type.to_s,
           description: concept.description.to_s,
           tags: Array(concept.tags).map(&:to_s),
@@ -119,13 +119,31 @@ module OKF
           status: concept.frontmatter["status"]&.to_s,
           backlog_ref: concept.frontmatter["backlog_ref"]&.to_s,
           dir: File.dirname("#{id}.md"),
-          area: id.include?("/") ? id.split("/").first : "(root)",
+          area: area_of(id),
           links_out: out_degree[id],
           links_in: in_degree[id]
         }
       end.sort_by { |entry| entry[:id] }
     end
 
+    # Concepts ranked by inbound link degree, each with the areas its inbound
+    # links come from — the evidence for "is this hub well-homed?": a hub whose
+    # inbound majority is foreign to its own area is a move candidate, one with
+    # a single dominant foreign area already names its better home. Only
+    # concepts with at least one inbound link appear. Pure: derived from the
+    # graph edges. Shared by the `okf graph --hubs` view.
+    def hubs
+      inbound = {}
+      graph(minimal: true).edges.each do |edge|
+        (inbound[edge[:target]] ||= Hash.new(0))[area_of(edge[:source])] += 1
+      end
+
+      inbound.map do |id, sources|
+        by_area = sources.sort_by { |area, count| [ -count, area ] }.to_h
+        { id: id, area: area_of(id), inbound: by_area.values.reduce(0, :+), by_area: by_area }
+      end.sort_by { |row| [ -row[:inbound], row[:id] ] }
+    end
+
     # The progressive-disclosure map (spec §6): one entry per directory that holds
     # concepts or carries an index.md, sorted with the root (".") first. Each entry
     # gives the authored index body (frontmatter stripped) when an index.md is
@@ -159,7 +177,7 @@ module OKF
           listing: here.map do |concept|
             {
               id: concept.id,
-              title: (concept.title || concept.id).to_s,
+              title: OKF.blank?(concept.title) ? File.basename(concept.id) : concept.title.to_s,
               description: concept.description.to_s,
               type: concept.type.to_s,
               tags: Array(concept.tags).map(&:to_s)
@@ -171,6 +189,12 @@ module OKF
 
     private
 
+    # A concept's top-level area, derived from its id — the same derivation the
+    # catalog exposes, so every grouped view labels the bundle root "(root)".
+    def area_of(id)
+      id.include?("/") ? id.split("/").first : "(root)"
+    end
+
     # Every directory to show: those holding concepts or an index.md, plus each of
     # their ancestors up to the root, so the subdir tree stays connected even when
     # an intermediate directory holds nothing directly. Sorted with "." first.

data/lib/okf/cli/catalog.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module OKF
+  class CLI
+    # Every concept with its metadata, grouped by area. The widest of the read
+    # views, and the one the others narrow down from.
+    class Catalog < Command
+      def self.id
+        :catalog
+      end
+
+      def self.group
+        :read
+      end
+
+      def self.help_rows
+        [
+          [ "catalog   <dir|@slug> [--json] [filters]", "list concepts with metadata, by area" ]
+        ]
+      end
+
+      def call(argv)
+        options = { json: false }
+        parser = OptionParser.new do |o|
+          o.banner = "Usage: okf catalog <dir|@slug> [--type T] [--area A] [--tag T] [--json]"
+          json_flags(o, options, "emit the catalog as JSON")
+          projection_flags(o, options)
+          filter_flags(o, options, :type, :area, :tag)
+          help_flag(o)
+        end
+        dir = positional_dir(parser, argv) or return 2
+
+        folder = OKF::Bundle::Folder.load(dir)
+        report_skipped(folder)
+        entries = folder.catalog
+        selected = filter_entries(entries, options)
+        return print_catalog_json(dir, selected, options) if options[:json]
+
+        print_catalog(dir, selected, entries.size)
+        0
+      end
+
+      private
+
+      def print_catalog(dir, entries, total)
+        @out.puts "Catalog — #{bundle_label(dir)} (#{counted(entries.size, total, "concept")})"
+        entries.group_by { |entry| entry[:area] }.sort_by(&:first).each do |area, group|
+          @out.puts
+          @out.puts "  #{area == "(root)" ? "(root)" : "#{area}/"} (#{group.size})"
+          group.each do |entry|
+            links = entry[:links_out] + entry[:links_in]
+            meta = [ entry[:type], (links.positive? ? "↳#{links}" : nil), entry[:status] ].compact.join("  ·  ")
+            @out.puts "    #{entry[:title]}  ·  #{meta}"
+            @out.puts "      #{truncate(entry[:description], 92)}" unless entry[:description].empty?
+          end
+        end
+      end
+
+      def print_catalog_json(dir, entries, options)
+        emit_list_json(dir, "concepts", entries.map { |entry| stringify(entry) }, options)
+      end
+    end
+
+    register(Catalog)
+  end
+end