okf 1.7.0 → 1.9.0

data/lib/okf/bundle/reader.rb

@@ -8,10 +8,14 @@ module OKF
     #
     # It parses eagerly: each concept file becomes an OKF::Concept, each
     # index.md/log.md is kept as raw text (its structure is validated as text), and
-    # a concept file whose frontmatter does not parse is retained as an unparseable
-    # entry (carrying the ParseError message, so §9.1 can report it) rather than
-    # dropped or raised. Every read goes through Path.join_under! so a
-    # symlinked or crafted path cannot escape the bundle root.
+    # a file the reader cannot use — frontmatter that does not parse, or a file it
+    # cannot open at all — is retained as an unparseable entry (carrying the
+    # ParseError message or the errno, so §9.1 can report it) rather than dropped
+    # or raised. That tolerance is the whole §9 best-effort promise: one bad file
+    # never breaks the rest, and this is the read every verb shares. Every read
+    # goes through Path.join_under! so a symlinked or crafted path cannot escape
+    # the bundle root — that guard still raises, because a path leaving the root
+    # is not a bad file, it is a bundle lying about its shape.
     class Reader
       def self.read(dir)
         new(dir).read
@@ -39,6 +43,19 @@ module OKF
             end
           rescue Markdown::Frontmatter::ParseError => e
             unparseable << Entry.new(path: path, content: content, error: e.message)
+          rescue SystemCallError => e
+            # A file that cannot be opened is one unusable file, not a broken
+            # bundle. Letting the errno out of here breaks "one bad file never
+            # breaks the rest" for every verb at once — the read is the one path
+            # they all share — and it breaks it in the worst way: a backtrace,
+            # under an exit code that claims the bundle is non-conformant. So it
+            # joins the same bucket a bad frontmatter block does, and §9.1 reports
+            # it naming the file and the errno.
+            #
+            # Its content is "" rather than nil: unknown, but every analyzer reads
+            # it as text, and empty is the honest shape of a file we never saw —
+            # no links to resolve, no encoding to be invalid, nothing claimed.
+            unparseable << Entry.new(path: path, content: "", error: e.message)
           end
         end
 

data/lib/okf/bundle/search/index.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "minifts"
+require "okf/bundle/search"
+
+module OKF
+  class Bundle
+    class Search
+      # The default engine: a MiniFTS full-text index — the same engine, and the
+      # same BM25+ arithmetic, the browser page already runs as MiniSearch, so a
+      # Ruby-built index and the page's rank identically.
+      #
+      # Matching is by *token*: a term matches a whole word or a word it prefixes
+      # ("dedup" reaches "deduplication"), and `fuzzy:` opts into typo tolerance.
+      # The index is built per call — see .okf/capabilities/search.md for why that
+      # ceiling stands and what lifts it.
+      module Index
+        CAPABILITIES = %i[fuzzy prefix].freeze
+
+        class << self
+          def id
+            :index
+          end
+
+          def capabilities
+            CAPABILITIES
+          end
+
+          # minifts is a hard runtime dependency with no native extension, so it
+          # is here whenever the gem is. An addon backed by a native build is the
+          # case this predicate exists for.
+          def available?
+            true
+          end
+
+          # `fields:` narrows where a term may hit, so a field the caller excluded
+          # can neither match nor be credited. The hit's `terms` are MiniFTS's
+          # matched *document* terms — already lowercased, and present in the text
+          # verbatim even when the query only prefixed them.
+          def call(documents, terms, fields:, fuzzy: false, **_options)
+            index = MiniFTS.new(fields: FIELDS, id_field: "key")
+            index.add_all(documents)
+
+            options = { combine_with: "AND", prefix: true, boost: WEIGHTS, fields: fields }
+            options[:fuzzy] = FUZZY_DISTANCE if fuzzy
+
+            index.search(terms.join(" "), options).map do |hit|
+              { key: hit[:id], matched: matched_in(hit), score: hit[:score], terms: hit[:terms] }
+            end
+          end
+
+          private
+
+          # The union of fields any term hit, in WEIGHTS order. MiniFTS reports it
+          # per query term as { term => [field, …] }.
+          def matched_in(hit)
+            FIELDS.select { |field| hit[:match].any? { |_term, found| found.include?(field) } }
+          end
+        end
+
+        Search.register(self)
+      end
+    end
+  end
+end

data/lib/okf/bundle/search/scan.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "okf/bundle/search"
+
+module OKF
+  class Bundle
+    class Search
+      # The linear engine: terms matched against raw field text, one document at a
+      # time. No index, so nothing is tokenized and nothing is normalized — a
+      # phrase stays a phrase, `7.2.0` stays one string, and an infix matches.
+      # That is the exactness a token index gives up, and the reason this engine
+      # survived the swap.
+      #
+      # Two readings of a term, and the engine is the *raw text* half of the split
+      # rather than the regexp half:
+      #
+      #   `regexp: false` — literal substring, which is what this engine did
+      #                     before the index landed, and what `--engine scan`
+      #                     restores. Terms are escaped, so `7.2.0` does not
+      #                     match `7x2y0` and `[draft]` is not a character class.
+      #   `regexp: true`  — the term is a pattern, opted into with `-e`.
+      #
+      # Conflating the two would make choosing the engine silently change what the
+      # terms mean, and turn an ordinary term like `review (pending` into exit 2.
+      #
+      # Scoring is the summed weight of the fields that matched: absolute, and so
+      # comparable across bundles without a corpus to normalize against.
+      module Scan
+        CAPABILITIES = %i[regexp].freeze
+
+        class << self
+          def id
+            :scan
+          end
+
+          def capabilities
+            CAPABILITIES
+          end
+
+          # No backing store to fail: the engine is Regexp and Enumerable.
+          def available?
+            true
+          end
+
+          # Raises RegexpError on an invalid pattern under `regexp: true` — the
+          # caller owns turning that into a usage error. A literal term cannot
+          # raise, because it is escaped before it is compiled. The hit's `terms`
+          # are the compiled patterns, which is what the facade points its snippet
+          # window at.
+          def call(documents, terms, fields:, regexp: false, **_options)
+            patterns = terms.map do |term|
+              Regexp.new(regexp ? term : Regexp.escape(term), Regexp::IGNORECASE)
+            end
+
+            hits = []
+            documents.each do |document|
+              matched = matched_fields(document, patterns, fields)
+              next if matched.nil?
+
+              hits << {
+                key: document["key"],
+                matched: matched,
+                score: matched.map { |field| WEIGHTS[field] }.reduce(0, :+),
+                terms: patterns
+              }
+            end
+            hits
+          end
+
+          private
+
+          # The union of fields any pattern hit, in WEIGHTS order — or nil when
+          # some pattern hit nothing (terms are ANDed).
+          def matched_fields(document, patterns, fields)
+            hits = patterns.map do |pattern|
+              found = fields.select { |field| pattern.match?(document[field]) }
+              return nil if found.empty?
+
+              found
+            end
+            FIELDS.select { |field| hits.any? { |found| found.include?(field) } }
+          end
+        end
+
+        Search.register(self)
+      end
+    end
+  end
+end

data/lib/okf/bundle/search.rb

@@ -2,24 +2,97 @@
 
 module OKF
   class Bundle
-    # Deterministic text retrieval over an in-memory bundle — the browser page's
-    # search brought server-side and extended to bodies. Terms are ANDed: every
-    # term must hit at least one searched field, though not necessarily the same
-    # one. A term is a case-insensitive substring, or a Ruby regular expression
-    # with `regexp: true`. Matches rank by where they hit (a title hit outranks a
-    # body hit) and carry one bounded context snippet, so answering "which concept
-    # covers X?" costs a row, not a body read.
+    # Ranked text retrieval over one or more in-memory bundles. Terms are ANDed:
+    # every term must hit at least one searched field, though not necessarily the
+    # same one. Rows carry the fields each term hit, so a result stays explainable
+    # rather than being a bare relevance number.
     #
-    # Deliberately not fuzzy: the consuming agent is the fuzzy layer — synonyms
-    # and vocabulary drift are judgment over the index map, not string distance.
+    # This class is a *facade*. It owns everything that defines what a result is —
+    # the documents, the row and its key order, the snippet window, the final sort
+    # — and delegates only "which documents match, how well, and where" to an
+    # engine (Search::Index by default, Search::Scan for regexp). An engine that
+    # built its own rows could disagree about what a match is; this split makes
+    # that unrepresentable.
     #
-    # Pure — no disk, no stdio. The CLI's `okf search` and any embedding app share
-    # it: OKF::Bundle::Search.call(bundle, [ "dedup", "key" ]).
+    # Pure — no disk, no stdio. The CLI's `okf search` and any embedding app
+    # share it: OKF::Bundle::Search.call(bundle, [ "dedup", "key" ]).
     class Search
+      # Raised when the query needs something the engine cannot do — either the
+      # one that was named, or any that is available. Carries structured data
+      # rather than a finished sentence, because the shell says "--regexp" where
+      # the core says ":regexp"; the CLI formats it and exits 2.
+      class UnsupportedQuery < OKF::Error
+        attr_reader :missing, :engine
+
+        def initialize(missing, engine: nil)
+          @missing = missing
+          @engine = engine
+          super(build_message(missing, engine))
+        end
+
+        private
+
+        def build_message(missing, engine)
+          return "no search engine is available" if missing.empty?
+
+          offered = missing.map { |name| ":#{name}" }.join(", ")
+          engine.nil? ? "no available search engine offers #{offered}" : "engine #{engine} does not offer #{offered}"
+        end
+      end
+
+      # Raised when `--engine` names something that is not on offer. An engine
+      # registered but reporting `available? == false` is absent from the list for
+      # the same reason it is absent from routing: it cannot answer. A future
+      # addon whose native build failed will want a kinder message than this one.
+      class UnknownEngine < OKF::Error
+        attr_reader :name, :available
+
+        def initialize(name, available)
+          @name = name
+          @available = available
+          super("unknown search engine: #{name} (available: #{available.join(", ")})")
+        end
+      end
+
+      # The **declarable** vocabulary: what an engine may claim about itself.
+      # Frozen so an engine declaring `:regex` is refused at registration rather
+      # than silently never selected — a typo in an addon would otherwise present
+      # as "my engine is ignored".
+      #
+      # `:prefix` lives here and *not* in ROUTABLE on purpose. Nothing asks for
+      # its absence, so it selects nothing; what it does is document that this
+      # engine grows a term to the tokens it prefixes, which an FTS5 engine may
+      # not do by default. Declarative, and honest about being declarative.
+      CAPABILITIES = %i[regexp fuzzy prefix].freeze
+
+      # The **routable** subset: the capabilities a query can actually require,
+      # and therefore the only ones that pick an engine. Kept distinct from
+      # CAPABILITIES because a capability nothing selects on, filed among the ones
+      # that do, is documentation posing as code.
+      #
+      # Each entry is also the option name the facade hands an engine that
+      # declares it — see #engine_options, which is what keeps a meaningful
+      # option from reaching an engine that would quietly drop it.
+      ROUTABLE = %i[regexp fuzzy].freeze
+
+      # Chosen when the query requires nothing in particular, which is the
+      # overwhelming majority of searches.
+      #
+      # The scan, not the index, because a one-shot CLI builds an index, asks one
+      # question and exits — a build with a single query to amortize it over.
+      # Measured end to end: 3.00s vs 0.24s at 1,000 concepts, 0.83s vs 0.18s at
+      # 250, and the gap widens with the bundle. Raw-text matching also carries no
+      # tokenizer, so the terms that are glued to symbols and therefore
+      # unreachable by token (`minifts`, $OKF_HOME) stay findable by default.
+      #
+      # What it gives up is BM25+ ranking, reachable with `--engine index` — and
+      # that is also the engine the browser page runs, so the two rank alike only
+      # when the index is named. See .okf/design/search-engines.md.
+      DEFAULT_ENGINE = :scan
+
       # The searchable fields with their rank weight, strongest signal first.
-      # A concept's score sums the weights of the fields that matched; hitting a
-      # field twice does not stack. Tags match against the space-joined list,
-      # mirroring the server page's haystack.
+      # In the index engine these ride as MiniFTS per-field `boost`; the scan
+      # sums the weights of the fields that matched instead.
       WEIGHTS = {
         "title" => 5,
         "id" => 4,
@@ -38,53 +111,165 @@ module OKF
       # Characters of context kept on each side of the first matched term.
       SNIPPET_RADIUS = 44
 
-      def self.call(bundle, terms, fields: nil, regexp: false)
-        new(bundle, terms, fields: fields, regexp: regexp).results
+      # Edit distance as a fraction of term length, under `fuzzy: true` — the
+      # same 0.2 the browser page passes, so both forgive the same typos.
+      FUZZY_DISTANCE = 0.2
+
+      # The unique document key is "<slug>\0<id>": ids are only unique *within* a
+      # bundle, and a merge that collided two bundles' same-named concepts would
+      # silently drop one.
+      KEY_SEPARATOR = "\0"
+
+      # Append-only and idempotent by id: a second registration of an id already
+      # present is a no-op, so a double `require` cannot double the registry and
+      # an addon cannot quietly displace a built-in. Deliberately the same shape
+      # as the Linter's planned register hook — two extension points, one idiom.
+      def self.register(engine)
+        rogue = engine.capabilities - CAPABILITIES
+        raise ArgumentError, "unknown search capability: #{rogue.join(", ")}" unless rogue.empty?
+
+        @engines ||= []
+        @engines << engine unless @engines.any? { |registered| registered.id == engine.id }
+        engine
+      end
+
+      # A frozen snapshot in registration order. Frozen because the registry is
+      # only meant to grow through .register, where the vocabulary is checked.
+      def self.engines
+        (@engines ||= []).dup.freeze
+      end
+
+      # The router. Naming an engine is an override, not a hint: it is how a
+      # caller reaches semantics no capability flag asks for — `--engine scan`
+      # means "match raw text", which the flags cannot express because there is
+      # nothing to *require*. A named engine that cannot do what was also asked
+      # is an error rather than a silent fallback, since falling back would answer
+      # a different question than the one that was posed.
+      #
+      # Unnamed, the default engine leads, then registration order; the first
+      # available engine offering *every* required capability answers. Partition
+      # rather than sort_by, because sort_by is not stable and registration order
+      # is the tie-break.
+      def self.engine_for(required, engines: self.engines, name: nil)
+        available = engines.select(&:available?)
+        return named_engine(name, required, available) unless OKF.blank?(name)
+
+        default, rest = available.partition { |engine| engine.id == DEFAULT_ENGINE }
+        found = (default + rest).find { |engine| (required - engine.capabilities).empty? }
+        return found if found
+
+        raise UnsupportedQuery, required
+      end
+
+      def self.named_engine(name, required, available)
+        wanted = name.to_s.downcase
+        found = available.find { |engine| engine.id.to_s == wanted }
+        raise UnknownEngine.new(name, available.map(&:id)) if found.nil?
+
+        missing = required - found.capabilities
+        raise UnsupportedQuery.new(missing, engine: found.id) unless missing.empty?
+
+        found
+      end
+      private_class_method :named_engine
+
+      def self.call(bundle, terms, fields: nil, regexp: false, fuzzy: false, engine: nil, engines: nil)
+        new([ [ nil, bundle ] ], terms, fields: fields, regexp: regexp, fuzzy: fuzzy, engine: engine, engines: engines).results
+      end
+
+      # Several bundles as [ slug, bundle ] pairs, ranked into one list with every
+      # row labeled by its slug. They share **one** index on purpose: BM25 weighs a
+      # term by how rare it is in the corpus, so per-bundle indexes would score the
+      # same match differently depending on which bundle it came from. One index
+      # makes one corpus, and the merged ranking is comparable by construction.
+      def self.across(bundles, terms, fields: nil, regexp: false, fuzzy: false, engine: nil, engines: nil)
+        new(bundles, terms, fields: fields, regexp: regexp, fuzzy: fuzzy, engine: engine, engines: engines).results
       end
 
-      # Raises RegexpError on an invalid pattern with `regexp: true` — the caller
-      # owns turning that into a usage error.
-      def initialize(bundle, terms, fields: nil, regexp: false)
-        @bundle = bundle
-        raw = Array(terms).reject { |term| OKF.blank?(term) }
-        @matchers = raw.map { |term| regexp ? Regexp.new(term.to_s, Regexp::IGNORECASE) : term.to_s.downcase }
+      # Raises RegexpError on an invalid pattern with `regexp: true`, and
+      # UnsupportedQuery when no engine can answer — the caller owns turning
+      # either into a usage error. `engines:` overrides the registry, which is how
+      # the "nothing qualifies" path stays reachable without an addon installed.
+      def initialize(bundles, terms, fields: nil, regexp: false, fuzzy: false, engine: nil, engines: nil)
+        @bundles = bundles
+        @terms = Array(terms).reject { |term| OKF.blank?(term) }.map(&:to_s)
         @fields = fields.nil? || fields.empty? ? FIELDS : fields
+        @regexp = regexp
+        @fuzzy = fuzzy
+        @engine = engine
+        @engines = engines
+        @sources = {}
       end
 
       # Ranked match rows, catalog-style identity plus where the terms hit:
-      # [{ id:, title:, type:, area:, tags:, matched: [field, …], score:, snippet: }, …]
-      # ordered by score descending, then id. No terms means no matches.
+      # [{ slug:, id:, title:, type:, area:, tags:, matched: [field, …], score:, snippet: }, …]
+      # ordered by score descending, then slug, then id. `slug` is present only
+      # when searching across bundles. No terms means no matches.
       def results
-        return [] if @matchers.empty?
+        return [] if @terms.empty?
 
-        @bundle.concepts
-               .map { |concept| match(concept) }
-               .compact
-               .sort_by { |row| [ -row[:score], row[:id] ] }
+        chosen = engine
+        rows = chosen.call(documents, @terms, **engine_options(chosen)).map do |hit|
+          slug, concept = @sources[hit[:key]]
+          row(slug, concept, hit[:matched], hit[:score], hit[:terms])
+        end
+        rows.sort_by { |row| [ -row[:score], row[:slug].to_s, row[:id] ] }
       end
 
       private
 
-      def match(concept)
-        texts = searchable_texts(concept)
-        matched = matched_fields(texts)
-        return nil if matched.nil?
+      # The engine is chosen by what the query needs, not by a flag naming one.
+      # `-e` unambiguously means "regexp semantics", so it routes on its own and
+      # says nothing about it — there is no --engine flag to reconcile with.
+      def engine
+        Search.engine_for(required_capabilities, engines: @engines || Search.engines, name: @engine)
+      end
 
-        {
-          id: concept.id,
-          title: (concept.title || concept.id).to_s,
-          type: concept.type.to_s,
-          area: area_of(concept.id),
-          tags: Array(concept.tags).map(&:to_s),
-          matched: matched,
-          score: matched.map { |field| WEIGHTS[field] }.reduce(0, :+),
-          snippet: snippet(texts, matched)
-        }
+      # What the query requires, in the routable vocabulary. `:prefix` never
+      # appears: it is declarable, not routable — nothing asks for its absence.
+      def required_capabilities
+        ROUTABLE.select { |capability| requested[capability] }
+      end
+
+      # `fields:` always, plus exactly the routable options the chosen engine
+      # declared it understands.
+      #
+      # The facade used to hand every engine every option and trust it to ignore
+      # what it could not use. Routing makes that harmless in practice — a fuzzy
+      # query only ever reaches a :fuzzy engine — but "harmless because something
+      # else prevents it" is precisely how an option comes to be dropped in
+      # silence the day that something else changes. An engine now receives only
+      # what it can act on, so there is nothing left for it to ignore.
+      def engine_options(chosen)
+        options = { fields: @fields }
+        ROUTABLE.each do |capability|
+          options[capability] = requested[capability] if chosen.capabilities.include?(capability)
+        end
+        options
+      end
+
+      def requested
+        @requested ||= { regexp: @regexp, fuzzy: @fuzzy }
+      end
+
+      # Every concept as an indexable document, keyed uniquely across bundles.
+      # @sources keeps the way back, so the index stores no fields of its own.
+      def documents
+        docs = []
+        @bundles.each do |slug, bundle|
+          bundle.concepts.each do |concept|
+            key = "#{slug}#{KEY_SEPARATOR}#{concept.id}"
+            @sources[key] = [ slug, concept ]
+            docs << field_texts(concept).merge("key" => key)
+          end
+        end
+        docs
       end
 
-      # { field => original-case text } for the fields this search reads.
-      def searchable_texts(concept)
-        texts = {
+      # { field => original-case text } for every searchable field. The index reads
+      # all of them; `fields:` narrows the search, not the document.
+      def field_texts(concept)
+        {
           "id" => concept.id,
           "title" => concept.title.to_s,
           "type" => concept.type.to_s,
@@ -92,35 +277,46 @@ module OKF
           "tags" => Array(concept.tags).join(" "),
           "body" => concept.body
         }
-        texts.each_with_object({}) do |(field, text), acc|
-          acc[field] = text if @fields.include?(field)
-        end
-      end
-
-      # The union of fields any term hit, in WEIGHTS order — or nil when some term
-      # hit nothing (terms are ANDed).
-      def matched_fields(texts)
-        hits = @matchers.map do |matcher|
-          fields = texts.keys.select { |field| hit?(matcher, texts[field]) }
-          return nil if fields.empty?
-
-          fields
-        end
-        FIELDS.select { |field| hits.flatten.include?(field) }
       end
 
-      def hit?(matcher, text)
-        matcher.is_a?(Regexp) ? matcher.match?(text) : text.downcase.include?(matcher)
+      # `slug` leads the row so a merged result reads bundle-first, and drops
+      # entirely for a single bundle, which has no slug to carry.
+      def row(slug, concept, matched, score, terms)
+        texts = field_texts(concept)
+        built = {
+          slug: slug,
+          id: concept.id,
+          title: (concept.title || concept.id).to_s,
+          type: concept.type.to_s,
+          area: area_of(concept.id),
+          tags: Array(concept.tags).map(&:to_s),
+          matched: matched,
+          score: score.round(4),
+          snippet: snippet(texts, matched, terms)
+        }
+        built.delete(:slug) if slug.nil?
+        built
       end
 
       # One bounded context window around the first term that hit the strongest
       # snippet-worthy field; "" when the match needs no context (id/title/type/tags).
-      def snippet(texts, matched)
+      def snippet(texts, matched, terms)
         field = SNIPPET_FIELDS.find { |candidate| matched.include?(candidate) }
         return "" if field.nil?
 
-        matcher = @matchers.find { |candidate| hit?(candidate, texts[field]) }
-        context(texts[field], matcher)
+        matcher = snippet_matcher(texts[field], terms)
+        matcher.nil? ? "" : context(texts[field], matcher)
+      end
+
+      # What to point the window at: the first of the engine's reported matchers
+      # that this text actually contains. Engine-agnostic on purpose — the scan
+      # reports compiled patterns, the index reports lowercased document terms,
+      # and both are things `locate` can find again in the flattened text.
+      def snippet_matcher(text, terms)
+        down = text.downcase
+        Array(terms).find do |term|
+          term.is_a?(Regexp) ? term.match?(text) : down.include?(term)
+        end
       end
 
       def context(text, matcher)

data/lib/okf/bundle.rb

@@ -111,7 +111,7 @@ module OKF
         id = concept.id
         {
           id: id,
-          title: (concept.title || id).to_s,
+          title: OKF.blank?(concept.title) ? File.basename(id) : concept.title.to_s,
           type: concept.type.to_s,
           description: concept.description.to_s,
           tags: Array(concept.tags).map(&:to_s),
@@ -159,7 +159,7 @@ module OKF
           listing: here.map do |concept|
             {
               id: concept.id,
-              title: (concept.title || concept.id).to_s,
+              title: OKF.blank?(concept.title) ? File.basename(concept.id) : concept.title.to_s,
               description: concept.description.to_s,
               type: concept.type.to_s,
               tags: Array(concept.tags).map(&:to_s)