index_util 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dce5305bc6964aed62507ac4f5bc18f584cd658ad514ed8dd79348c982f0cc4c
+  data.tar.gz: 37e66070cd90e3f0ba284229cdb1e3908604fb82fb55e08b90d527523d15a921
+SHA512:
+  metadata.gz: d4240cf2c8cabf7e603c95f135982b7ea0cd454db695873b62eda8c5c4380f9b545ee396d47f1e86692266f6eb5b4714a6738e3c33d0ca6ec8f86bf132b71caf
+  data.tar.gz: 8b8469fdb26cbd6d8f6dcc438ce494325a88ccdaa88717a9e959487245d390582684997febbd423d09923a95e15d3ebb5f12bf1c94bff4e005dde5537d2f61ba

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation

data/.rubocop.yml

@@ -0,0 +1,28 @@
+AllCops:
+  TargetRubyVersion: 3.2
+  NewCops: disable
+  SuggestExtensions: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+  Exclude:
+    - "spec/**/*"
+    - "index_util.gemspec"
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 170
+
+Metrics/BlockLength:
+  Enabled: false
+Metrics/ClassLength:
+  Enabled: false
+Metrics/MethodLength:
+  Enabled: false
+Metrics/AbcSize:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## 0.1.0 - 2026-06-10
+
+- Initial release of `index_util`.
+- Added subclass-defined indexing hooks for document discovery, content loading, checksums, sections, postprocessing, and query amendments.
+- Added SQLite storage with FTS5 keyword search and `sqlite-vec` vector search.
+- Added `embedding_util` integration for embeddings and reranking.
+- Added CLI support for rebuild, new-only indexing, update indexing, querying, and indexing progress output.
+- Added standalone examples for embedded recipes, Kreuzberg PDF extraction, and RubyAPI method search.

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 hmdne
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,134 @@
+# index_util
+
+Subclass-defined local search indexes for Ruby.
+
+`index_util` lets a small Ruby class define where documents come from, how they are split, and how final searchable fragments are produced. The gem stores those fragments in SQLite with FTS5 keyword lookup, `sqlite-vec` vector lookup, and `embedding_util` embeddings/reranking.
+
+This gem is in the `0.x` series. The API is intentionally unstable until `1.0`, and public method names, configuration options, return shapes, and default profiles may change between minor releases.
+
+## Installation
+
+Add the gem to your Gemfile:
+
+```ruby
+gem "index_util"
+```
+
+Then install dependencies:
+
+```sh
+bundle install
+```
+
+## Quick Start
+
+```ruby
+#!/usr/bin/env ruby
+
+require "index_util"
+
+class MyIndex < IndexUtil::Index
+  def database_file
+    "myindex.sqlite3"
+  end
+
+  def document_list
+    Dir["docs/**/*.md"]
+  end
+
+  def document_sections(_document, content)
+    content.split(/^## /).each_with_index.to_h { |section, index| [index, section] }
+  end
+end
+
+MyIndex.cli if $PROGRAM_NAME == __FILE__
+```
+
+Build and query:
+
+```sh
+./myindex.rb --index
+./myindex.rb "how do I split text?" --limit 5
+```
+
+## CLI
+
+Subclass scripts can call `MyIndex.cli` and use:
+
+```sh
+./myindex.rb --index
+./myindex.rb --index-new
+./myindex.rb --index-update
+./myindex.rb "query text" --limit 5
+```
+
+The installed executable can load a class explicitly:
+
+```sh
+index_util --require ./myindex.rb --class MyIndex index
+index_util --require ./myindex.rb --class MyIndex query "query text" --limit 5
+```
+
+When a script should support both direct execution and installed CLI loading, guard the class CLI call with `if $PROGRAM_NAME == __FILE__`.
+
+Indexing commands print one-line progress to stderr while they list, load, embed, and store documents. Successful queries print a JSON array with `document` and `content`. Failures print compact JSON to stderr and exit non-zero.
+
+## Examples
+
+Examples live under `examples/`. Each example is a small standalone directory with its own Gemfile pointing back to this checkout.
+
+Recipe-card example:
+
+```sh
+cd examples/recipes
+bundle install
+bundle exec ruby recipes.rb --index
+bundle exec ruby recipes.rb "quick vegetarian dinner" --limit 3
+```
+
+Kreuzberg PDF example:
+
+```sh
+cd examples/pdf_notes
+bundle install
+bundle exec ruby pdf_notes.rb --index
+bundle exec ruby pdf_notes.rb "how should a risk assessment be conducted?" --limit 3
+```
+
+Ruby API example:
+
+```sh
+cd examples/ruby_api
+bundle install
+bundle exec ruby ruby_api.rb --index
+bundle exec ruby ruby_api.rb "how to split a string" --limit 5
+```
+
+## API
+
+Subclass `IndexUtil::Index` and define:
+
+- `database_file` returns the SQLite path.
+- `document_list` returns source documents.
+- `document_content(document)` defaults to `File.read(document)`.
+- `document_checksum(document, content)` defaults to stable SHA-256 over `content.to_s`.
+- `document_sections(document, content)` defaults to `{ nil => content }`.
+- `document_postprocess(fragment_document, content)` defaults to identity and must return a `String`.
+- `query_amendments(query)` defaults to `{}` and returns extra `{document_id => content}` candidates.
+
+Fragment document ids are built as `document` for nil sections or `document#section_id` otherwise. `#` is not escaped.
+
+## Development
+
+```sh
+bundle install
+bundle exec rake
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome at `https://github.com/rbutils/index_util`.
+
+## License
+
+The gem is available as open source under the terms of the MIT License.

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rubocop/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/exe/index_util

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "index_util"
+
+IndexUtil::CLI.run(ARGV)

data/lib/index_util/cli.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "json"
+require "thor"
+
+module IndexUtil
+  class CLI < Thor
+    class_option :require, type: :string, aliases: "-r", desc: "Ruby file that defines the index class"
+    class_option :class, type: :string, aliases: "-c", desc: "Index class name to instantiate"
+    class_option :verbose, type: :boolean, default: false
+
+    desc "index", "Rebuild the index from scratch"
+    def index
+      index_instance.index_all!(progress: progress)
+    end
+
+    desc "index-new", "Index only new documents"
+    def index_new
+      index_instance.index_new!(progress: progress)
+    end
+
+    desc "index-update", "Reconcile changed and stale documents"
+    def index_update
+      index_instance.index_update!(progress: progress)
+    end
+
+    desc "query QUERY", "Search the index"
+    option :limit, type: :numeric, default: Index::DEFAULT_LIMIT
+    def query(query)
+      puts JSON.pretty_generate(index_instance.query(query, limit: options[:limit], verbose: options[:verbose]))
+    end
+
+    def self.run(argv = ARGV, index_class: nil)
+      if index_class
+        run_for_class(index_class, argv)
+      else
+        start(normalize_legacy_args(argv))
+      end
+    rescue StandardError => e
+      warn JSON.generate(error: e.class.name, message: e.message)
+      warn e.backtrace.join("\n") if argv.include?("--verbose") && e.backtrace
+      exit 1
+    end
+
+    def self.run_for_class(index_class, argv)
+      command_argv = normalize_legacy_args(argv)
+      start(command_argv, index_instance: index_class.new)
+    end
+
+    def self.normalize_legacy_args(argv)
+      args = argv.dup
+      legacy_commands = { "--index" => "index", "--index-new" => "index-new", "--index-update" => "index-update" }
+      legacy_flag = legacy_commands.keys.find { |flag| args.include?(flag) }
+      return [legacy_commands.fetch(legacy_flag), *args.reject { |arg| arg == legacy_flag }] if legacy_flag
+      return ["query", *args] unless args.empty? || %w[index index-new index-update query].include?(args.first)
+
+      args
+    end
+
+    no_commands do
+      def initialize(args = [], local_options = {}, config = {})
+        super
+        @provided_index_instance = config[:index_instance]
+      end
+
+      def index_instance
+        @index_instance ||= begin
+          return @provided_index_instance if @provided_index_instance
+
+          require options[:require] if options[:require]
+          class_name = options[:class] || ENV.fetch("INDEX_UTIL_CLASS", nil)
+          raise Error, "Pass --class INDEX_CLASS or set INDEX_UTIL_CLASS" unless class_name
+
+          class_name.split("::").reduce(Object) { |scope, name| scope.const_get(name) }.new
+        end
+      end
+
+      def progress
+        @progress ||= Progress.new($stderr)
+      end
+    end
+  end
+end

data/lib/index_util/database.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "sequel"
+require "sequel/extensions/migration"
+require "sqlite3"
+
+module IndexUtil
+  module Database
+    module_function
+
+    def connect(database_file)
+      db = Sequel.sqlite(database_file)
+      configure!(db)
+      migrate!(db)
+      db
+    end
+
+    def rebuild!(database_file)
+      close_database_files(database_file)
+      FileUtils.rm_f(database_file)
+      FileUtils.rm_f("#{database_file}-wal")
+      FileUtils.rm_f("#{database_file}-shm")
+      connect(database_file)
+    end
+
+    def configure!(db)
+      db.run "PRAGMA foreign_keys = ON"
+      db.run "PRAGMA journal_mode = WAL"
+      db.run "PRAGMA synchronous = NORMAL"
+      verify_fts5!(db)
+      VectorIndex.load!(db)
+      db
+    end
+
+    def migrate!(db)
+      Sequel::Migrator.run(db, migrations_path)
+    rescue Sequel::Migrator::Error => e
+      raise Error, "Unsupported index schema migration state: #{e.message}"
+    end
+
+    def migrations_path
+      File.expand_path("migrations", __dir__)
+    end
+
+    def verify_fts5!(db)
+      db.run "CREATE VIRTUAL TABLE temp.index_util_fts5_check USING fts5(content)"
+      db.run "DROP TABLE temp.index_util_fts5_check"
+    rescue Sequel::DatabaseError => e
+      raise Error, "SQLite FTS5 support is required by index_util: #{e.message}"
+    end
+
+    def close_database_files(database_file)
+      return unless database_file
+
+      FileUtils.mkdir_p(File.dirname(File.expand_path(database_file)))
+    end
+  end
+end

data/lib/index_util/document.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "time"
+
+module IndexUtil
+  module Document
+    module_function
+
+    def exist?(db, document)
+      db[:documents].where(document: document.to_s).count.positive?
+    end
+
+    def unchanged?(db, document, checksum)
+      db[:documents].where(document: document.to_s, checksum: checksum.to_s).count.positive?
+    end
+
+    def delete_stale(db, current_documents)
+      current = current_documents.map(&:to_s)
+      dataset = db[:documents]
+      dataset = dataset.exclude(document: current) unless current.empty?
+      dataset.all.each { |row| delete(db, row.fetch(:document)) }
+    end
+
+    def delete(db, document)
+      row = db[:documents].where(document: document.to_s).first
+      return unless row
+
+      db[:fragments].where(document_id: row.fetch(:id)).all.each do |fragment|
+        FragmentsFts.delete(db, fragment)
+        VectorIndex.delete(db, fragment.fetch(:id))
+      end
+      db[:documents].where(id: row.fetch(:id)).delete
+    end
+
+    def replace_with_fragments(db, attributes)
+      document = attributes.fetch(:document)
+      checksum = attributes.fetch(:checksum)
+      fragments = attributes.fetch(:fragments)
+      embeddings = attributes.fetch(:embeddings)
+      profile = attributes.fetch(:profile)
+      raise Error, "fragment and embedding counts differ" unless fragments.length == embeddings.length
+
+      db.transaction do
+        delete(db, document)
+        now = timestamp
+        document_id = db[:documents].insert(document: document.to_s, checksum: checksum.to_s, indexed_at: now, created_at: now, updated_at: now)
+
+        fragments.zip(embeddings).each do |fragment, embedding|
+          fragment_id = db[:fragments].insert(
+            document_id: document_id,
+            document: document.to_s,
+            section_id: fragment.fetch(:section_id),
+            fragment_document: fragment.fetch(:fragment_document),
+            content: fragment.fetch(:content),
+            created_at: now,
+            updated_at: now
+          )
+          stored_fragment = db[:fragments].where(id: fragment_id).first
+          db[:fragment_embeddings].insert(
+            fragment_id: fragment_id,
+            profile: profile.to_s,
+            dimensions: embedding.length,
+            vector: FragmentEmbedding.pack(embedding),
+            vector_norm: FragmentEmbedding.norm(embedding),
+            created_at: now
+          )
+          FragmentsFts.insert(db, stored_fragment)
+          VectorIndex.insert(db, fragment_id, embedding)
+        end
+      end
+    end
+
+    def timestamp
+      Time.now.utc.iso8601
+    end
+  end
+end

data/lib/index_util/fragment.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module IndexUtil
+  module Fragment
+    module_function
+
+    def fragment_document(document, section_id)
+      document = document.to_s
+      return document if section_id.nil?
+
+      "#{document}##{section_id}"
+    end
+  end
+end

data/lib/index_util/fragment_embedding.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "sequel"
+
+module IndexUtil
+  module FragmentEmbedding
+    module_function
+
+    def pack(vector)
+      Sequel.blob(Array(vector).map(&:to_f).pack("f*"))
+    end
+
+    def unpack(blob)
+      blob.unpack("f*")
+    end
+
+    def norm(vector)
+      Math.sqrt(Array(vector).sum { |value| value.to_f * value.to_f })
+    end
+  end
+end

data/lib/index_util/fragments_fts.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module IndexUtil
+  module FragmentsFts
+    module_function
+
+    def insert(db, fragment)
+      db[
+        "INSERT INTO fragments_fts(rowid, fragment_document, content) VALUES (?, ?, ?)",
+        fragment.fetch(:id),
+        fragment.fetch(:fragment_document),
+        fragment.fetch(:content)
+      ].insert
+    end
+
+    def delete(db, fragment)
+      db[
+        "INSERT INTO fragments_fts(fragments_fts, rowid, fragment_document, content) VALUES ('delete', ?, ?, ?)",
+        fragment.fetch(:id),
+        fragment.fetch(:fragment_document),
+        fragment.fetch(:content)
+      ].insert
+    end
+
+    def search(db, query, limit:)
+      db.fetch(<<~SQL, query.to_s, Integer(limit)).all
+        SELECT fragments.id, fragments.fragment_document, fragments.content, bm25(fragments_fts) AS score
+        FROM fragments_fts
+        JOIN fragments ON fragments.id = fragments_fts.rowid
+        WHERE fragments_fts MATCH ?
+        ORDER BY score
+        LIMIT ?
+      SQL
+    rescue Sequel::DatabaseError
+      []
+    end
+  end
+end

data/lib/index_util/index.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "time"
+
+require "embedding_util"
+
+module IndexUtil
+  class Index
+    EMBED_MANY_LIMIT = 1000
+    DEFAULT_LIMIT = 5
+
+    class << self
+      def cli(argv = ARGV)
+        CLI.run(argv, index_class: self)
+      end
+    end
+
+    def database_file
+      raise Error, "#{self.class} must define #database_file"
+    end
+
+    def document_list
+      raise Error, "#{self.class} must define #document_list"
+    end
+
+    def document_content(document)
+      File.read(document.to_s)
+    end
+
+    def document_checksum(_document, content)
+      Digest::SHA256.hexdigest(content.to_s)
+    end
+
+    def document_sections(_document, content)
+      { nil => content }
+    end
+
+    def document_postprocess(_fragment_document, content)
+      content
+    end
+
+    def query_amendments(_query)
+      {}
+    end
+
+    def index_all!(progress: nil)
+      progress&.step("Preparing index", detail: "rebuilding #{database_file}")
+      db = Database.rebuild!(database_file)
+      documents = list_documents(progress)
+      index_documents(db, documents, mode: :all, progress: progress)
+    ensure
+      db&.disconnect
+    end
+
+    def index_new!(progress: nil)
+      progress&.step("Preparing index", detail: "opening #{database_file}")
+      db = Database.connect(database_file)
+      documents = list_documents(progress)
+      index_documents(db, documents, mode: :new, progress: progress)
+    ensure
+      db&.disconnect
+    end
+
+    def index_update!(progress: nil)
+      progress&.step("Preparing index", detail: "opening #{database_file}")
+      db = Database.connect(database_file)
+      documents = list_documents(progress)
+      progress&.step("Updating index", current: 0, total: documents.length, detail: "removing stale documents")
+      Document.delete_stale(db, documents)
+      index_documents(db, documents, mode: :update, progress: progress)
+    ensure
+      db&.disconnect
+    end
+
+    def query(query, limit: DEFAULT_LIMIT, verbose: false)
+      db = Database.connect(database_file)
+      limit = Integer(limit)
+      candidate_limit = [limit * 10, 50].max
+      query_embedding = EmbeddingUtil.embed(query.to_s)
+      vector_candidates = VectorIndex.search(db, query_embedding, limit: candidate_limit)
+      keyword_candidates = FragmentsFts.search(db, query.to_s, limit: candidate_limit)
+      amendments = query_amendments(query.to_s)
+      candidates = merge_candidates(vector_candidates, keyword_candidates, amendments)
+      rerank(query.to_s, candidates, limit: limit, verbose: verbose)
+    ensure
+      db&.disconnect
+    end
+
+    private
+
+    def list_documents(progress)
+      progress&.step("Listing documents")
+      documents = document_list.to_a
+      progress&.step("Listing documents", current: documents.length, total: documents.length, detail: "found #{documents.length}")
+      documents
+    end
+
+    def index_documents(db, documents, mode:, progress:)
+      stats = { indexed: 0, skipped: 0 }
+      total = documents.length
+      documents.each_with_index do |document, index|
+        current = index + 1
+        document_key = document.to_s
+        progress&.step("Indexing documents", current: current, total: total, detail: "checking #{document_key}")
+        if mode == :new && Document.exist?(db, document_key)
+          stats[:skipped] += 1
+          progress&.step("Indexing documents", current: current, total: total, detail: "skipped existing #{document_key}")
+          next
+        end
+
+        progress&.step("Indexing documents", current: current, total: total, detail: "loading #{document_key}")
+        content = document_content(document)
+        checksum = document_checksum(document, content).to_s
+        if mode == :update && Document.unchanged?(db, document_key, checksum)
+          stats[:skipped] += 1
+          progress&.step("Indexing documents", current: current, total: total, detail: "skipped unchanged #{document_key}")
+          next
+        end
+
+        progress&.step("Indexing documents", current: current, total: total, detail: "sectioning #{document_key}")
+        fragments = build_fragments(document, content)
+        progress&.step("Indexing documents", current: current, total: total, detail: "embedding #{fragments.length} fragments from #{document_key}")
+        embeddings = embed_fragments(fragments)
+        ensure_vector_index(db, embeddings)
+        progress&.step("Indexing documents", current: current, total: total, detail: "storing #{document_key}")
+        Document.replace_with_fragments(db, { document: document_key, checksum: checksum, fragments: fragments, embeddings: embeddings, profile: embedding_profile })
+        stats[:indexed] += 1
+        progress&.step("Indexing documents", current: current, total: total, detail: "indexed #{document_key}")
+      end
+      progress&.finish("Indexing complete: #{stats.fetch(:indexed)} indexed, #{stats.fetch(:skipped)} skipped, #{total} total")
+    end
+
+    def build_fragments(document, content)
+      sections = document_sections(document, content)
+      raise Error, "document_sections must return a hash-like object of section_id => content" unless sections.respond_to?(:each_pair)
+
+      sections.each_pair.map do |section_id, section_content|
+        section_key = section_id&.to_s
+        fragment_document = Fragment.fragment_document(document, section_key)
+        processed = document_postprocess(fragment_document, section_content)
+        raise Error, "document_postprocess must return a String for #{fragment_document}" unless processed.is_a?(String)
+
+        { section_id: section_key, fragment_document: fragment_document, content: processed }
+      end
+    end
+
+    def embed_fragments(fragments)
+      fragments.each_slice(EMBED_MANY_LIMIT).flat_map do |batch|
+        EmbeddingUtil.embed_many(batch.map { |fragment| fragment.fetch(:content) })
+      end
+    end
+
+    def ensure_vector_index(db, embeddings)
+      first = embeddings.first
+      raise Error, "EmbeddingUtil returned no embeddings" unless first
+
+      VectorIndex.ensure!(db, dimensions: first.length)
+    end
+
+    def embedding_profile
+      profile = EmbeddingUtil.configuration.resolved_profile
+      profile.respond_to?(:name) ? profile.name : profile.to_s
+    rescue StandardError
+      "default"
+    end
+
+    def merge_candidates(vector_candidates, keyword_candidates, amendments)
+      merged = {}
+      vector_candidates.each do |row|
+        key = row.fetch(:fragment_document)
+        merged[key] = { document: key, content: row.fetch(:content), sources: [], vector_score: nil, keyword_score: nil }
+        merged[key][:sources] << "vector"
+        merged[key][:vector_score] = row[:distance]
+      end
+      keyword_candidates.each do |row|
+        key = row.fetch(:fragment_document)
+        merged[key] ||= { document: key, content: row.fetch(:content), sources: [], vector_score: nil, keyword_score: nil }
+        merged[key][:sources] << "keyword"
+        merged[key][:keyword_score] = row[:score]
+      end
+      amendments.each_pair do |document, content|
+        key = document.to_s
+        merged[key] ||= { document: key, content: content.to_s, sources: [], vector_score: nil, keyword_score: nil }
+        merged[key][:sources] << "amendment"
+      end
+      merged.values
+    end
+
+    def rerank(query, candidates, limit:, verbose:)
+      return [] if candidates.empty?
+
+      documents = candidates.map { |candidate| candidate.fetch(:content) }
+      ranked = EmbeddingUtil.rerank(query, documents)
+      ranked.first(limit).map do |result|
+        candidate = candidates.fetch(result.index)
+        output = { document: candidate.fetch(:document), content: candidate.fetch(:content) }
+        next output unless verbose
+
+        output.merge(
+          sources: candidate.fetch(:sources).uniq,
+          vector_score: candidate[:vector_score],
+          keyword_score: candidate[:keyword_score],
+          reranker_score: result.score
+        )
+      end
+    end
+  end
+end

data/lib/index_util/index_metadata.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "time"
+
+module IndexUtil
+  module IndexMetadata
+    module_function
+
+    def get(db, key)
+      db[:index_metadata].where(key: key.to_s).get(:value)
+    end
+
+    def set(db, key, value)
+      now = timestamp
+      row = { key: key.to_s, value: value.to_s, updated_at: now }
+      db[:index_metadata].insert_conflict(target: :key, update: { value: row[:value], updated_at: now }).insert(row)
+    end
+
+    def timestamp
+      Time.now.utc.iso8601
+    end
+  end
+end

data/lib/index_util/migrations/001_create_core_tables.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:index_metadata) do
+      String :key, primary_key: true
+      String :value, null: false
+      String :updated_at, null: false
+    end
+
+    create_table(:documents) do
+      primary_key :id
+      String :document, null: false, unique: true
+      String :checksum, null: false
+      String :indexed_at, null: false
+      String :created_at, null: false
+      String :updated_at, null: false
+
+      index :document, unique: true
+      index :checksum
+    end
+
+    create_table(:fragments) do
+      primary_key :id
+      foreign_key :document_id, :documents, null: false, on_delete: :cascade
+      String :document, null: false
+      String :section_id
+      String :fragment_document, null: false, unique: true
+      String :content, text: true, null: false
+      String :created_at, null: false
+      String :updated_at, null: false
+
+      index :document_id
+      index :document
+      index :fragment_document, unique: true
+    end
+
+    create_table(:fragment_embeddings) do
+      foreign_key :fragment_id, :fragments, null: false, on_delete: :cascade
+      String :profile, null: false
+      Integer :dimensions, null: false
+      File :vector, null: false
+      Float :vector_norm
+      String :created_at, null: false
+
+      primary_key %i[fragment_id profile]
+      index :profile
+    end
+  end
+end

data/lib/index_util/migrations/002_create_fragments_fts.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    run <<~SQL
+      CREATE VIRTUAL TABLE fragments_fts
+      USING fts5(fragment_document, content, content='fragments', content_rowid='id')
+    SQL
+  end
+
+  down do
+    run "DROP TABLE IF EXISTS fragments_fts"
+  end
+end

data/lib/index_util/migrations/003_create_fragment_vectors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    # The sqlite-vec table depends on the active embedding dimension and is
+    # created by IndexUtil::VectorIndex when the first embeddings are stored.
+  end
+
+  down do
+    run "DROP TABLE IF EXISTS fragment_vectors"
+  end
+end

data/lib/index_util/progress.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module IndexUtil
+  class Progress
+    def initialize(io = $stderr, width: 24)
+      @io = io
+      @width = width
+      @last_length = 0
+    end
+
+    def step(message, current: nil, total: nil, detail: nil)
+      write(format_line(message, current: current, total: total, detail: detail))
+    end
+
+    def finish(message)
+      write(message)
+      @io.puts
+      @last_length = 0
+    end
+
+    private
+
+    def format_line(message, current:, total:, detail:)
+      parts = [message]
+      parts << bar(current, total) if current && total
+      parts << "#{current}/#{total}" if current && total
+      parts << detail if detail && !detail.empty?
+      parts.join(" ")
+    end
+
+    def bar(current, total)
+      ratio = total.zero? ? 1.0 : current.to_f / total
+      filled = (ratio.clamp(0.0, 1.0) * @width).round
+
+      "[#{'=' * filled}#{'.' * (@width - filled)}]"
+    end
+
+    def write(line)
+      padding = [@last_length - line.length, 0].max
+      @io.print("\r#{line}#{' ' * padding}")
+      @io.flush
+      @last_length = line.length
+    end
+  end
+end

data/lib/index_util/vector_index.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module IndexUtil
+  module VectorIndex
+    module_function
+
+    def load!(db)
+      require "sqlite_vec"
+      db.synchronize do |connection|
+        connection.enable_load_extension(true)
+        SqliteVec.load(connection)
+        connection.enable_load_extension(false)
+      end
+      db.get(Sequel.lit("vec_version()"))
+    rescue LoadError => e
+      raise Error, "sqlite-vec gem is required by index_util: #{e.message}"
+    rescue StandardError => e
+      raise Error, "sqlite-vec support is required by index_util: #{e.message}"
+    end
+
+    def ensure!(db, dimensions:)
+      dimensions = Integer(dimensions)
+      stored_dimensions = IndexMetadata.get(db, :embedding_dimensions)
+      if stored_dimensions && stored_dimensions.to_i != dimensions
+        raise Error, "Embedding dimensions changed from #{stored_dimensions} to #{dimensions}; rebuild the index with --index"
+      end
+
+      db.run <<~SQL
+        CREATE VIRTUAL TABLE IF NOT EXISTS fragment_vectors USING vec0(
+          fragment_id integer primary key,
+          embedding float[#{dimensions}]
+        )
+      SQL
+      IndexMetadata.set(db, :embedding_dimensions, dimensions)
+    rescue Sequel::DatabaseError => e
+      raise Error, "Failed to create sqlite-vec table: #{e.message}"
+    end
+
+    def insert(db, fragment_id, vector)
+      db["INSERT INTO fragment_vectors(fragment_id, embedding) VALUES (?, ?)", Integer(fragment_id), FragmentEmbedding.pack(vector)].insert
+    end
+
+    def delete(db, fragment_id)
+      db[:fragment_vectors].where(fragment_id: Integer(fragment_id)).delete
+    rescue Sequel::DatabaseError
+      nil
+    end
+
+    def search(db, query_embedding, limit:)
+      db.fetch(<<~SQL, FragmentEmbedding.pack(query_embedding), Integer(limit)).all
+        SELECT fragments.id, fragments.fragment_document, fragments.content, fragment_vectors.distance AS distance
+        FROM fragment_vectors
+        JOIN fragments ON fragments.id = fragment_vectors.fragment_id
+        WHERE embedding MATCH ? AND k = ?
+        ORDER BY distance
+      SQL
+    rescue Sequel::DatabaseError
+      []
+    end
+  end
+end

data/lib/index_util/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module IndexUtil
+  VERSION = "0.1.0"
+end

data/lib/index_util.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "index_util/version"
+
+module IndexUtil
+  class Error < StandardError; end
+
+  autoload :CLI, "index_util/cli"
+  autoload :Database, "index_util/database"
+  autoload :Document, "index_util/document"
+  autoload :Fragment, "index_util/fragment"
+  autoload :FragmentEmbedding, "index_util/fragment_embedding"
+  autoload :FragmentsFts, "index_util/fragments_fts"
+  autoload :Index, "index_util/index"
+  autoload :IndexMetadata, "index_util/index_metadata"
+  autoload :Progress, "index_util/progress"
+  autoload :VectorIndex, "index_util/vector_index"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: index_util
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- hmdne
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: embedding_util
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite-vec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: A small rbutils gem for building local document indexes with SQLite FTS,
+  vector search, embeddings, and reranking.
+email:
+- 54514036+hmdne@users.noreply.github.com
+executables:
+- index_util
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/index_util
+- lib/index_util.rb
+- lib/index_util/cli.rb
+- lib/index_util/database.rb
+- lib/index_util/document.rb
+- lib/index_util/fragment.rb
+- lib/index_util/fragment_embedding.rb
+- lib/index_util/fragments_fts.rb
+- lib/index_util/index.rb
+- lib/index_util/index_metadata.rb
+- lib/index_util/migrations/001_create_core_tables.rb
+- lib/index_util/migrations/002_create_fragments_fts.rb
+- lib/index_util/migrations/003_create_fragment_vectors.rb
+- lib/index_util/progress.rb
+- lib/index_util/vector_index.rb
+- lib/index_util/version.rb
+homepage: https://github.com/rbutils/index_util
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/rbutils/index_util
+  changelog_uri: https://github.com/rbutils/index_util/blob/master/CHANGELOG.md
+  documentation_uri: https://github.com/rbutils/index_util#readme
+  bug_tracker_uri: https://github.com/rbutils/index_util/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Subclass-defined local search indexes for Ruby
+test_files: []