tina4ruby 3.13.61 → 3.13.63

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a1fe8987a4b278ffa81f920f3a04af6747b306c06d991543f9cc9813d4c4fd9
-  data.tar.gz: fbc9186a54212acd053e83ba9980faf482b65e4831c7ef912d9717c741d662f4
+  metadata.gz: 51cd83415eaad0fc64019fcbab89e92fb0cfb90cd9a3e1a3db6122499c14c83f
+  data.tar.gz: 9f1283a11d1f433d3ca39ae961db99bc73aa0e1f462336508c696ebbceaf9642
 SHA512:
-  metadata.gz: 29bc302b4ed44939105eaad0181fe863ed655b4bc28a3271fce8964943d61c4240f5f12a8ff2d9da221588445d5e4ff7071eaebac52f3bb632e4c3d9051b7d64
-  data.tar.gz: 5911bf0ffa529102fa933ae7841b64cef1b6bbe02bd6b9a693ea2db8bce4bd290f986291b0f3313e7eb42a6f5b20f1d44552cf642a12b7d65c30de6500e71bd1
+  metadata.gz: 59ceaebe4dea54ec1b9e4894efee36cb1d98af2e9bdfc919033a055f5824d12f37f523177003b4a0f2497d0ca0f0a4c3086628f8ae1e29e8035beda64a7ea4b5
+  data.tar.gz: 6aa2471f6a38fe85d13e8832f68c23fd093ec0bbf9837f6ff6da220880428af9851c044cbf4d89f9cbd4ae64c1cc1c4f73e9373d3037902731018227fb6d0bbb

data/lib/tina4/auto_crud.rb

@@ -9,15 +9,32 @@ module Tina4
         @models ||= []
       end
 
-      # Register a model for auto-CRUD
-      def register(model_class)
+      # Per-model public-writes flag (model_class => Boolean).
+      # Default secure; only set true when a model is registered `public: true`.
+      # Mirrors tina4-php's `$this->public` map. Write routes stay gated unless
+      # a model is explicitly opted out.
+      def public_flags
+        @public_flags ||= {}
+      end
+
+      # Register a model for auto-CRUD.
+      #
+      # @param model_class [Class] the ORM subclass to expose
+      # @param public [Boolean] when true, the generated write routes
+      #   (POST/PUT/DELETE) opt OUT of the framework's secure-by-default write
+      #   gate — they are marked `.no_auth`. Default false keeps writes gated
+      #   (a valid bearer token is required), matching the router's rule that
+      #   POST/PUT/PATCH/DELETE require auth unless explicitly opened. Parity
+      #   with tina4-python / tina4-php.
+      def register(model_class, public: false)
         models << model_class unless models.include?(model_class)
+        public_flags[model_class] = public
       end
 
       # Generate REST endpoints for all registered models
       def generate_routes(prefix: "/api")
         models.each do |model_class|
-          generate_routes_for(model_class, prefix: prefix)
+          generate_routes_for(model_class, prefix: prefix, public: public_flags[model_class])
         end
       end
 
@@ -45,13 +62,25 @@ module Tina4
         example
       end
 
-      # Generate REST endpoints for a single model class
-      def generate_routes_for(model_class, prefix: "/api")
+      # Generate REST endpoints for a single model class.
+      #
+      # @param model_class [Class] the ORM subclass to expose
+      # @param prefix [String] URL prefix for the generated routes
+      # @param public [Boolean, nil] when true, the write routes
+      #   (POST/PUT/DELETE) opt out of the secure-by-default gate via
+      #   `.no_auth`. When nil (the default), the flag stored at register()
+      #   time is used, falling back to false (secure). The read routes (GET)
+      #   are never gated regardless.
+      def generate_routes_for(model_class, prefix: "/api", public: nil)
         table = model_class.table_name
         pk = model_class.primary_key_field || :id
         pretty_name = table.to_s.split("_").map(&:capitalize).join(" ")
         example_body = build_example(model_class)
 
+        # Explicit arg wins; otherwise fall back to the flag recorded by
+        # register(), otherwise secure-by-default.
+        is_public = public.nil? ? (public_flags[model_class] || false) : public
+
         # GET /api/{table} -- list all with pagination, filtering, sorting
         Tina4::Router.add("GET", "#{prefix}/#{table}", proc { |req, res|
           begin
@@ -109,7 +138,7 @@ module Tina4
         }, swagger_meta: { summary: "Get #{pretty_name} by ID", tags: [table.to_s] })
 
         # POST /api/{table} -- create record
-        Tina4::Router.add("POST", "#{prefix}/#{table}", proc { |req, res|
+        post_route = Tina4::Router.add("POST", "#{prefix}/#{table}", proc { |req, res|
           begin
             attributes = req.body_parsed
             record = model_class.create(attributes)
@@ -136,8 +165,11 @@ module Tina4
           }
         })
 
+        # Secure-by-default: only opt out of the write gate when public: true.
+        post_route.no_auth if is_public
+
         # PUT /api/{table}/{id} -- update record
-        Tina4::Router.add("PUT", "#{prefix}/#{table}/{id}", proc { |req, res|
+        put_route = Tina4::Router.add("PUT", "#{prefix}/#{table}/{id}", proc { |req, res|
           begin
             id = req.params["id"]
             record = model_class.find_by_id(id.to_i)
@@ -174,8 +206,11 @@ module Tina4
           }
         })
 
+        # Secure-by-default: only opt out of the write gate when public: true.
+        put_route.no_auth if is_public
+
         # DELETE /api/{table}/{id} -- delete record
-        Tina4::Router.add("DELETE", "#{prefix}/#{table}/{id}", proc { |req, res|
+        delete_route = Tina4::Router.add("DELETE", "#{prefix}/#{table}/{id}", proc { |req, res|
           begin
             id = req.params["id"]
             record = model_class.find_by_id(id.to_i)
@@ -192,14 +227,20 @@ module Tina4
             res.json({ error: e.message }, status: 500)
           end
         }, swagger_meta: { summary: "Delete #{pretty_name}", tags: [table.to_s] })
+
+        # Secure-by-default: only opt out of the write gate when public: true.
+        delete_route.no_auth if is_public
       end
 
       # Discover ORM model classes from a directory and register them.
       #
       # @param models_dir [String] directory to scan (default "src/orm")
       # @param prefix [String] URL prefix for generated routes (default "/api")
+      # @param public [Boolean] when true, opt discovered models' write routes
+      #   out of the secure-by-default gate (see .register). Default false keeps
+      #   writes gated. Parity with tina4-python / tina4-php.
       # @return [Array<String>] list of discovered model class names
-      def discover(models_dir = "src/orm", prefix: "/api")
+      def discover(models_dir = "src/orm", prefix: "/api", public: false)
         discovered = []
         return discovered unless Dir.exist?(models_dir)
 
@@ -210,7 +251,7 @@ module Tina4
         # Find all ORM subclasses that have a table_name
         ObjectSpace.each_object(Class).select { |c| c < Tina4::ORM rescue false }.each do |klass|
           next unless klass.respond_to?(:table_name) && klass.table_name
-          register(klass)
+          register(klass, public: public)
           discovered << klass.name
         end
 
@@ -220,6 +261,7 @@ module Tina4
 
       def clear!
         @models = []
+        @public_flags = {}
       end
 
       # Alias for parity with other frameworks

data/lib/tina4/context/chunker.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+# Tina4 Ruby
+#
+# Text folding + chunking for the Context subsystem.
+#
+# A thin, idiomatic port of the proven slice of tina4-python's
+# tina4_python/context/chunker.py (itself a port of neemee's
+# tokenizer/chunkers). Pure stdlib — nothing here touches SQLite; it produces
+# the (folded body, raw text) pairs the FTS5 index stores.
+
+module Tina4
+  class Context
+    # Folding + chunking helpers. All methods are module functions so they read
+    # as `Chunker.fold(text)`, matching the Python free-function layout.
+    module Chunker
+      module_function
+
+      # Join comma-grouped numbers ("24,601" -> "24601") and split camelCase
+      # ("ForeignKeyField" -> "foreign key field") so a query token reaches a
+      # code identifier. snake_case already splits for free at the tokenizer.
+      NUM_COMMA = /(?<=\d),(?=\d)/
+      CAMEL     = /(?<=[a-z0-9])(?=[A-Z])/
+      WORD_RE   = /[a-z0-9]+/
+
+      # Sentence boundary = end punctuation FOLLOWED BY whitespace/EOL, or a
+      # newline. NOT a bare '.', which would shred embedded code in prose
+      # ("db.fetch()" -> "db. fetch()"). Intra-token dots stay intact.
+      SENT = /(?<=[.!?])\s+|\n+/
+
+      # Chunk boundaries across languages. Ruby is the native target so
+      # def/class/module are accepted at any indent (Ruby nests methods inside
+      # classes/modules); the cross-language alternatives (php/js/ts function,
+      # ts export, Object Pascal unit/routine headers) mirror the Python port so
+      # the SAME index can hold .rb, .py, .php, .js, .ts, .pas … files.
+      TOPLEVEL = Regexp.new(
+        '\A(?:' \
+          '\s*(?:async )?def |\s*class |\s*module |@\w' \
+          '|\s*(?:public |private |protected |static |final |abstract )*function \w' \
+          '|(?:export )?(?:default )?(?:abstract )?class ' \
+          '|interface |trait ' \
+          '|export (?:const|function|interface|type|async) ' \
+          '|[Uu]nit |[Pp]rocedure |[Ff]unction |[Cc]onstructor |[Dd]estructor ' \
+          '|[Tt]ype$' \
+        ')'
+      )
+
+      # Lowercase, strip diacritics, join comma-grouped numbers, split camelCase.
+      # Applied symmetrically to the indexed body and to query tokens so matching
+      # is consistent: a query for "field" reaches "IntegerField".
+      def fold(text)
+        s = text.to_s.gsub(NUM_COMMA, "")
+        s = s.gsub(CAMEL, " ").downcase
+        s.unicode_normalize(:nfkd)
+         .encode(Encoding::ASCII, invalid: :replace, undef: :replace, replace: "")
+      end
+
+      # Fold simple plurals: strip one trailing 's', never from ss/us/is endings
+      # (class, status, axis). QUERY-side only — so "fields" also reaches "Field".
+      def light_stem(token)
+        if token.length > 3 && token.end_with?("s") && !token.end_with?("ss", "us", "is")
+          token[0..-2]
+        else
+          token
+        end
+      end
+
+      # Accent-folded lowercase alphanumeric tokens (digits kept).
+      def terms(text)
+        fold(text).scan(WORD_RE)
+      end
+
+      # Pack line-segments into chunks of at most max_lines lines, hard-splitting
+      # any single oversized segment.
+      def pack(segments, max_lines)
+        chunks = []
+        cur = []
+        segments.each do |seg|
+          seg = seg.dup
+          while seg.length > max_lines # oversized segment: hard split
+            unless cur.empty?
+              chunks << cur
+              cur = []
+            end
+            chunks << seg[0...max_lines]
+            seg = seg[max_lines..] || []
+          end
+          if !cur.empty? && cur.length + seg.length > max_lines
+            chunks << cur
+            cur = []
+          end
+          cur += seg
+        end
+        chunks << cur unless cur.empty?
+        chunks
+      end
+
+      # Chunk source on top-level def/class/module boundaries (sentence chunking
+      # shreds code). Segments are packed up to max_lines, and every chunk starts
+      # with a "# file: <path>" line so the path's tokens are indexed — "where is
+      # the router?" should match core/router.rb by name.
+      #
+      # Returns an array of [index, chunk_text] pairs.
+      def chunk_code(text, path: "", max_lines: 60)
+        lines = text.to_s.split("\n", -1)
+        lines.pop if lines.last == "" # drop trailing empty from a final newline
+        bounds = []
+        lines.each_with_index { |ln, i| bounds << i if TOPLEVEL.match?(ln) }
+        bounds = [0] + bounds if bounds.empty? || bounds.first != 0
+
+        ends = bounds[1..] + [lines.length]
+        segments = bounds.zip(ends).map { |a, b| lines[a...b] }
+
+        header = path.to_s.empty? ? nil : "# file: #{path}"
+        out = []
+        pack(segments, max_lines).each_with_index do |ch, i|
+          body = ((header ? [header] : []) + ch).join("\n")
+          out << [i, body]
+        end
+        out
+      end
+
+      # Chunk prose/docs into sentence-packed windows of at most max_words words.
+      # Returns an array of [index, chunk_text] pairs.
+      def chunk_text(text, max_words: 350)
+        sents = text.to_s.split(SENT).map(&:strip).reject(&:empty?)
+        chunks = []
+        cur = []
+        cur_words = 0
+        sents.each do |s|
+          w = s.split.length
+          if !cur.empty? && cur_words + w > max_words
+            chunks << cur.join(" ")
+            cur = []
+            cur_words = 0
+          end
+          cur << s
+          cur_words += w
+        end
+        chunks << cur.join(" ") unless cur.empty?
+        chunks.each_with_index.map { |c, i| [i, c] }
+      end
+    end
+  end
+end

data/lib/tina4/context.rb

@@ -0,0 +1,406 @@
+# frozen_string_literal: true
+
+# Tina4 Ruby
+#
+# Context — a native, zero-dependency code/doc grounding index.
+#
+# Lets a Tina4 app ground its own AI assistant on its own source, offline: it
+# walks the project, chunks code on def/class/module boundaries and docs as
+# prose, and answers keyword/fuzzy queries over a SQLite FTS5 index. The
+# sqlite3 gem is already a Tina4-Ruby runtime dependency and modern SQLite ships
+# FTS5 + bm25() built in, so this adds NO new dependency.
+#
+# The retrieval core is a thin port of tina4-python's tina4_python/context
+# (itself the proven slice of neemee: SqliteFTS + the stable
+# source-over-tests / definition-first reorderings). It COMPLEMENTS the api_*
+# reflection tools: api_* is exact structural lookup, Context is fuzzy/semantic
+# FTS over source + docs.
+#
+#   ctx = Tina4::Context.new(".tina4/context.db")
+#   ctx.index_root("src")
+#   ctx.search("where is the auth token issued?", k: 5)
+#   #  -> [{ path: "auth.rb", score: 2.31, snippet: "..." }, ...]
+#
+# On-disk index defaults to ".tina4/context.db" (gitignored). Guards a sqlite
+# build without FTS5: if absent, the Context degrades to safe no-ops rather than
+# crashing the app.
+
+require "pathname"
+require "fileutils"
+require "set"
+require_relative "context/chunker"
+
+module Tina4
+  class Context
+    # File classification — mirrors the Python port's repo walk.
+    CODE_EXTS = %w[.py .php .js .mjs .ts .rb .pas .dpr .dpk .inc .dfm .fmx].freeze
+    DOC_EXTS  = %w[.md .txt .rst .twig .html].freeze
+    # deploy/CLI/env answers live in config files, not sources — chunk as code
+    # (line windows), since sentence chunking shreds YAML/Dockerfiles.
+    CONFIG_EXTS = %w[.toml .yml .yaml].freeze
+    SPECIAL_FILES = %w[dockerfile makefile docker-compose.yml package.json
+                       composer.json gemfile .env.example .env.sample].freeze
+    # Same dirs the Python port skips, plus Tina4 runtime dirs that hold no
+    # source of truth (our own index/backups, session blobs, logs).
+    SKIP_DIRS = %w[.git __pycache__ node_modules vendor dist build coverage
+                   .idea .venv venv .pytest_cache .tina4 sessions logs].freeze
+
+    # Generic question/code vocabulary that never NAMES a symbol — kept small.
+    DEF_STOP = %w[the and what how does can which where when who why list all
+                  available module class function functions method methods def
+                  get set new return import from with that this are is was for
+                  into use used].to_set.freeze
+
+    # A chunk that DEFINES a queried symbol ("def get_token", "class Widget",
+    # "module Auth") should out-rank one that merely USES it. Matched against
+    # the folded body.
+    DEF_KW = '(?:async def|def|class|module|function|fn|func|interface|trait)'
+
+    attr_reader :path, :available, :root
+
+    # `path` is the on-disk index file (its parent dir is created). `fts5_check`
+    # overrides FTS5 detection (used by tests to exercise the graceful-
+    # degradation path); defaults to a real probe of this build.
+    def initialize(path = "./.tina4/context.db", fts5_check: nil)
+      @path = path.to_s
+      @lock = Mutex.new
+      @conn = nil
+      @root = nil # set by index_root; reindex_file relabels against it
+      check = fts5_check || -> { self.class.fts5_available? }
+      @available = !!check.call
+      return unless @available
+
+      require "sqlite3"
+      parent = File.dirname(@path)
+      FileUtils.mkdir_p(parent) unless parent.empty? || parent == "." || File.directory?(parent)
+      # One connection guarded by a Mutex — the dev-MCP reload hook may run on a
+      # Puma worker thread, so access is serialized rather than same-thread-only.
+      @conn = SQLite3::Database.new(@path)
+      ensure_table
+    end
+
+    # ── FTS5 availability ───────────────────────────────────────
+    # Whether this Ruby's sqlite3 build supports FTS5 (a real in-memory probe).
+    def self.fts5_available?
+      require "sqlite3"
+      db = SQLite3::Database.new(":memory:")
+      begin
+        db.execute("CREATE VIRTUAL TABLE _probe USING fts5(x)")
+        true
+      rescue SQLite3::Exception
+        false
+      ensure
+        db.close
+      end
+    rescue LoadError, StandardError
+      false
+    end
+
+    # ── schema ──────────────────────────────────────────────────
+    def ensure_table
+      # `body` holds fold(text) so query/index tokenization is symmetric; `raw`
+      # (UNINDEXED) keeps the original text to return as a snippet; `path`/`cid`
+      # (UNINDEXED) are metadata used for upsert + citation.
+      @conn.execute(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS chunks " \
+        "USING fts5(cid UNINDEXED, path UNINDEXED, raw UNINDEXED, body)"
+      )
+    end
+    private :ensure_table
+
+    # Drop and recreate the index (full rebuild starting point).
+    def reset
+      return unless @available
+
+      @lock.synchronize do
+        @conn.execute("DROP TABLE IF EXISTS chunks")
+        ensure_table
+      end
+    end
+
+    # ── indexing ────────────────────────────────────────────────
+    # [index, chunk_text] pairs — code/config files on def/class boundaries,
+    # docs as prose.
+    def self.chunks_for(label, text)
+      ext = File.extname(label).downcase
+      special = SPECIAL_FILES.include?(File.basename(label).downcase)
+      if CODE_EXTS.include?(ext) || CONFIG_EXTS.include?(ext) || special
+        Chunker.chunk_code(text, path: label)
+      else
+        Chunker.chunk_text(text)
+      end
+    end
+
+    # UPSERT one file into the index: delete this path's existing chunks,
+    # re-chunk the current contents, insert. A single file save re-indexes just
+    # that file. `label` is the stored/citation path (defaults to `file`) and
+    # MUST be stable across calls for the same file so the delete targets the
+    # right rows. Returns rows inserted.
+    def index_path(file, label: nil)
+      return 0 unless @available
+
+      stored = as_text(label.nil? ? file : label)
+      begin
+        text = File.read(file, encoding: "UTF-8", invalid: :replace, undef: :replace, replace: "")
+      rescue SystemCallError
+        return 0
+      end
+      rows = self.class.chunks_for(stored, text).map do |i, chunk|
+        ["#{stored}:#{i}", stored, chunk, Chunker.fold(chunk)]
+      end
+      @lock.synchronize do
+        @conn.execute("DELETE FROM chunks WHERE path = ?", [stored])
+        unless rows.empty?
+          stmt = @conn.prepare("INSERT INTO chunks(cid, path, raw, body) VALUES (?, ?, ?, ?)")
+          begin
+            rows.each { |r| stmt.execute(r) }
+          ensure
+            stmt.close
+          end
+        end
+      end
+      rows.length
+    end
+
+    # True if a file should be indexed (the per-file filter used by both
+    # index_root and reindex_file). Directory skipping is handled separately.
+    def self.eligible?(filename)
+      fn = filename.downcase
+      return false if fn.end_with?(".min.js")
+
+      ext = File.extname(fn)
+      CODE_EXTS.include?(ext) || DOC_EXTS.include?(ext) ||
+        CONFIG_EXTS.include?(ext) || SPECIAL_FILES.include?(fn)
+    end
+
+    # Walk `root`, indexing every eligible file (skips vendor/build/runtime
+    # dirs). Paths are stored RELATIVE to `root` for clean citations. Records
+    # `root` so reindex_file can relabel a changed file consistently. Returns
+    # the total number of chunks inserted.
+    def index_root(root)
+      return 0 unless @available
+
+      root = canonical_dir(root)
+      @root = root
+      total = 0
+      walk(root) do |full|
+        rel = Pathname.new(full).relative_path_from(Pathname.new(root)).to_s
+        total += index_path(full, label: rel)
+      end
+      total
+    end
+
+    # Re-index a single changed file into the LIVE index — the hook the dev
+    # reload trigger (POST /__dev/api/reload) calls so code_search tracks edits
+    # without a rebuild. Resolves `changed_path` against the indexed root, then:
+    # outside root / under a skip-or-dot dir / ineligible -> skip (-1);
+    # deleted -> drop its chunks (0); otherwise UPSERT (rows). No-op (-1) until
+    # index_root has run (nothing to keep fresh yet).
+    def reindex_file(changed_path)
+      return -1 unless @available && @root
+
+      p = changed_path.to_s
+      # The reload trigger reports paths relative to the project root (cwd during
+      # `tina4 serve`); the index root may be a subdir like src/.
+      p = File.join(Dir.pwd, p) unless Pathname.new(p).absolute?
+      resolved = canonical_path(p)
+      begin
+        rel_pn = Pathname.new(resolved).relative_path_from(Pathname.new(@root))
+      rescue ArgumentError
+        return -1 # different mount/drive — outside the indexed root
+      end
+      parts = rel_pn.each_filename.to_a
+      return -1 if parts.first == ".." # outside the indexed root
+      return -1 if parts.any? { |pt| SKIP_DIRS.include?(pt) }
+      return -1 if parts[0...-1].any? { |pt| pt.start_with?(".") }
+      return -1 unless self.class.eligible?(rel_pn.basename.to_s)
+
+      stored = as_text(rel_pn.to_s)
+      unless File.exist?(resolved) # deleted → drop its chunks
+        @lock.synchronize { @conn.execute("DELETE FROM chunks WHERE path = ?", [stored]) }
+        return 0
+      end
+      index_path(resolved, label: stored)
+    end
+
+    # ── query ───────────────────────────────────────────────────
+    # Build the FTS5 MATCH string: each folded query token (plus its light stem)
+    # as a quoted OR term. Quoting keeps it injection-safe.
+    def match_expr(query)
+      toks = []
+      Chunker.terms(query).each do |t|
+        toks << t
+        toks << Chunker.light_stem(t)
+      end
+      toks = toks.uniq.reject(&:empty?)
+      return nil if toks.empty?
+
+      toks.sort.map { |t| %("#{t}") }.join(" OR ")
+    end
+    private :match_expr
+
+    def self.testlike?(path)
+      s = (path || "").downcase
+      ["/test", "test/", "test_", "_test.", ".test.", "/example", "example/"].any? { |p| s.include?(p) }
+    end
+
+    def defines?(folded_body, symbols)
+      return false if symbols.empty?
+
+      alt = symbols.map { |s| Regexp.escape(s) }.join("|")
+      pat = Regexp.new("#{DEF_KW}\\s+(?:#{alt})(?![a-z0-9])")
+      !pat.match(folded_body).nil?
+    end
+    private :defines?
+
+    # Return the top-`k` chunks as [{ path:, score:, snippet: }], ranked by
+    # bm25() then reordered with two stable, proven passes:
+    #   - source-over-tests: a test that merely mentions a symbol sinks below the
+    #     source that defines it (skipped when the query is about tests);
+    #   - definition-first: a chunk that DEFINES a queried symbol rises above
+    #     chunks that only use it.
+    # Score is a higher-is-better float (sqlite's bm25 sign flipped).
+    def search(query, k: 5)
+      return [] unless @available
+
+      expr = match_expr(query)
+      return [] if expr.nil?
+
+      pool_n = [k * 3, 15].max
+      rows = @lock.synchronize do
+        @conn.execute(
+          "SELECT path, raw, body, bm25(chunks) AS s FROM chunks " \
+          "WHERE chunks MATCH ? ORDER BY s LIMIT ?",
+          [expr, pool_n]
+        )
+      end
+      return [] if rows.empty?
+
+      # candidate symbol names from the query (drop generic vocab).
+      symbols = Chunker.terms(query).select { |t| t.length >= 3 && !DEF_STOP.include?(t) }.uniq
+      about_tests = query.downcase.include?("test")
+
+      ordered = rows.each_with_index.sort_by do |row, idx|
+        path, _raw, body, = row
+        is_test = (!about_tests && self.class.testlike?(path)) ? 1 : 0
+        is_def  = defines?(body, symbols) ? 0 : 1
+        [is_test, is_def, idx] # bm25 order (idx) breaks ties
+      end
+
+      ordered.first(k).map do |row, _idx|
+        path, raw, _body, s = row
+        { path: path, score: (-s.to_f).round(6), snippet: snippet(raw) }
+      end
+    end
+
+    def snippet(raw, limit: 280)
+      text = (raw || "").strip
+      return text if text.length <= limit
+
+      "#{text[0...limit].rstrip} ..."
+    end
+    private :snippet
+
+    # ── misc ────────────────────────────────────────────────────
+    def count
+      return 0 unless @available
+
+      @lock.synchronize { @conn.execute("SELECT count(*) FROM chunks").first.first }
+    end
+
+    def empty?
+      count.zero?
+    end
+
+    def close
+      return if @conn.nil?
+
+      @lock.synchronize do
+        @conn.close
+        @conn = nil
+      end
+    end
+
+    # Tag a string as UTF-8 TEXT for SQLite binding. Path strings from
+    # Pathname#relative_path_from / File.realpath arrive as ASCII-8BIT (BINARY),
+    # which the sqlite3 gem binds as a BLOB — and in SQLite a BLOB never equals a
+    # TEXT column value, so `WHERE path = ?` would silently match nothing and the
+    # UPSERT's delete would leave stale rows behind. Re-tagging the same bytes as
+    # UTF-8 makes them bind as TEXT; invalid byte sequences are scrubbed so the
+    # bind can never raise. Filesystem paths on the supported platforms are UTF-8.
+    def as_text(str)
+      s = str.to_s.dup.force_encoding(Encoding::UTF_8)
+      s.valid_encoding? ? s : s.scrub("?")
+    end
+    private :as_text
+
+    # ── path canonicalization ───────────────────────────────────
+    # Resolve symlinks like Python's Path.resolve() so a root under /var and a
+    # changed path reported under /private/var (macOS) compare equal.
+    def canonical_dir(dir)
+      File.exist?(dir) ? File.realpath(dir) : File.expand_path(dir)
+    end
+    private :canonical_dir
+
+    def canonical_path(p)
+      return File.realpath(p) if File.exist?(p)
+
+      d = File.dirname(p)
+      d = File.realpath(d) if File.exist?(d)
+      File.join(d, File.basename(p))
+    end
+    private :canonical_path
+
+    # Top-down walk that prunes skip-dirs and dotdirs, yielding each eligible
+    # file's absolute path in sorted order (mirrors the Python os.walk prune).
+    def walk(dir, &blk)
+      children = Dir.children(dir).sort
+      files = children.select { |c| File.file?(File.join(dir, c)) }.sort
+      files.each do |fn|
+        blk.call(File.join(dir, fn)) if self.class.eligible?(fn)
+      end
+      subdirs = children.select do |c|
+        full = File.join(dir, c)
+        File.directory?(full) && !SKIP_DIRS.include?(c) && !c.start_with?(".")
+      end
+      subdirs.sort.each { |d| walk(File.join(dir, d), &blk) }
+    end
+    private :walk
+
+    # ── process-wide shared index ─────────────────────────────────
+    # code_search (dev MCP) and the dev-reload reindex hook must share ONE index
+    # so a saved file is immediately searchable. Keyed by resolved db path.
+    @shared_contexts = {}
+
+    class << self
+      def db_key(db = nil)
+        base = db ? db.to_s : File.join(Dir.pwd, ".tina4", "context.db")
+        File.expand_path(base)
+      end
+
+      # Get (or create) the process-wide Context at `db` (default
+      # <cwd>/.tina4/context.db). If `root` is given and the index is empty,
+      # builds it once. This is what code_search uses so the reload hook can keep
+      # the SAME index fresh.
+      def default_context(root: nil, db: nil)
+        key = db_key(db)
+        ctx = (@shared_contexts[key] ||= new(key))
+        ctx.index_root(root) if !root.nil? && ctx.available && ctx.empty?
+        ctx
+      end
+
+      # Return the already-created shared Context for `db` (or nil). Used by the
+      # reload hook so a file change reindexes an EXISTING index but never
+      # creates one on its own (nothing to keep fresh until code_search runs).
+      def existing_context(db: nil)
+        @shared_contexts[db_key(db)]
+      end
+
+      # Test/teardown seam — forget every shared Context.
+      def clear_shared_contexts!
+        @shared_contexts.each_value { |c| c.close rescue nil }
+        @shared_contexts = {}
+      end
+    end
+  end
+end

data/lib/tina4/dev_admin.rb

@@ -413,6 +413,19 @@ module Tina4
           rescue StandardError => e
             Tina4::Log.error("Re-discover on reload failed: #{e.message}")
           end
+          # Keep the code Context index LIVE on the same reload trigger: reindex
+          # just the changed file (UPSERT) so the dev-MCP code_search reflects
+          # the edit immediately. Only touches an already-built index
+          # (existing_context never creates one); guarded so a context failure
+          # never breaks the reload.
+          begin
+            unless @reload_file.to_s.empty?
+              ctx = Tina4::Context.existing_context
+              ctx&.reindex_file(@reload_file)
+            end
+          rescue StandardError => e
+            Tina4::Log.error("Context reindex on reload failed: #{e.message}")
+          end
           # WebSocket-primary reload: push an instant message to every browser
           # connected on /__dev_reload. The toolbar client (and the dev-admin
           # dashboard) act on this immediately — the mtime poll above is only a

data/lib/tina4/mcp.rb

@@ -1339,6 +1339,28 @@ module Tina4
         Tina4::Docs.cached(project_root).method_spec(class_name.to_s, name.to_s)
       }, "Single method spec (signature, summary, file, line) from the live API index")
 
+      # ── Code/doc grounding (semantic FTS over this repo's source) ──
+      # code_search is the fuzzy DUAL of the structural api_* tools:
+      # api_* = exact signature lookup; code_search = "where/how is X done in
+      # THIS codebase?" over source + docs (zero-dep SQLite FTS5). Backed by a
+      # PROCESS-WIDE shared Context at .tina4/context.db so the dev-reload hook
+      # keeps the SAME index fresh on every save. First call (or rebuild:true)
+      # indexes src/ (falls back to the project root).
+      code_root = -> { File.directory?(File.join(project_root, "src")) ? File.join(project_root, "src") : project_root }
+      server.register_tool("code_search", lambda { |query:, k: 5, rebuild: false|
+        ctx = Tina4::Context.default_context(
+          root: code_root.call, db: File.join(project_root, ".tina4", "context.db")
+        )
+        unless ctx.available
+          next { "error" => "SQLite FTS5 is not available in this Ruby's sqlite3 build; code_search is disabled." }
+        end
+        if rebuild == true || rebuild.to_s == "true"
+          ctx.reset
+          ctx.index_root(code_root.call)
+        end
+        ctx.search(query.to_s, k: k.to_i)
+      }, "Fuzzy/semantic search over THIS project's source + docs (FTS5). Use for 'where/how is X done here?'; api_* for exact signatures.")
+
       # ── System Tools ──────────────────────────────────
       server.register_tool("system_info", lambda {
         {

data/lib/tina4/migration.rb

@@ -158,14 +158,16 @@ module Tina4
 
     # Resolve migrations directory: prefer src/migrations, fall back to migrations/
     def resolve_migrations_dir
-      src_dir = File.join(Dir.pwd, "src", "migrations")
-      return src_dir if Dir.exist?(src_dir)
-
+      # Canonical location is migrations/ (project root) — matches the Python reference, the CLI,
+      # and auto-migrate. A legacy src/migrations/ is honoured only as a fallback.
       root_dir = File.join(Dir.pwd, "migrations")
       return root_dir if Dir.exist?(root_dir)
 
-      # Default to src/migrations (will be created when needed)
-      src_dir
+      src_dir = File.join(Dir.pwd, "src", "migrations")
+      return src_dir if Dir.exist?(src_dir)
+
+      # Neither exists yet -> default to the canonical migrations/ (created when needed)
+      root_dir
     end
 
     def ensure_tracking_table

data/lib/tina4/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tina4
-  VERSION = "3.13.61"
+  VERSION = "3.13.63"
 end

data/lib/tina4.rb

@@ -51,6 +51,7 @@ require_relative "tina4/test_client"
 require_relative "tina4/test"
 require_relative "tina4/docs"
 require_relative "tina4/docstore"
+require_relative "tina4/context"
 require_relative "tina4/mcp"
 require_relative "tina4/realtime"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tina4ruby
 version: !ruby/object:Gem::Version
-  version: 3.13.61
+  version: 3.13.63
 platform: ruby
 authors:
 - Tina4 Team
@@ -296,6 +296,8 @@ files:
 - lib/tina4/cli.rb
 - lib/tina4/constants.rb
 - lib/tina4/container.rb
+- lib/tina4/context.rb
+- lib/tina4/context/chunker.rb
 - lib/tina4/cors.rb
 - lib/tina4/crud.rb
 - lib/tina4/database.rb