klee 0.1.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ad4060f69da403186a89fb198ddfdb917606eac5e3344dcb5b0cc58f3d1bdd8
-  data.tar.gz: 90b540e449684b48e71714194b3c622f9fd35ecae931cb4b94fb53545f338958
+  metadata.gz: c8efdb5e1472295c0778a374828e8af7cb49e741a281d4d7558790863221ee0b
+  data.tar.gz: 99c547540505671159ace42750ba0ba6708aeb16f2888276a60426fc11316176
 SHA512:
-  metadata.gz: 46ed4f0a4469aee13b165138f3ff04da7b30819e6e37b01524e750df591a52cf6da53dc2f082a8c353a1bfa6a47e37111814baf8bbcdd377f246797ea12ec283
-  data.tar.gz: 77af3a015eb3e3fd8267b78525e46f53bb51694b2160d12a04e05bf6c04734549ed628a9cb5a8955cf4c9d6d4ec3ac1a544b011556125fdd5d07d2e1b6b71037
+  metadata.gz: 4009d050d3439f95c2d451f7ec9a5c56f407c0fe95b269fb802335c2262886cb2bb0e159d71f7068aaa4f35c87836c366ee8745421d55e425e5791d110530df0
+  data.tar.gz: 68ffb23b66ae05b66874e677ca47bc3ed96274f681d9d0e9f3f1c84726db35851cb2f3657404690b80189284c8b1460a96d57339b9106175146f3200dae97016

data/CHANGELOG.md

@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.0.0] - 2026-09-19
+
+### Added
+
+- Shared Words tokenizer with acronym splitting and predicate stripping (acff7d2)
+- Nested class names, attr_* methods, and constant collaborators (18d27de)
+- Klee.scan(profile: :rails) for models/lib globs and English stopwords (801fab2)
+- ConceptIndex#catalog, #gaps, and #units (801fab2)
+
+### Changed
+
+- Concepts#[] returns a Hash of concept words to method names (acff7d2)
+- FileAnalyzer records each message receiver, not only locals and ivars (18d27de)
+- Klee::Collaborators wraps FileAnalyzer instead of lexing the file (18d27de)
+
+### Fixed
+
+- Klee.concepts now forwards the modifiers: keyword (acff7d2)
+- Operator messages, block params, and conversion chains are ignored (18d27de)
+
 ## [0.1.2] - 2026-02-01
 
 ### Added
@@ -20,22 +40,3 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 
 - README now includes MCP server setup and available tools (f84bdc5)
-
-## [0.1.1] - 2026-01-29
-
-### Changed
-
-- Switch to supporting ruby 3.4 and up. (84fdd9c)
-- Require ruby 3.4 and above (067ec87)
-
-### Added
-
-- Reissue for version and release support. (885fe28)
-- SimpleCov for coverage reporting (889d169)
-- Coverage comment from PRs (067ec87)
-- Klee.scan() method for codebase-wide analysis with glob patterns (3e509a1)
-- ConceptIndex for aggregating domain concepts across files (3e509a1)
-- CollaboratorIndex for pairwise co-occurrence and clustering (3e509a1)
-- FileAnalyzer using Prism AST to extract classes, methods, and collaborators (3e509a1)
-- Configurable threshold and ignore list for filtering noise (3e509a1)
-- Method-level scope option for tighter coupling analysis (3e509a1)

data/README.md

@@ -45,11 +45,14 @@ end
 
 concept = Klee.object_concepts(Something)
 
-concept[4] #=> Set of words that appear at least 4 times
-concept[2] #=> larger Set of words that appear at least 2 times
+concept[4]
+# => { banana: [:banana_berry, :banana_boat, :banana_bunch, :banana_hammock], ... }
+
+concept[3][:address]
+# => [:address_informal_name, :address_street, :address_zip]
 
 filtered = Klee.object_concepts(Something, modifiers: %i[fill_in_ hover_over_ _message])
-filtered[4] #=> Set of concepts excluding any common modifiers
+filtered[4] # modifiers stripped before words are counted
 ```
 
 ### Scan a codebase for domain concepts
@@ -77,6 +80,31 @@ codebase = Klee.scan("app/**/*.rb",
                      threshold: 3)
 ```
 
+### Rails profile
+
+Skip views, drop English function words, and get the extract-type read model:
+
+```ruby
+cb = Klee.scan(profile: :rails) # app/models/**/*.rb + lib/**/*.rb from cwd
+
+cb.concepts.catalog.first
+# => #<data Klee::ConceptIndex::Entry word="intensity" class_count=71 method_count=29 bias=:type>
+
+cb.concepts.gaps
+# => { "seconds" => { classes: 0, methods: 95, names: #<Set: {"trailing_rest_seconds", ...}> }, ... }
+
+cb.concepts.units
+# => { "seconds" => ["cap_clock_seconds", "trailing_rest_seconds", ...], "minutes" => [...], "kg" => [...] }
+```
+
+`catalog` bias is `:type` (class-heavy), `:verb` (`build`/`calculate`/`extract`/…), `:gap` (many methods, almost no classes), or `:mixed`.
+
+Pass explicit globs with the profile to keep the stopwords:
+
+```ruby
+Klee.scan("app/models/**/*.rb", profile: :rails, threshold: 4)
+```
+
 ### Find collaborator clusters
 
 Discover which objects frequently work together across your codebase.

data/lib/klee/collaborators.rb

@@ -1,41 +1,31 @@
 # frozen_string_literal: true
 
-require "prism"
-
 module Klee
   class Collaborators
     def initialize(const)
-      @const = const
-    end
-    attr_reader :const
-
-    def lexed
-      Prism.lex_file(Object.const_source_location(const.name).first)
-    end
-
-    def additional_processing_types = [:DOT, :BRACKET_LEFT]
+      unless const.is_a?(Module)
+        raise ArgumentError, "expected a class or module, got #{const.inspect}"
+      end
 
-    def identifier_with_additional_processing
-      lexed.value.each_cons(2).filter_map do |(token, next_token)|
-        if token.first.type == :IDENTIFIER &&
-            additional_processing_types.include?(next_token.first.type)
-          token.first.value
-        end
+      path = Object.const_source_location(const.name)&.first
+      unless path && File.file?(path)
+        raise Error, "cannot locate source for #{const}"
       end
+
+      @const = const
+      @analyzer = FileAnalyzer.new(path)
     end
+    attr_reader :const
 
     def tally
-      identifier_with_additional_processing.tally
+      @analyzer.collaborators.tally
     end
 
     def rank
-      tally.group_by do |_, count|
-        count
-      end.sort.reverse.to_h.transform_values do |value|
-        value.map(&:first)
-      end.reject do |key, value|
-        value.empty? || key < 3
-      end
+      tally.group_by { |_, count| count }
+        .sort.reverse.to_h
+        .transform_values { |value| value.map(&:first) }
+        .reject { |key, _value| key < 3 }
     end
   end
 end

data/lib/klee/concept_index.rb

@@ -1,9 +1,28 @@
 # frozen_string_literal: true
 
+require_relative "words"
+
 module Klee
   class ConceptIndex
     include Enumerable
 
+    Entry = Data.define(:word, :class_count, :method_count, :class_names, :method_names, :bias)
+
+    VERBS = %w[
+      build calculate extract apply compute detect update create find get set
+      add ensure render save destroy validate serialize
+      parse process fetch refresh call fill infer display generate
+      normalize coerce prepare predict split sanitize mark
+    ].freeze
+
+    UNIT_PATTERNS = {
+      "seconds" => /_seconds\z/,
+      "minutes" => /_minutes\z/,
+      "kg" => /_kg\z/,
+      "m" => /_m\z/,
+      "reps" => /_reps\z/
+    }.freeze
+
     def initialize(ignore: [], threshold: 2)
       @ignore = Set.new(ignore.map(&:to_s))
       @threshold = threshold
@@ -32,16 +51,60 @@ module Klee
       filtered.sort_by { |word, locs| -(locs[:classes].size + locs[:methods].size) }.to_h
     end
 
+    def catalog
+      rank.map do |word, locs|
+        Entry.new(
+          word: word,
+          class_count: locs[:classes].size,
+          method_count: locs[:methods].size,
+          class_names: locs[:classes],
+          method_names: locs[:methods],
+          bias: bias_for(word, locs)
+        )
+      end
+    end
+
+    def gaps(min_methods: 8, max_classes: 3)
+      filtered.each_with_object({}) do |(word, locs), acc|
+        next if VERBS.include?(word)
+        next if locs[:methods].size < min_methods
+        next if locs[:classes].size > max_classes
+
+        acc[word] = {
+          classes: locs[:classes].size,
+          methods: locs[:methods].size,
+          names: locs[:methods]
+        }
+      end
+    end
+
+    def units
+      names = @data.each_value.flat_map { |locs| locs[:methods].to_a }.uniq
+      UNIT_PATTERNS.each_with_object({}) do |(unit, pattern), acc|
+        matched = names.grep(pattern)
+        acc[unit] = matched.sort if matched.size >= @threshold
+      end
+    end
+
     private
 
     def words_from(name)
-      name.to_s.gsub(/([a-z])([A-Z])/, '\1_\2')
-        .downcase.split("_")
-        .reject { |w| w.empty? || @ignore.include?(w) }
+      Words.from(name, ignore: @ignore)
     end
 
     def filtered
       @data.select { |_, locs| (locs[:classes].size + locs[:methods].size) >= @threshold }
     end
+
+    def bias_for(word, locs)
+      return :verb if VERBS.include?(word)
+
+      class_count = locs[:classes].size
+      method_count = locs[:methods].size
+      return :type if class_count > method_count
+      return :gap if method_count >= 8 && class_count <= 3
+
+      :mixed
+    end
   end
 end

data/lib/klee/concepts.rb

@@ -1,26 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "words"
+
 module Klee
   class Concepts
     include Enumerable
 
     def initialize(*method_names, modifiers: [])
       @method_names = method_names
-      @modifiers = modifiers
+      @modifiers = Array(modifiers).map(&:to_s)
     end
     attr_reader :method_names, :modifiers
 
     def call(threshold)
-      warn "threshold is beyond the max count of #{max}" if threshold > max
-      warn "threshold is below the min count of #{min}" if threshold < min
-      clear
-      @ideas = Set.new samples
-        .select { |key, value|
-          if value.to_i >= threshold
-            key
-          end
-        }
-        .compact
-        .transform_keys(&:to_sym)
-        .keys
+      unless samples.empty?
+        warn "threshold is beyond the max count of #{max}" if threshold > max
+        warn "threshold is below the min count of #{min}" if threshold < min
+      end
+
+      method_groups.select { |_, methods| methods.size >= threshold }
     end
     alias_method :[], :call
 
@@ -38,10 +36,14 @@ module Klee
     end
 
     def max
+      return 0 if samples.empty?
+
       max_by { |_, v| v }.last
     end
 
     def min
+      return 0 if samples.empty?
+
       min_by { |_, v| v }.last
     end
 
@@ -51,22 +53,22 @@ module Klee
 
     private
 
+    def method_groups
+      groups = Hash.new { |h, k| h[k] = [] }
+      method_names.each do |name|
+        words(name).uniq.each { |word| groups[word.to_sym] << name.to_sym }
+      end
+      groups
+    end
+
     def modifier_matcher
-      @modifier_matcher ||= Regexp.new modifiers.map { Regexp.quote(it) }.join("|")
+      @modifier_matcher ||= Regexp.new(modifiers.map { Regexp.quote(it) }.join("|"))
     end
 
     def words(method_name)
-      method_name.to_s
-        .then do |string|
-          if modifiers.empty?
-            string
-          else
-            string.gsub(modifier_matcher, "")
-          end
-        end
-        .gsub(/\s/, "")
-        .split("_")
-        .delete_if { it.empty? }
+      string = method_name.to_s
+      string = string.gsub(modifier_matcher, "") unless modifiers.empty?
+      Words.from(string)
     end
   end
 end

data/lib/klee/file_analyzer.rb

@@ -4,6 +4,22 @@ require "prism"
 
 module Klee
   class FileAnalyzer
+    ATTR_METHODS = %i[attr attr_reader attr_writer attr_accessor].freeze
+    INDEX_OPERATORS = Set[:[], :[]=].freeze
+    OPERATORS = Set.new(
+      %i[! != !~ + +@ - -@ * / % ** << >> & | ^ ~ <=> > >= < <= == === =~] +
+      INDEX_OPERATORS.to_a
+    ).freeze
+    CHAIN_NOISE = %w[
+      to_s to_str to_i to_int to_a to_ary to_h to_hash to_f to_sym to_proc
+      intern itself dup clone freeze tap new
+      where includes joins order limit offset eager_load preload
+      unscope merge distinct group having lock reselect reorder
+      first last take count map select collect compact flatten uniq
+      sort sort_by reverse each
+      Hash Array Set Rails Data Logger Kernel Object
+    ].to_set.freeze
+
     attr_reader :path, :class_names, :method_names, :collaborators, :method_collaborators
 
     def initialize(path)
@@ -19,38 +35,114 @@ module Klee
 
     def parse
       result = Prism.parse_file(@path)
-      visit(result.value, current_method: nil)
+      visit(result.value, current_method: nil, namespace: [], block_params: Set.new)
     end
 
-    def visit(node, current_method:)
+    def visit(node, current_method:, namespace:, block_params:)
       case node
       when Prism::ClassNode, Prism::ModuleNode
-        @class_names << node.name.to_s
+        full_name = qualified_name(node, namespace)
+        @class_names << full_name
+        nested = full_name.split("::")
+        node.child_nodes.compact.each do |child|
+          visit(child, current_method: current_method, namespace: nested, block_params: block_params)
+        end
+        return
       when Prism::DefNode
         @method_names << node.name.to_s
         current_method = node.name.to_s
+      when Prism::BlockNode, Prism::LambdaNode
+        nested_params = block_params | Array(node.locals).map(&:to_s)
+        node.child_nodes.compact.each do |child|
+          visit(child, current_method: current_method, namespace: namespace, block_params: nested_params)
+        end
+        return
       when Prism::CallNode
-        name = extract_collaborator(node)
-        if name
-          @collaborators << name
-          @method_collaborators[current_method] << name if current_method
+        # Walk the receiver first so `user.account` records left-to-right.
+        if (receiver = node.receiver)
+          visit(receiver, current_method: current_method, namespace: namespace, block_params: block_params)
+        end
+        record_call(node, current_method, block_params)
+        node.child_nodes.compact.each do |child|
+          next if child.equal?(receiver)
+
+          visit(child, current_method: current_method, namespace: namespace, block_params: block_params)
         end
+        return
+      end
+
+      node.child_nodes.compact.each do |child|
+        visit(child, current_method: current_method, namespace: namespace, block_params: block_params)
+      end
+    end
+
+    def record_call(node, current_method, block_params)
+      if attr_call?(node)
+        @method_names.concat(attr_names(node))
+        return
       end
 
-      node.child_nodes.compact.each { |child| visit(child, current_method: current_method) }
+      return if skip_operator_call?(node)
+
+      name = extract_collaborator(node)
+      return if skip_collaborator?(name, block_params)
+
+      @collaborators << name
+      @method_collaborators[current_method] << name if current_method
     end
 
+    def skip_operator_call?(node)
+      OPERATORS.include?(node.name) && !INDEX_OPERATORS.include?(node.name)
+    end
+
+    def skip_collaborator?(name, block_params)
+      return true if name.nil? || block_params.include?(name)
+
+      key = name.to_sym
+      OPERATORS.include?(key) || CHAIN_NOISE.include?(name)
+    end
+
+    def attr_call?(node)
+      node.receiver.nil? && ATTR_METHODS.include?(node.name)
+    end
+
+    def attr_names(node)
+      args = node.arguments&.arguments || []
+      args.filter_map { |arg| attr_name_from(arg) }.map do |name|
+        (node.name == :attr_writer) ? "#{name}=" : name
+      end
+    end
+
+    def attr_name_from(arg)
+      case arg
+      when Prism::SymbolNode, Prism::StringNode
+        arg.unescaped
+      end
+    end
+
+    # A collaborator is the immediate receiver of a non-operator message.
     def extract_collaborator(call_node)
-      receiver = call_node.receiver
-      case receiver
+      case (receiver = call_node.receiver)
       when Prism::LocalVariableReadNode
         receiver.name.to_s
       when Prism::InstanceVariableReadNode
         receiver.name.to_s.delete_prefix("@")
       when Prism::CallNode
-        # Chain like user.account.name - extract the root
-        extract_collaborator(receiver)
+        receiver.name.to_s
+      when Prism::ConstantReadNode, Prism::ConstantPathNode
+        constant_name(receiver)
       end
     end
+
+    def qualified_name(node, namespace)
+      path = constant_name(node.constant_path)
+      return path if node.constant_path.is_a?(Prism::ConstantPathNode)
+
+      [*namespace, path].join("::")
+    end
+
+    def constant_name(node)
+      node.slice.delete_prefix("::")
+    end
   end
 end

data/lib/klee/gestalt.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Klee
   class Gestalt
     def initialize(object, patterns:, ignored:)
@@ -33,7 +35,7 @@ module Klee
       end
 
       plot["unusual"].merge(unusual_set(threshold))
-      plot["concepts"].merge(concepts(modifiers: modifiers, threshold: concept_threshold))
+      plot["concepts"].merge(concepts(modifiers: modifiers, threshold: concept_threshold).keys)
 
       self
     end

data/lib/klee/profile.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Klee
+  class Profile
+    RAILS_GLOBS = %w[app/models/**/*.rb lib/**/*.rb].freeze
+
+    STOPWORDS = %w[
+      a an the and or not to for from of in on at by as
+      is are was were been being have has had do does did
+      will would could should may might must can
+      this that these those it its they them we our you your
+      if else then when
+      with without via into over per vs all any
+      id ids key keys current value values
+    ].freeze
+
+    def self.resolve(name, patterns:, ignore:, threshold:)
+      case name&.to_sym
+      when nil
+        {patterns: patterns, ignore: Array(ignore), threshold: threshold}
+      when :rails
+        globs = patterns.empty? ? RAILS_GLOBS : patterns
+        {
+          patterns: globs,
+          ignore: (STOPWORDS + Array(ignore).map(&:to_s)).uniq,
+          threshold: threshold
+        }
+      else
+        raise ArgumentError, "unknown profile #{name.inspect}"
+      end
+    end
+  end
+end

data/lib/klee/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Klee
-  VERSION = "0.1.2"
+  VERSION = "1.0.0"
 end

data/lib/klee/words.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Klee
+  module Words
+    def self.from(name, ignore: [])
+      ignored = ignore.map(&:to_s)
+      tokenize(name).reject { |word| word.empty? || ignored.include?(word) }
+    end
+
+    def self.tokenize(name)
+      name.to_s
+        .gsub("::", "_")
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .downcase
+        .gsub(/[?!=]/, "")
+        .split(/[_\s]+/)
+    end
+  end
+end

data/lib/klee.rb

@@ -2,13 +2,15 @@
 
 # require "classifier-reborn"
 require_relative "klee/version"
+require_relative "klee/words"
 require_relative "klee/patterns"
 require_relative "klee/gestalt"
 require_relative "klee/concepts"
-require_relative "klee/collaborators"
 require_relative "klee/file_analyzer"
+require_relative "klee/collaborators"
 require_relative "klee/concept_index"
 require_relative "klee/collaborator_index"
+require_relative "klee/profile"
 require_relative "klee/codebase"
 
 module Klee
@@ -27,7 +29,7 @@ module Klee
   end
 
   def self.concepts(*method_names, modifiers: [])
-    Concepts.new(*method_names, modifiers: [])
+    Concepts.new(*method_names, modifiers: modifiers)
   end
 
   # def self.classifier
@@ -53,7 +55,8 @@ module Klee
     Klee::Collaborators.new(const)
   end
 
-  def self.scan(*patterns, ignore: [], threshold: 2)
-    Codebase.new(*patterns, ignore: ignore, threshold: threshold)
+  def self.scan(*patterns, ignore: [], threshold: 2, profile: nil)
+    spec = Profile.resolve(profile, patterns: patterns, ignore: ignore, threshold: threshold)
+    Codebase.new(*spec[:patterns], ignore: spec[:ignore], threshold: spec[:threshold])
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klee
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Jim Gay
@@ -60,7 +60,9 @@ files:
 - lib/klee/mcp/tools/find_concept_clusters.rb
 - lib/klee/mcp/version.rb
 - lib/klee/patterns.rb
+- lib/klee/profile.rb
 - lib/klee/version.rb
+- lib/klee/words.rb
 - sig/klee.rbs
 homepage: https://github.com/saturnflyer/klee
 licenses:
@@ -90,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.5
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Evaluate the similarities and differences in your objects
 test_files: []