truffler 0.1.0 → 0.1.2

data/lib/truffler/search/query.rb

@@ -3,14 +3,15 @@ module Truffler
     # A normalized, tokenized search query. Quoted phrases stay one token.
     # Exact-text signals (quoted phrases, digit-bearing tokens, emails, and
     # identifier shapes such as `INV-4471` or `order_id`) are detected
-    # locally and never asked of Jev (R18).
+    # locally and never asked of Jev (R18), and so is a time phrase ("this
+    # week"), whose words are neither exact tokens nor keywords.
     class Query
       QUOTED = /"([^"]*)"/
       EMAIL = /\A[^@\s]+@[^@\s]+\.[a-z]{2,}\z/
       IDENTIFIER = /\A[a-z0-9]+(?:[-_.][a-z0-9]+)+\z/
       EDGE_PUNCTUATION = /\A[^\p{Alnum}@]+|[^\p{Alnum}]+\z/
 
-      attr_reader :raw, :normalized, :tokens, :exact_tokens
+      attr_reader :raw, :normalized, :tokens, :exact_tokens, :time_phrase
 
       def self.wrap(query)
         query.is_a?(self) ? query : new(query)
@@ -23,7 +24,18 @@ module Truffler
       def initialize(raw)
         @raw = raw.to_s
         @normalized = self.class.normalize(raw)
-        @tokens, @exact_tokens = tokenize(normalized)
+        @tokens, exact = tokenize(normalized)
+        @time_phrase = TimePhrase.find(@tokens)
+        @exact_tokens = exact.reject { |position| time_position?(position) }.map { |position| @tokens[position] }.freeze
+      end
+
+      def time_position?(position)
+        time_phrase&.positions&.include?(position) || false
+      end
+
+      # The tokens a keyword source may match: all but the time phrase.
+      def search_tokens
+        tokens.each_index.reject { |position| time_position?(position) }.map { |position| tokens[position] }
       end
 
       def blank?
@@ -44,19 +56,19 @@ module Truffler
             phrase = part.squish
             next if phrase.empty?
 
+            exact << tokens.size
             tokens << phrase
-            exact << phrase
           else
             part.split.each do |word|
               word = word.gsub(EDGE_PUNCTUATION, "")
               next if word.empty?
 
+              exact << tokens.size if exact?(word)
               tokens << word
-              exact << word if exact?(word)
             end
           end
         end
-        [ tokens.freeze, exact.freeze ]
+        [ tokens.freeze, exact ]
       end
 
       def exact?(word)

data/lib/truffler/search/result.rb

@@ -29,12 +29,14 @@ module Truffler
         records.map(&:id)
       end
 
-      # Applied filters, then boosts, as `{key:, label:, kind:, name:}`.
+      # Applied filters, then boosts, then the time range, as `{key:, label:, kind:, name:}`.
       def chips
         return [] unless encoding
 
         filters = encoding.filters.keys.map { |key| chip(key, :filter) }
-        filters + (encoding.boosts.keys - encoding.filters.keys).map { |key| chip(key, :boost) }
+        chips = filters + (encoding.boosts.keys - encoding.filters.keys).map { |key| chip(key, :boost) }
+        time = encoding.time
+        time ? chips + [ { key: TimeRange.key, label: TimeRange.key, kind: :time, name: time.name } ] : chips
       end
 
       def score(record)

data/lib/truffler/search/sql.rb

@@ -24,16 +24,18 @@ module Truffler
         @store = store
       end
 
-      # The caller's relation, ANDed with the tenant and the hard filters.
+      # The caller's relation, ANDed with the tenant, the time range, and the hard filters.
       def base(scope)
         scope = scope.where(definition.tenant_column => tenant_key) if definition.scoped?
+        scope = within_time(scope, encoding.time) if encoding.time
         encoding.filters.reduce(scope) { |relation, (key, threshold)| relation.where(Arel.sql(label_filter_sql(key, threshold))) }
       end
 
-      # Every record the query can return, before ranking.
+      # Every record the query can return, before ranking. Under a label
+      # filter the filter decides membership and text matches only rank.
       def candidates(scope)
         base = base(scope)
-        return base if label_only?
+        return base if label_only? || encoding.filters.any?
 
         conditions = [ keyword_sql, exact_sql, (text_candidate_sql if text_score_sql) ].compact
         base.where(Arel.sql(conditions.any? ? conditions.map { |condition| "(#{condition})" }.join(" OR ") : "1 = 0"))
@@ -104,6 +106,12 @@ module Truffler
         "#{label_column('record_type')} = #{quote(model.polymorphic_name)}#{tenant} AND #{label_column('record_id')} = #{primary_key}"
       end
 
+      def within_time(scope, range)
+        arrived_at = model.arel_table[definition.arrived_at_column]
+        scope = scope.where(arrived_at.gteq(range.from))
+        range.to ? scope.where(arrived_at.lt(range.to)) : scope
+      end
+
       def label_filter_sql(key, threshold)
         "EXISTS (SELECT 1 FROM #{quoted_labels} WHERE #{label_scope_sql} AND #{label_column('label_key')} = #{quote(key)} " \
           "AND #{label_column('value')} >= #{Float(threshold)})"

data/lib/truffler/search/time_phrase.rb

@@ -0,0 +1,87 @@
+module Truffler
+  module Search
+    # A time window named in a search query (R16, R18): computed here from
+    # the query tokens and a clock, never asked of Jev. Matching needs no
+    # clock, so the phrase is part of the query; `window(now)` resolves it
+    # to `[from, to]`, where `to` is nil for "until now".
+    #
+    #   today, yesterday, this week, last week, this month, last month,
+    #   past|last N hour(s)|day(s)|week(s)|month(s), past hour|week|month,
+    #   last hour, since monday..sunday
+    #
+    # "past/last N units" and "past week" are rolling: they end now and
+    # start N units back. "this/last week/month" are calendar windows.
+    # Weeks start on `Date.beginning_of_week` (Monday by default). Only the
+    # first phrase in a query counts; a quoted phrase is exact text.
+    TimePhrase = Data.define(:name, :positions, :resolver) do
+      def self.find(tokens)
+        tokens.each_index do |start|
+          PATTERNS.each do |pattern|
+            phrase = pattern.call(tokens, start)
+            return phrase if phrase
+          end
+        end
+        nil
+      end
+
+      def window(now)
+        resolver.call(now)
+      end
+
+      def range(now)
+        from, to = window(now)
+        TimeRange.new(name: name, from: from, to: to)
+      end
+
+      def self.fixed(words, name, &resolver)
+        lambda do |tokens, start|
+          new(name: name, positions: (start...start + words.size).to_a, resolver: resolver) if tokens[start, words.size] == words
+        end
+      end
+
+      UNITS = %i[hours days weeks months].to_h { |unit| [ unit.to_s, unit ] }
+        .merge(%i[hour day week month].to_h { |unit| [ unit.to_s, :"#{unit}s" ] }).freeze
+
+      def self.rolling(tokens, start)
+        return unless %w[past last].include?(tokens[start]) && tokens[start + 1].to_s.match?(/\A\d{1,3}\z/)
+
+        count = Integer(tokens[start + 1], 10)
+        unit = UNITS[tokens[start + 2]]
+        return unless unit && count.positive?
+
+        name = "Last #{count} #{count == 1 ? unit.to_s.singularize : unit}"
+        new(name: name, positions: [ start, start + 1, start + 2 ], resolver: ->(now) { [ now - count.public_send(unit), nil ] })
+      end
+
+      # "past hour", "past week", "past month", "last hour": one unit back
+      # from now. "last week" and "last month" stay calendar phrases.
+      def self.rolling_unit(tokens, start)
+        unit = tokens[start + 1]
+        return unless (tokens[start] == "past" && %w[hour week month].include?(unit)) || tokens[start, 2] == %w[last hour]
+
+        new(name: "#{tokens[start].capitalize} #{unit}", positions: [ start, start + 1 ],
+          resolver: ->(now) { [ now - 1.public_send(unit), nil ] })
+      end
+
+      def self.since(tokens, start)
+        wday = Date::DAYNAMES.map(&:downcase).index(tokens[start + 1]) if tokens[start] == "since"
+        return unless wday
+
+        new(name: "Since #{Date::DAYNAMES[wday]}", positions: [ start, start + 1 ],
+          resolver: ->(now) { [ (now - ((now.wday - wday) % 7).days).beginning_of_day, nil ] })
+      end
+
+      PATTERNS = [
+        method(:rolling),
+        method(:rolling_unit),
+        method(:since),
+        fixed(%w[today], "Today") { |now| [ now.beginning_of_day, nil ] },
+        fixed(%w[yesterday], "Yesterday") { |now| [ now.yesterday.beginning_of_day, now.beginning_of_day ] },
+        fixed(%w[this week], "This week") { |now| [ now.beginning_of_week, nil ] },
+        fixed(%w[last week], "Last week") { |now| [ now.prev_week.beginning_of_week, now.beginning_of_week ] },
+        fixed(%w[this month], "This month") { |now| [ now.beginning_of_month, nil ] },
+        fixed(%w[last month], "Last month") { |now| [ now.prev_month.beginning_of_month, now.beginning_of_month ] }
+      ].freeze
+    end
+  end
+end

data/lib/truffler/search/time_range.rb

@@ -0,0 +1,10 @@
+module Truffler
+  module Search
+    # A resolved time phrase: records whose `arrived_at` column is at or
+    # after `from` and, when `to` is set, before it. Shown as the chip
+    # `{key: "time", kind: :time}` and dropped by suppressing "time".
+    TimeRange = Data.define(:name, :from, :to) do
+      def self.key = "time"
+    end
+  end
+end

data/lib/truffler/smart_search/starter.rb

@@ -55,13 +55,13 @@ module Truffler
         pool = local_ids.first(depth)
         return pool if result.encoding_status == :cached || definition.labels.empty? || pool.size >= depth
 
-        pool + newest_in_scope(exclude: pool, limit: [ @config.smart_candidate_pool - pool.size, 0 ].max)
+        pool + newest_in_scope(exclude: pool, limit: [ @config.smart_candidate_pool - pool.size, 0 ].max, time: result.encoding&.time)
       end
 
-      def newest_in_scope(exclude:, limit:)
+      def newest_in_scope(exclude:, limit:, time:)
         return [] if limit.zero?
 
-        sql = Search::Sql.new(@model, tenant_key: keystroke.tenant_key, query: @query)
+        sql = Search::Sql.new(@model, tenant_key: keystroke.tenant_key, query: @query, encoding: Search::Encoding.new(time: time))
         relation = sql.base(keystroke.scope).where.not(@model.primary_key => exclude)
         column, direction = definition.order
         relation = column ? relation.reorder(column => direction) : relation.unscope(:order)

data/lib/truffler/version.rb

@@ -1,3 +1,3 @@
 module Truffler
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/truffler/vocabulary.rb

@@ -17,11 +17,13 @@ module Truffler
       @model = model
     end
 
-    # {key => LabelDefinition or Lenses::LensLabel}: the declared labels,
-    # then the lens labels keyed "lens:<id>:<label>".
+    # {key => LabelDefinition or Lenses::LensLabel}: the declared labels
+    # the tenant has (a per-tenant choice with no options for it is left
+    # out), then the lens labels keyed "lens:<id>:<label>".
     def labels_for(tenant_key: nil, user_key: nil, all_users: false)
+      declared = definition.per_tenant_vocabulary? ? definition.labels.select { |_, label| label.available?(tenant_key) } : definition.labels
       lenses = Lenses.labels(definition.model, tenant_key: tenant_key, user_key: user_key, all_users: all_users)
-      lenses.empty? ? definition.labels : definition.labels.merge(lenses)
+      lenses.empty? ? declared : declared.merge(lenses)
     end
 
     def fingerprint(label_key, tenant_key: nil)

data/lib/truffler.rb

@@ -19,6 +19,10 @@ loader.do_not_eager_load("#{__dir__}/truffler/lenses/ruby_llm_generator.rb")
 loader.setup
 
 module Truffler
+  # The answer to a choice question in query encoding that means "no
+  # option". The colon keeps it out of host option names, which reject it.
+  NO_OPTION = "truffler:none".freeze
+
   class << self
     def config
       @config ||= Configuration.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: truffler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Kieran Klaassen
@@ -198,6 +198,8 @@ files:
 - lib/generators/truffler/install/templates/channel.rb.tt
 - lib/generators/truffler/install/templates/initializer.rb.tt
 - lib/generators/truffler/install/templates/migration.rb.tt
+- lib/generators/truffler/upgrade/templates/backfill_spends_migration.rb.tt
+- lib/generators/truffler/upgrade/upgrade_generator.rb
 - lib/tasks/truffler.rake
 - lib/tasks/truffler/bench.rake
 - lib/tasks/truffler/suggestions.rake
@@ -283,6 +285,7 @@ files:
 - lib/truffler/query_encoding/prefetch.rb
 - lib/truffler/questions.rb
 - lib/truffler/railtie.rb
+- lib/truffler/records/backfill_spend.rb
 - lib/truffler/records/embedding.rb
 - lib/truffler/records/label.rb
 - lib/truffler/records/query_miss.rb
@@ -295,6 +298,8 @@ files:
 - lib/truffler/search/query.rb
 - lib/truffler/search/result.rb
 - lib/truffler/search/sql.rb
+- lib/truffler/search/time_phrase.rb
+- lib/truffler/search/time_range.rb
 - lib/truffler/smart_search.rb
 - lib/truffler/smart_search/dispatcher.rb
 - lib/truffler/smart_search/reranker.rb