whoosh 1.0.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d739583e7066d3f04675dbd6eb4e17809226c909f5c7d4d226b7672957e4a88
-  data.tar.gz: 5f12610ddf11479be668f3b0dc88de6f6c3f5640620299ca3512f48ffaf14922
+  metadata.gz: 133af39d011ac77fc6e0773ea19652cb990f5c8d129d40b7f301796062e23e1f
+  data.tar.gz: 5145a7c3177f2ddef7667f8faa6a6c2681b38fd279351c95ceda93ce69999c2a
 SHA512:
-  metadata.gz: 305d2ea3aac7ba3a22be62c52c996aca50956cce5a0983a8d38cc0f76a55a817c25bdb3fe8877127752a0f3d1e3d81d894c80dc395ae4da39d562d2fad58098f
-  data.tar.gz: 2cd02f1ab2374bd1334d6dc6bab05fc460e108081eadb8119c9082af41f522c7fb5a8e7f63781b9451aeb319cd001522b7d6998290c662b3c8dbf0706ac03308
+  metadata.gz: 4e99f75634ea05e7e2673164f6bdd8b71ea704e6ee02023370b8b77207666bee6291a02d7423d8d036c4014651ba17673c3f23639e64a2a68b5038799b1dca99
+  data.tar.gz: 4427cce1a58e9d35841a372ca782f831d6aa3ef513e2309f14e4f6b284a4d12461b102fe5a963768e1a01af2e98db7fc82f7084e970f1a150ef7092f2033c8bf

data/lib/whoosh/app.rb

@@ -29,6 +29,7 @@ module Whoosh
       auto_register_database
       auto_register_storage
       auto_register_http
+      auto_register_vectors
       auto_configure_jobs
       @metrics = Metrics.new
       auto_register_metrics
@@ -302,8 +303,12 @@ module Whoosh
       @di.provide(:http) { HTTP }
     end
 
+    def auto_register_vectors
+      @di.provide(:vectors) { VectorStore.build(@config.data) }
+    end
+
     def auto_configure_jobs
-      backend = Jobs::MemoryBackend.new
+      backend = Jobs.build_backend(@config.data)
       Jobs.configure(backend: backend, di: @di)
     end
 
@@ -321,7 +326,7 @@ module Whoosh
         worker = Jobs::Worker.new(
           backend: Jobs.backend, di: @di,
           max_retries: max_retries, retry_delay: retry_delay,
-          instrumentation: @instrumentation
+          instrumentation: @instrumentation, logger: @logger
         )
         thread = Thread.new { worker.run_loop }
         thread.abort_on_exception = false

data/lib/whoosh/cache.rb

@@ -1,4 +1,3 @@
-# lib/whoosh/cache.rb
 # frozen_string_literal: true
 
 module Whoosh
@@ -6,20 +5,21 @@ module Whoosh
     autoload :MemoryStore, "whoosh/cache/memory_store"
     autoload :RedisStore,  "whoosh/cache/redis_store"
 
+    # Auto-detect: REDIS_URL set → Redis, otherwise → Memory
     def self.build(config_data = {})
       cache_config = config_data["cache"] || {}
-      store = cache_config["store"] || "memory"
       default_ttl = cache_config["default_ttl"] || 300
+      redis_url = ENV["REDIS_URL"] || cache_config["url"]
 
-      case store
-      when "memory"
-        MemoryStore.new(default_ttl: default_ttl)
-      when "redis"
-        url = cache_config["url"] || "redis://localhost:6379"
-        pool_size = cache_config["pool_size"] || 5
-        RedisStore.new(url: url, default_ttl: default_ttl, pool_size: pool_size)
+      if redis_url && cache_config["store"] != "memory"
+        begin
+          RedisStore.new(url: redis_url, default_ttl: default_ttl)
+        rescue Errors::DependencyError
+          # Redis gem not installed, fall back to memory
+          MemoryStore.new(default_ttl: default_ttl)
+        end
       else
-        raise ArgumentError, "Unknown cache store: #{store}"
+        MemoryStore.new(default_ttl: default_ttl)
       end
     end
   end

data/lib/whoosh/cli/project_generator.rb

@@ -175,8 +175,10 @@ module Whoosh
               max_connections: 10
               log_level: debug
 
+            # Cache & Jobs auto-detect:
+            # No REDIS_URL → in-memory (just works)
+            # Set REDIS_URL → auto-switches to Redis
             cache:
-              store: memory
               default_ttl: 300
 
             jobs:
@@ -188,6 +190,12 @@ module Whoosh
               level: info
               format: json
 
+            # Vector store auto-detect:
+            # zvec gem installed → uses zvec, otherwise → in-memory
+            # vector:
+            #   adapter: auto
+            #   path: db/vectors
+
             docs:
               enabled: true
 

data/lib/whoosh/job.rb

@@ -1,4 +1,3 @@
-# lib/whoosh/job.rb
 # frozen_string_literal: true
 
 module Whoosh
@@ -12,9 +11,41 @@ module Whoosh
         @dependencies || []
       end
 
+      def queue(name = nil)
+        if name
+          @queue_name = name.to_s
+        else
+          @queue_name || "default"
+        end
+      end
+
+      def retry_limit(n = nil)
+        if n
+          @retry_limit = n
+        else
+          @retry_limit
+        end
+      end
+
+      def retry_backoff(strategy = nil)
+        if strategy
+          @retry_backoff = strategy
+        else
+          @retry_backoff || :linear
+        end
+      end
+
       def perform_async(**args)
         Jobs.enqueue(self, **args)
       end
+
+      def perform_in(delay_seconds, **args)
+        Jobs.enqueue(self, run_at: Time.now.to_f + delay_seconds, **args)
+      end
+
+      def perform_at(time, **args)
+        Jobs.enqueue(self, run_at: time.to_f, **args)
+      end
     end
 
     def perform(**args)

data/lib/whoosh/jobs/memory_backend.rb

@@ -1,4 +1,3 @@
-# lib/whoosh/jobs/memory_backend.rb
 # frozen_string_literal: true
 
 module Whoosh
@@ -6,6 +5,7 @@ module Whoosh
     class MemoryBackend
       def initialize
         @queue = []
+        @scheduled = []
         @records = {}
         @mutex = Mutex.new
         @cv = ConditionVariable.new
@@ -13,14 +13,25 @@ module Whoosh
 
       def push(job_data)
         @mutex.synchronize do
-          @queue << job_data
+          if job_data[:run_at] && job_data[:run_at] > Time.now.to_f
+            @scheduled << job_data
+            @scheduled.sort_by! { |j| j[:run_at] }
+          else
+            @queue << job_data
+          end
           @cv.signal
         end
       end
 
       def pop(timeout: 5)
         @mutex.synchronize do
-          @cv.wait(@mutex, timeout) if @queue.empty?
+          # Promote scheduled jobs that are ready
+          promote_scheduled
+
+          if @queue.empty?
+            @cv.wait(@mutex, timeout)
+            promote_scheduled
+          end
           @queue.shift
         end
       end
@@ -34,12 +45,37 @@ module Whoosh
       end
 
       def size
+        @mutex.synchronize { @queue.size + @scheduled.size }
+      end
+
+      def pending_count
         @mutex.synchronize { @queue.size }
       end
 
+      def scheduled_count
+        @mutex.synchronize { @scheduled.size }
+      end
+
       def shutdown
         @mutex.synchronize { @cv.broadcast }
       end
+
+      private
+
+      def promote_scheduled
+        now = Time.now.to_f
+        ready = []
+        remaining = []
+        @scheduled.each do |job|
+          if job[:run_at] <= now
+            ready << job
+          else
+            remaining << job
+          end
+        end
+        @scheduled = remaining
+        @queue.concat(ready)
+      end
     end
   end
 end

data/lib/whoosh/jobs/redis_backend.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Whoosh
+  module Jobs
+    class RedisBackend
+      @redis_available = nil
+
+      def self.available?
+        if @redis_available.nil?
+          @redis_available = begin
+            require "redis"
+            true
+          rescue LoadError
+            false
+          end
+        end
+        @redis_available
+      end
+
+      def initialize(url:, prefix: "whoosh:jobs")
+        unless self.class.available?
+          raise Errors::DependencyError, "Jobs Redis backend requires the 'redis' gem"
+        end
+        @redis = Redis.new(url: url)
+        @prefix = prefix
+      end
+
+      def push(job_data)
+        serialized = Serialization::Json.encode(job_data)
+        if job_data[:run_at] && job_data[:run_at] > Time.now.to_f
+          # Scheduled: use sorted set with run_at as score
+          @redis.zadd("#{@prefix}:scheduled", job_data[:run_at], serialized)
+        else
+          @redis.lpush("#{@prefix}:queue:#{job_data[:queue] || "default"}", serialized)
+        end
+      end
+
+      def pop(timeout: 5, queues: ["default"])
+        # First, promote scheduled jobs
+        promote_scheduled
+
+        # Try each queue in priority order
+        queues.each do |queue|
+          result = @redis.rpop("#{@prefix}:queue:#{queue}")
+          if result
+            return Serialization::Json.decode(result).transform_keys(&:to_sym)
+          end
+        end
+
+        # Block-wait on default queue
+        result = @redis.brpop("#{@prefix}:queue:#{queues.first}", timeout: timeout)
+        if result
+          Serialization::Json.decode(result[1]).transform_keys(&:to_sym)
+        end
+      rescue => e
+        nil
+      end
+
+      def save(record)
+        serialized = Serialization::Json.encode(record)
+        @redis.set("#{@prefix}:record:#{record[:id]}", serialized, ex: 86400) # 24h TTL
+      end
+
+      def find(id)
+        raw = @redis.get("#{@prefix}:record:#{id}")
+        return nil unless raw
+        data = Serialization::Json.decode(raw)
+        data.transform_keys(&:to_sym)
+      end
+
+      def size
+        pending_count + scheduled_count
+      end
+
+      def pending_count
+        count = 0
+        @redis.keys("#{@prefix}:queue:*").each do |key|
+          count += @redis.llen(key)
+        end
+        count
+      rescue => e
+        0
+      end
+
+      def scheduled_count
+        @redis.zcard("#{@prefix}:scheduled")
+      rescue => e
+        0
+      end
+
+      def shutdown
+        @redis.close
+      rescue => e
+        # Already closed
+      end
+
+      private
+
+      def promote_scheduled
+        now = Time.now.to_f
+        # Get all jobs ready to run
+        ready = @redis.zrangebyscore("#{@prefix}:scheduled", "-inf", now.to_s)
+        ready.each do |raw|
+          # Remove from scheduled set
+          removed = @redis.zrem("#{@prefix}:scheduled", raw)
+          next unless removed
+
+          job_data = Serialization::Json.decode(raw)
+          queue = job_data["queue"] || "default"
+          @redis.lpush("#{@prefix}:queue:#{queue}", raw)
+        end
+      rescue => e
+        # Don't crash on promote errors
+      end
+    end
+  end
+end

data/lib/whoosh/jobs/worker.rb

@@ -1,15 +1,15 @@
-# lib/whoosh/jobs/worker.rb
 # frozen_string_literal: true
 
 module Whoosh
   module Jobs
     class Worker
-      def initialize(backend:, di: nil, max_retries: 3, retry_delay: 5, instrumentation: nil)
+      def initialize(backend:, di: nil, max_retries: 3, retry_delay: 5, instrumentation: nil, logger: nil)
         @backend = backend
         @di = di
         @max_retries = max_retries
         @retry_delay = retry_delay
         @instrumentation = instrumentation
+        @logger = logger
         @running = true
       end
 
@@ -33,13 +33,27 @@ module Whoosh
 
       def execute(job_data)
         id = job_data[:id]
+        class_name = job_data[:class_name]
+
+        # Skip scheduled jobs that aren't ready yet
+        if job_data[:run_at] && job_data[:run_at].to_f > Time.now.to_f
+          @backend.push(job_data)
+          return
+        end
+
         record = @backend.find(id) || {}
-        record = record.merge(status: :running, started_at: Time.now.to_f)
+        record = record.merge(id: id, status: :running, started_at: Time.now.to_f)
         @backend.save(record)
 
-        job_class = Object.const_get(job_data[:class_name])
+        @logger&.info("job_started", job_id: id, class: class_name)
+
+        job_class = Object.const_get(class_name)
         job = job_class.new
 
+        # Determine retry settings from job class or defaults
+        max_retries = job_class.respond_to?(:retry_limit) && job_class.retry_limit ? job_class.retry_limit : @max_retries
+        backoff_strategy = job_class.respond_to?(:retry_backoff) ? job_class.retry_backoff : :linear
+
         # Inject DI deps
         if @di && job_class.respond_to?(:dependencies)
           job_class.dependencies.each do |dep|
@@ -49,23 +63,43 @@ module Whoosh
           end
         end
 
-        args = job_data[:args].transform_keys(&:to_sym)
+        args = job_data[:args]
+        args = args.transform_keys(&:to_sym) if args.is_a?(Hash)
         result = job.perform(**args)
         serialized = Serialization::Json.decode(Serialization::Json.encode(result))
 
         @backend.save(record.merge(status: :completed, result: serialized, completed_at: Time.now.to_f))
+        @logger&.info("job_completed", job_id: id, class: class_name)
+
       rescue => e
-        record = @backend.find(id) || {}
+        record = @backend.find(id) || { id: id }
         retry_count = (record[:retry_count] || 0) + 1
 
-        if retry_count <= @max_retries
-          sleep(@retry_delay) if @retry_delay > 0
-          @backend.save(record.merge(retry_count: retry_count, status: :pending))
-          @backend.push(job_data)
+        if retry_count <= max_retries
+          # Non-blocking retry: re-enqueue with delay timestamp instead of sleeping
+          delay = calculate_delay(retry_count, backoff_strategy)
+          run_at = Time.now.to_f + delay
+          @backend.save(record.merge(retry_count: retry_count, status: :scheduled, run_at: run_at))
+          @backend.push(job_data.merge(run_at: run_at))
+          @logger&.warn("job_retry", job_id: id, class: class_name, retry_count: retry_count, delay: delay)
         else
           error = { message: e.message, backtrace: e.backtrace&.first(10)&.join("\n") }
-          @backend.save(record.merge(status: :failed, error: error, retry_count: retry_count, completed_at: Time.now.to_f))
+          @backend.save(record.merge(
+            status: :failed, error: error, retry_count: retry_count, completed_at: Time.now.to_f
+          ))
           @instrumentation&.emit(:job_failed, { job_id: id, error: error })
+          @logger&.error("job_failed", job_id: id, class: class_name, error: e.message)
+        end
+      end
+
+      def calculate_delay(retry_count, strategy)
+        case strategy
+        when :exponential
+          @retry_delay * (2**(retry_count - 1))  # 5, 10, 20, 40...
+        when :linear
+          @retry_delay * retry_count              # 5, 10, 15, 20...
+        else
+          @retry_delay
         end
       end
     end

data/lib/whoosh/jobs.rb

@@ -1,4 +1,3 @@
-# lib/whoosh/jobs.rb
 # frozen_string_literal: true
 
 require "securerandom"
@@ -6,6 +5,7 @@ require "securerandom"
 module Whoosh
   module Jobs
     autoload :MemoryBackend, "whoosh/jobs/memory_backend"
+    autoload :RedisBackend,  "whoosh/jobs/redis_backend"
     autoload :Worker,        "whoosh/jobs/worker"
 
     @backend = nil
@@ -23,16 +23,27 @@ module Whoosh
         !!@backend
       end
 
-      def enqueue(job_class, **args)
+      def enqueue(job_class, run_at: nil, **args)
         raise Errors::DependencyError, "Jobs not configured — boot a Whoosh::App first" unless configured?
+
         id = SecureRandom.uuid
+        queue_name = job_class.respond_to?(:queue) ? job_class.queue : "default"
         record = {
-          id: id, class_name: job_class.name, args: args, status: :pending,
-          result: nil, error: nil, retry_count: 0,
-          created_at: Time.now.to_f, started_at: nil, completed_at: nil
+          id: id,
+          class_name: job_class.name,
+          args: args,
+          queue: queue_name,
+          status: run_at ? :scheduled : :pending,
+          run_at: run_at,
+          result: nil,
+          error: nil,
+          retry_count: 0,
+          created_at: Time.now.to_f,
+          started_at: nil,
+          completed_at: nil
         }
         @backend.save(record)
-        @backend.push({ id: id, class_name: job_class.name, args: args })
+        @backend.push({ id: id, class_name: job_class.name, args: args, queue: queue_name, run_at: run_at })
         id
       end
 
@@ -41,6 +52,18 @@ module Whoosh
         @backend.find(id)
       end
 
+      # Build the right backend from config (auto-detect pattern)
+      def build_backend(config_data = {})
+        jobs_config = config_data["jobs"] || {}
+        redis_url = ENV["REDIS_URL"] || jobs_config["redis_url"]
+
+        if redis_url && jobs_config["backend"] != "memory"
+          RedisBackend.new(url: redis_url)
+        else
+          MemoryBackend.new
+        end
+      end
+
       def reset!
         @backend = nil
         @di = nil

data/lib/whoosh/vector_store/memory_store.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Whoosh
+  module VectorStore
+    class MemoryStore
+      def initialize
+        @collections = {}
+        @mutex = Mutex.new
+      end
+
+      # Store a vector with metadata
+      def insert(collection, id:, vector:, metadata: {})
+        @mutex.synchronize do
+          @collections[collection] ||= {}
+          @collections[collection][id] = { vector: vector, metadata: metadata }
+        end
+      end
+
+      # Search by cosine similarity, return top-k results
+      def search(collection, vector:, limit: 10)
+        @mutex.synchronize do
+          items = @collections[collection]
+          return [] unless items && !items.empty?
+
+          scored = items.map do |id, data|
+            score = cosine_similarity(vector, data[:vector])
+            { id: id, score: score, metadata: data[:metadata] }
+          end
+
+          scored.sort_by { |r| -r[:score] }.first(limit)
+        end
+      end
+
+      # Delete a vector
+      def delete(collection, id:)
+        @mutex.synchronize do
+          @collections[collection]&.delete(id)
+        end
+      end
+
+      # Count vectors in a collection
+      def count(collection)
+        @mutex.synchronize do
+          @collections[collection]&.size || 0
+        end
+      end
+
+      # Drop a collection
+      def drop(collection)
+        @mutex.synchronize do
+          @collections.delete(collection)
+        end
+      end
+
+      def close
+        # No-op
+      end
+
+      private
+
+      def cosine_similarity(a, b)
+        return 0.0 if a.empty? || b.empty? || a.length != b.length
+
+        dot = 0.0
+        mag_a = 0.0
+        mag_b = 0.0
+
+        a.length.times do |i|
+          dot += a[i] * b[i]
+          mag_a += a[i] * a[i]
+          mag_b += b[i] * b[i]
+        end
+
+        denom = Math.sqrt(mag_a) * Math.sqrt(mag_b)
+        denom.zero? ? 0.0 : dot / denom
+      end
+    end
+  end
+end

data/lib/whoosh/vector_store.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Whoosh
+  module VectorStore
+    autoload :MemoryStore, "whoosh/vector_store/memory_store"
+
+    # Auto-detect: zvec gem → use it, otherwise → in-memory
+    def self.build(config_data = {})
+      vector_config = config_data["vector"] || {}
+      adapter = vector_config["adapter"] || "auto"
+
+      case adapter
+      when "auto"
+        # Try zvec first, fall back to memory
+        if zvec_available?
+          require "whoosh/vector_store/zvec_store"
+          ZvecStore.new(**zvec_options(vector_config))
+        else
+          MemoryStore.new
+        end
+      when "memory"
+        MemoryStore.new
+      when "zvec"
+        require "whoosh/vector_store/zvec_store"
+        ZvecStore.new(**zvec_options(vector_config))
+      else
+        MemoryStore.new
+      end
+    end
+
+    def self.zvec_available?
+      require "zvec"
+      true
+    rescue LoadError
+      false
+    end
+
+    def self.zvec_options(config)
+      { path: config["path"] || "db/vectors" }
+    end
+  end
+end

data/lib/whoosh/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Whoosh
-  VERSION = "1.0.2"
+  VERSION = "1.2.0"
 end

data/lib/whoosh.rb

@@ -28,6 +28,7 @@ module Whoosh
   autoload :Jobs,                "whoosh/jobs"
   autoload :Metrics,             "whoosh/metrics"
   autoload :Paginate,            "whoosh/paginate"
+  autoload :VectorStore,         "whoosh/vector_store"
 
   module Auth
     autoload :ApiKey,         "whoosh/auth/api_key"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whoosh
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Johannes Dwi Cahyo
@@ -199,6 +199,7 @@ files:
 - lib/whoosh/job.rb
 - lib/whoosh/jobs.rb
 - lib/whoosh/jobs/memory_backend.rb
+- lib/whoosh/jobs/redis_backend.rb
 - lib/whoosh/jobs/worker.rb
 - lib/whoosh/logger.rb
 - lib/whoosh/mcp/client.rb
@@ -240,6 +241,8 @@ files:
 - lib/whoosh/test.rb
 - lib/whoosh/types.rb
 - lib/whoosh/uploaded_file.rb
+- lib/whoosh/vector_store.rb
+- lib/whoosh/vector_store/memory_store.rb
 - lib/whoosh/version.rb
 homepage: https://github.com/johannesdwicahyo/whoosh
 licenses: