s3arch 0.0.9 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc88f938520606c4d992c896525041fe84635c7851ab3aba93fd6e06df28b370
-  data.tar.gz: abbd30f7e02a7e7ae5de4af5c85fb4f83b386f2ca11626c0a0f0a3831e99be7e
+  metadata.gz: 89f4fce79a53db6b2890bf9034c5702e556ce27a1f399e8d4c605b8953fcd033
+  data.tar.gz: 13b9092d968a9aa1a44dcacbe9bd9ede1304e8a9e67623219245e0f601a9d467
 SHA512:
-  metadata.gz: c88f194638ec1fa29f3fc1705efedaf9f5783f373e324fc7053188676572dd5e66f26bd81adb27248906f04779a614e633958a3a64eb3c365c36de17b942937b
-  data.tar.gz: bc74c8e5b04a667d7a1eb0b4339d9eb717b62c77e92cc0701dc139e2b7f5bf46aa2ada29a03df9f9b9578e7e8b84a7eee86ac6b77fa9b149d29ab442e2890bc8
+  metadata.gz: 1c2093cac0b0e12bc0a9fe679d218fa13aa6b1dfeb35b1bd69bef03dbf1699eaba32084931330e3322993981d541787966e270803e566716ea4b61de9edaddfe
+  data.tar.gz: c6d81a49d383c03c03638a0d6548acc7df697b3fa3f1dc6bc26f8347bf03bb6f29a63707af68baa8920f45dceea74f3fe886533a4914c3af281149271f0b00e1

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [0.1.0] - 2026-06-18
+
+### Added
+
+- `S3arch::Store` — thin adapter layer encapsulating all DynamoDB/S3 operations (fetch records, upload/download indexes, version tracking). Indexer and Searcher accept an injected `store:` parameter for testability.
+- `S3arch::Models::SearchVersion` — lightweight model for the version tracking table with `.all`, `.find`, `.increment!`, and `#to_h`
+- Comprehensive unit specs (84 examples): Tokenizer, Indexer, Searcher, StreamParser, Handler, Store, SearchVersion
+
+### Changed
+
+- `Indexer` and `Searcher` no longer call AWS SDKs directly — delegated to `Store`
+- `S3archController` simplified to use `SearchVersion.all` instead of inline DynamoDB scanning
+
 ## [0.0.5] - 2025-06-12
 
 ### Added

data/lambda/controllers/s3arch_controller.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'aws-sdk-dynamodb'
+require_relative '../models/search_version'
 
 class S3archController < BeltController::Base
   def index
-    owners = fetch_owners
-    success_response(owners: owners)
+    versions = S3arch::Models::SearchVersion.all
+    success_response(owners: versions.map(&:to_h))
   rescue StandardError => e
     error_response("Failed to load owners: #{e.message}", 500)
   end
@@ -20,41 +20,9 @@ class S3archController < BeltController::Base
     else
       S3arch::Indexer.new.rebuild(owner_id)
     end
+
     success_response(status: 'ok', owner_id: owner_id)
   rescue StandardError => e
     error_response("Failed to rebuild index: #{e.message}", 500)
   end
-
-  private
-
-  def fetch_owners
-    dynamodb = Aws::DynamoDB::Client.new
-    config = S3arch.configuration
-    items = []
-    scan_params = { table_name: config.version_table }
-
-    loop do
-      result = dynamodb.scan(scan_params)
-      items.concat(result.items)
-      break unless result.last_evaluated_key
-
-      scan_params[:exclusive_start_key] = result.last_evaluated_key
-    end
-
-    owners = items.map do |item|
-      {
-        owner_id: item[config.owner_key] || item.values.first,
-        version: format_version(item['version']),
-        record_count: item['record_count'],
-        updated_at: item['updated_at']
-      }
-    end
-    owners.sort_by { |o| o[:updated_at].to_s }.reverse
-  end
-
-  def format_version(version)
-    return version unless version
-
-    version.to_i.to_s
-  end
 end

data/lambda/models/search_version.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module S3arch
+  module Models
+    # Lightweight model for the version tracking table.
+    # Each record represents one owner's index state.
+    class SearchVersion
+      attr_reader :owner_id, :version, :record_count, :updated_at
+
+      def initialize(owner_id:, version: 0, record_count: 0, updated_at: nil)
+        @owner_id = owner_id
+        @version = version.to_i
+        @record_count = record_count.to_i
+        @updated_at = updated_at
+      end
+
+      def to_h
+        { owner_id: owner_id, version: version.to_s, record_count: record_count, updated_at: updated_at }
+      end
+
+      class << self
+        def all(config: S3arch.configuration)
+          items = []
+          params = { table_name: config.version_table }
+
+          loop do
+            result = Aws::DynamoDB::Client.new.scan(params)
+            items.concat(result.items)
+            break unless result.last_evaluated_key
+
+            params[:exclusive_start_key] = result.last_evaluated_key
+          end
+
+          items.map { |item| from_dynamo(item, config) }
+               .sort_by { |v| v.updated_at.to_s }
+               .reverse
+        end
+
+        def find(owner_id, config: S3arch.configuration)
+          result = Aws::DynamoDB::Client.new.get_item(table_name: config.version_table,
+                                                      key: { config.owner_key => owner_id },
+                                                      projection_expression: 'version')
+          return nil unless result.item
+
+          result.item['version']&.to_i
+        end
+
+        def increment!(owner_id, record_count:, config: S3arch.configuration)
+          Aws::DynamoDB::Client.new.update_item(
+            table_name: config.version_table,
+            key: { config.owner_key => owner_id },
+            update_expression: 'SET version = if_not_exists(version, :zero) + :one, ' \
+                               'updated_at = :now, record_count = :count',
+            expression_attribute_values: { ':zero' => 0, ':one' => 1, ':now' => Time.now.iso8601,
+                                           ':count' => record_count }
+          )
+        end
+
+        private
+
+        def from_dynamo(item, config)
+          new(
+            owner_id: item[config.owner_key] || item.values.first,
+            version: item['version'],
+            record_count: item['record_count'],
+            updated_at: item['updated_at']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/s3arch/indexer.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'aws-sdk-dynamodb'
-require 'aws-sdk-s3'
 require 'sqlite3'
 require 'json'
 require_relative 'indexer/stream_parser'
@@ -13,22 +11,20 @@ module S3arch
   class Indexer
     include StreamParser
 
-    def initialize(config: S3arch.configuration)
+    def initialize(config: S3arch.configuration, store: nil)
       config.validate!
       @config = config
-      @dynamodb = Aws::DynamoDB::Client.new
-      @s3 = Aws::S3::Client.new
+      @store = store || Store.new(config: config)
     end
 
     # Full rebuild — pulls all tokens from DynamoDB for an owner.
-    # Used for initial backfill or when incremental isn't possible.
     def rebuild(owner_id)
-      records = fetch_records(owner_id)
+      records = @store.fetch_records(owner_id)
       db_path = "/tmp/s3arch_#{owner_id}.sqlite3"
 
       build_database(db_path, records)
-      upload(owner_id, db_path)
-      increment_version(owner_id, records.size)
+      @store.upload_index(owner_id, db_path)
+      @store.increment_version(owner_id, records.size)
 
       log(:info, 'Index rebuilt', owner_id: owner_id, record_count: records.size)
     ensure
@@ -36,12 +32,10 @@ module S3arch
     end
 
     # Incremental update — applies INSERT/DELETE/UPDATE to an existing index.
-    # Downloads current DB from S3, applies changes, re-uploads.
     def apply_changes(owner_id, changes)
       db_path = "/tmp/s3arch_#{owner_id}.sqlite3"
-      download_existing(owner_id, db_path)
 
-      unless File.exist?(db_path)
+      unless @store.download_index(owner_id, db_path)
         log(:info, 'No existing index, doing full rebuild', owner_id: owner_id)
         return rebuild(owner_id)
       end
@@ -56,8 +50,8 @@ module S3arch
       record_count = db.get_first_value('SELECT COUNT(*) FROM records_meta')
       db.close
 
-      upload(owner_id, db_path)
-      increment_version(owner_id, record_count)
+      @store.upload_index(owner_id, db_path)
+      @store.increment_version(owner_id, record_count)
 
       log(:info, 'Index updated incrementally', owner_id: owner_id, changes: changes.size, record_count: record_count)
     ensure
@@ -65,7 +59,6 @@ module S3arch
     end
 
     # Process SQS event containing DynamoDB stream records.
-    # Groups by owner and applies incremental changes.
     def process_event(event)
       sqs_records = event['Records'] || []
       grouped = group_changes(sqs_records)
@@ -83,8 +76,6 @@ module S3arch
       { statusCode: 200, body: JSON.generate(rebuilt: grouped.size) }
     end
 
-    RESERVED_WORDS = Set.new(%w[status name comment count size type]).freeze
-
     private
 
     def apply_change(db, change)
@@ -101,7 +92,6 @@ module S3arch
           delete_row(db, rowid, change[:old_tokens])
           insert_row(db, rowid, change[:record_id], change[:new_tokens], change[:meta])
         else
-          # Record wasn't in index yet, just insert
           new_rowid = next_rowid(db)
           insert_row(db, new_rowid, change[:record_id], change[:new_tokens], change[:meta])
         end
@@ -126,7 +116,6 @@ module S3arch
       fts_cols = @config.searchable_fields
       fts_values = fts_cols.map { |f| tokens[f] || '' }
       placeholders = (['?'] * (fts_values.size + 1)).join(', ')
-      # FTS5 contentless delete: INSERT with special 'delete' command
       fts_delete_sql = "INSERT INTO records_fts(records_fts, rowid, #{fts_cols.join(', ')}) " \
                        "VALUES ('delete', #{placeholders})"
       db.execute(fts_delete_sql, [rowid] + fts_values)
@@ -142,70 +131,6 @@ module S3arch
       (max || 0) + 1
     end
 
-    def download_existing(owner_id, db_path)
-      @s3.get_object(bucket: @config.index_bucket, key: "#{owner_id}/index.sqlite3", response_target: db_path)
-    rescue Aws::S3::Errors::NoSuchKey
-      # No existing index — caller will fall back to rebuild
-    end
-
-    # Full rebuild: fetches token field from DynamoDB (never reads content)
-    def fetch_records(owner_id)
-      records = []
-      params = build_query_params(owner_id)
-
-      loop do
-        result = @dynamodb.query(params)
-        result.items.each do |item|
-          next unless @config.record_filter.call(item)
-
-          tokens = extract_tokens_from_item(item)
-          next unless tokens.is_a?(Hash) && tokens.any?
-
-          records << { 'id' => item['id'], 'tokens' => tokens, 'meta' => extract_meta_from_item(item) }
-        end
-        break unless result.last_evaluated_key
-
-        params[:exclusive_start_key] = result.last_evaluated_key
-      end
-
-      records
-    end
-
-    def extract_tokens_from_item(item)
-      # Try the dedicated token field first
-      tokens = item[@config.token_field]
-      return tokens if tokens.is_a?(Hash) && tokens.any?
-
-      # Fall back: synthesize tokens from searchable_fields directly
-      @config.searchable_fields.each_with_object({}) do |field, map|
-        val = item[field]
-        map[field] = val.to_s if val
-      end
-    end
-
-    def build_query_params(owner_id)
-      fields = (['id', @config.token_field, @config.owner_key] +
-                @config.searchable_fields + @config.metadata_fields + @config.filter_fields).compact.uniq
-      expression_names = {}
-      projected = fields.map { |f| reserved_word?(f) ? "##{f}".tap { |p| expression_names[p] = f } : f }
-
-      owner_placeholder = reserved_word?(@config.owner_key) ? "##{@config.owner_key}" : @config.owner_key
-      expression_names["##{@config.owner_key}"] = @config.owner_key if reserved_word?(@config.owner_key)
-
-      params = { table_name: @config.source_table, index_name: @config.source_index,
-                 key_condition_expression: "#{owner_placeholder} = :owner",
-                 expression_attribute_values: { ':owner' => owner_id },
-                 projection_expression: projected.join(', ') }
-      params[:expression_attribute_names] = expression_names if expression_names.any?
-      params
-    end
-
-    def extract_meta_from_item(item)
-      @config.metadata_fields.to_h { |field| [field, item[field].to_s] }
-    end
-
-    def reserved_word?(field) = RESERVED_WORDS.include?(field.downcase)
-
     def build_database(db_path, records)
       FileUtils.rm_f(db_path)
       db = SQLite3::Database.new(db_path)
@@ -237,20 +162,6 @@ module S3arch
       db.close
     end
 
-    def upload(owner_id, db_path)
-      @s3.put_object(bucket: @config.index_bucket, key: "#{owner_id}/index.sqlite3", body: File.open(db_path, 'rb'))
-    end
-
-    def increment_version(owner_id, record_count)
-      @dynamodb.update_item(
-        table_name: @config.version_table,
-        key: { @config.owner_key => owner_id },
-        update_expression: 'SET version = if_not_exists(version, :zero) + :one, ' \
-                           'updated_at = :now, record_count = :count',
-        expression_attribute_values: { ':zero' => 0, ':one' => 1, ':now' => Time.now.iso8601, ':count' => record_count }
-      )
-    end
-
     def log(level, message, **data)
       return unless @config.logger
 

data/lib/s3arch/searcher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'aws-sdk-dynamodb'
-require 'aws-sdk-s3'
 require 'sqlite3'
 
 module S3arch
@@ -18,11 +16,10 @@ module S3arch
       end
     end
 
-    def initialize(config: S3arch.configuration)
+    def initialize(config: S3arch.configuration, store: nil)
       config.validate!
       @config = config
-      @dynamodb = Aws::DynamoDB::Client.new
-      @s3 = Aws::S3::Client.new
+      @store = store || Store.new(config: config)
     end
 
     def search(query:, owner_ids:, filters: {})
@@ -74,23 +71,18 @@ module S3arch
       cached = self.class.version_cache[owner_id]
       return cached[:version] if cached && (Time.now - cached[:checked_at]) < @config.version_ttl
 
-      result = @dynamodb.get_item(table_name: @config.version_table,
-                                  key: { @config.owner_key => owner_id },
-                                  projection_expression: 'version')
-      version = result.item&.dig('version')
+      version = @store.fetch_version(owner_id)
       self.class.version_cache[owner_id] = { version: version, checked_at: Time.now } if version
       version
     end
 
     def download_database(owner_id)
       db_path = "/tmp/s3arch_#{owner_id}.sqlite3"
-      @s3.get_object(bucket: @config.index_bucket, key: "#{owner_id}/index.sqlite3",
-                     response_target: db_path)
+      return nil unless @store.download_index(owner_id, db_path)
+
       db = SQLite3::Database.new(db_path)
       db.results_as_hash = true
       db
-    rescue Aws::S3::Errors::NoSuchKey
-      nil
     end
 
     def query_fts(db, query, filters: {})

data/lib/s3arch/store.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-dynamodb'
+require 'aws-sdk-s3'
+
+module S3arch
+  # Thin adapter layer encapsulating all DynamoDB/S3 operations for index lifecycle.
+  # Indexer and Searcher delegate here instead of calling AWS SDKs directly.
+  class Store
+    def initialize(config: S3arch.configuration)
+      @config = config
+      @dynamodb = Aws::DynamoDB::Client.new
+      @s3 = Aws::S3::Client.new
+    end
+
+    # --- Source data (read tokens from host app's DynamoDB table) ---
+
+    def fetch_records(owner_id)
+      records = []
+      params = build_query_params(owner_id)
+
+      loop do
+        result = @dynamodb.query(params)
+        result.items.each do |item|
+          next unless @config.record_filter.call(item)
+
+          tokens = extract_tokens_from_item(item)
+          next unless tokens.is_a?(Hash) && tokens.any?
+
+          records << { 'id' => item['id'], 'tokens' => tokens, 'meta' => extract_meta(item) }
+        end
+        break unless result.last_evaluated_key
+
+        params[:exclusive_start_key] = result.last_evaluated_key
+      end
+
+      records
+    end
+
+    # --- Index files (SQLite databases on S3) ---
+
+    def upload_index(owner_id, db_path)
+      @s3.put_object(bucket: @config.index_bucket, key: index_key(owner_id), body: File.open(db_path, 'rb'))
+    end
+
+    def download_index(owner_id, db_path)
+      @s3.get_object(bucket: @config.index_bucket, key: index_key(owner_id), response_target: db_path)
+      true
+    rescue Aws::S3::Errors::NoSuchKey
+      false
+    end
+
+    # --- Version tracking ---
+
+    def fetch_version(owner_id)
+      result = @dynamodb.get_item(table_name: @config.version_table,
+                                  key: { @config.owner_key => owner_id },
+                                  projection_expression: 'version')
+      result.item&.dig('version')&.to_i
+    end
+
+    def increment_version(owner_id, record_count)
+      @dynamodb.update_item(
+        table_name: @config.version_table,
+        key: { @config.owner_key => owner_id },
+        update_expression: 'SET version = if_not_exists(version, :zero) + :one, ' \
+                           'updated_at = :now, record_count = :count',
+        expression_attribute_values: { ':zero' => 0, ':one' => 1, ':now' => Time.now.iso8601, ':count' => record_count }
+      )
+    end
+
+    RESERVED_WORDS = Set.new(%w[status name comment count size type]).freeze
+
+    private
+
+    def index_key(owner_id) = "#{owner_id}/index.sqlite3"
+
+    def build_query_params(owner_id)
+      fields = (['id', @config.token_field, @config.owner_key] +
+                @config.searchable_fields + @config.metadata_fields + @config.filter_fields).compact.uniq
+      expression_names = {}
+      projected = fields.map { |f| reserved_word?(f) ? "##{f}".tap { |p| expression_names[p] = f } : f }
+
+      owner_placeholder = reserved_word?(@config.owner_key) ? "##{@config.owner_key}" : @config.owner_key
+      expression_names["##{@config.owner_key}"] = @config.owner_key if reserved_word?(@config.owner_key)
+
+      params = { table_name: @config.source_table, index_name: @config.source_index,
+                 key_condition_expression: "#{owner_placeholder} = :owner",
+                 expression_attribute_values: { ':owner' => owner_id },
+                 projection_expression: projected.join(', ') }
+      params[:expression_attribute_names] = expression_names if expression_names.any?
+      params
+    end
+
+    def extract_tokens_from_item(item)
+      tokens = item[@config.token_field]
+      return tokens if tokens.is_a?(Hash) && tokens.any?
+
+      @config.searchable_fields.each_with_object({}) do |field, map|
+        val = item[field]
+        map[field] = val.to_s if val
+      end
+    end
+
+    def extract_meta(item)
+      @config.metadata_fields.to_h { |field| [field, item[field].to_s] }
+    end
+
+    def reserved_word?(field) = RESERVED_WORDS.include?(field.downcase)
+  end
+end

data/lib/s3arch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3arch
-  VERSION = '0.0.9'
+  VERSION = '0.1.1'
 end

data/lib/s3arch.rb

@@ -3,6 +3,7 @@
 require_relative 's3arch/version'
 require_relative 's3arch/configuration'
 require_relative 's3arch/tokenizer'
+require_relative 's3arch/store'
 require_relative 's3arch/indexer'
 require_relative 's3arch/searcher'
 require_relative 's3arch/handler'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3arch
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.1
 platform: ruby
 authors:
 - Adam Dalton
@@ -64,6 +64,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: belt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +121,7 @@ files:
 - infrastructure/routes.tf.rb
 - infrastructure/schema.tf.rb
 - lambda/controllers/s3arch_controller.rb
+- lambda/models/search_version.rb
 - lib/s3arch.rb
 - lib/s3arch/configuration.rb
 - lib/s3arch/handler.rb
@@ -114,6 +129,7 @@ files:
 - lib/s3arch/indexer/stream_parser.rb
 - lib/s3arch/routes.rb
 - lib/s3arch/searcher.rb
+- lib/s3arch/store.rb
 - lib/s3arch/tokenizer.rb
 - lib/s3arch/version.rb
 - lib/s3arch/web.rb