s3arch 0.0.6 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f15baec2213fb6a5cee584447843884e1b9aa5e1bf650d500c53cd6c17e69539
-  data.tar.gz: d00b7d3345ebf462c5ef0a450452502a818c2dd6969c34e82bfc5d763dc9cd7f
+  metadata.gz: 538d32a585dde80df0138c153bd7b715d7ae762a42c122ce793ab037440c6940
+  data.tar.gz: f7d53b318b898243fb27ff2e0f9a653c4749cd2e573a74130e93347aa429e9ed
 SHA512:
-  metadata.gz: 3fd7108161529a4d4d23ffcd49e483d944064ea6364438470f65859c86382f43a611e6867a4d69981772c76bc6e244d306a256dbb1ac6607622cf787d78551e9
-  data.tar.gz: a9895ba3ef76d2720330633cc6eeb44e6d70d9a2461d4602eea136d8de19f4032397e1195dba91c1d0160a95a19bc69ec8123d2dd25581683da47a547717cd3e
+  metadata.gz: 8219d1600287fe1ccffc5405cb89019edec9975814c409a03eed6236f15092e4a0621b452aae867d4cb6942c901c928c8e80afb4c601444659d4d8078e2c2d7e
+  data.tar.gz: 0fa20c2f54a2c27aa1f5b86fbb4c48809e792045ea5edeb8896d99d2cbde7bc0f5a21bbc7aa022c111391db427358a9ad2b8f8791af26318b82c280a4d80dc77

data/lambda/controllers/s3arch_controller.rb

@@ -44,11 +44,17 @@ class S3archController < BeltController::Base
     owners = items.map do |item|
       {
         owner_id: item[config.owner_key] || item.values.first,
-        version: item['version'],
+        version: format_version(item['version']),
         record_count: item['record_count'],
         updated_at: item['updated_at']
       }
     end
     owners.sort_by { |o| o[:updated_at].to_s }.reverse
   end
+
+  def format_version(version)
+    return version unless version
+
+    version.to_i.to_s
+  end
 end

data/lib/s3arch/indexer/stream_parser.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module S3arch
+  class Indexer
+    # Parses DynamoDB Stream events from SQS records into change operations.
+    module StreamParser
+      private
+
+      def group_changes(sqs_records)
+        grouped = Hash.new { |h, k| h[k] = [] }
+
+        sqs_records.each do |sqs_record|
+          stream_record = JSON.parse(sqs_record['body'])
+          event_name = stream_record['eventName']
+          new_image = stream_record.dig('dynamodb', 'NewImage')
+          old_image = stream_record.dig('dynamodb', 'OldImage')
+
+          owner_id = extract_owner(new_image || old_image)
+          next unless owner_id
+
+          change = build_change(event_name, new_image, old_image)
+          grouped[owner_id] << change if change
+        end
+
+        grouped
+      end
+
+      def build_change(event_name, new_image, old_image)
+        case event_name
+        when 'INSERT'
+          tokens = extract_tokens(new_image)
+          record_id = extract_record_id(new_image)
+          return nil unless tokens && record_id && passes_filter?(new_image)
+
+          { action: :insert, record_id: record_id, tokens: tokens, meta: extract_meta(new_image) }
+        when 'REMOVE'
+          tokens = extract_tokens(old_image)
+          record_id = extract_record_id(old_image)
+          return nil unless tokens && record_id
+
+          { action: :delete, record_id: record_id, tokens: tokens }
+        when 'MODIFY'
+          build_modify_change(new_image, old_image)
+        end
+      end
+
+      def build_modify_change(new_image, old_image)
+        old_tokens = extract_tokens(old_image)
+        new_tokens = extract_tokens(new_image)
+        record_id = extract_record_id(new_image)
+        return nil unless record_id
+
+        unless passes_filter?(new_image)
+          return old_tokens ? { action: :delete, record_id: record_id, tokens: old_tokens } : nil
+        end
+
+        unless old_tokens
+          return new_tokens ? { action: :insert, record_id: record_id, tokens: new_tokens,
+                                meta: extract_meta(new_image) } : nil
+        end
+
+        return nil unless new_tokens
+
+        { action: :update, record_id: record_id, old_tokens: old_tokens, new_tokens: new_tokens,
+          meta: extract_meta(new_image) }
+      end
+
+      def extract_owner(image)
+        return nil unless image
+
+        val = image[@config.owner_key]
+        val.is_a?(Hash) ? val['S'] : val
+      end
+
+      def extract_record_id(image)
+        return nil unless image
+
+        val = image['id']
+        val.is_a?(Hash) ? val['S'] : val
+      end
+
+      def extract_tokens(image)
+        return nil unless image
+
+        val = image[@config.token_field]
+        return nil unless val
+
+        if val.is_a?(Hash) && val.key?('M')
+          val['M'].transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
+        elsif val.is_a?(Hash) && !val.key?('S')
+          val.transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
+        end
+      end
+
+      def extract_meta(image)
+        @config.metadata_fields.each_with_object({}) do |field, meta|
+          val = image[field]
+          meta[field] = val.is_a?(Hash) ? (val['S'] || val['N'] || '') : val.to_s
+        end
+      end
+
+      def passes_filter?(image)
+        plain = image.transform_values { |v| v.is_a?(Hash) ? (v['S'] || v['N'] || v['BOOL']&.to_s || '') : v }
+        @config.record_filter.call(plain)
+      end
+    end
+  end
+end

data/lib/s3arch/indexer.rb

@@ -4,11 +4,15 @@ require 'aws-sdk-dynamodb'
 require 'aws-sdk-s3'
 require 'sqlite3'
 require 'json'
+require_relative 'indexer/stream_parser'
+
 module S3arch
   # Builds SQLite FTS5 databases per owner from pre-computed tokens stored in DynamoDB.
   # The indexer never sees raw content — only tokens. Supports incremental updates via
   # DynamoDB Stream events (INSERT/MODIFY/REMOVE).
   class Indexer
+    include StreamParser
+
     def initialize(config: S3arch.configuration)
       config.validate!
       @config = config
@@ -83,104 +87,6 @@ module S3arch
 
     private
 
-    def group_changes(sqs_records)
-      grouped = Hash.new { |h, k| h[k] = [] }
-
-      sqs_records.each do |sqs_record|
-        stream_record = JSON.parse(sqs_record['body'])
-        event_name = stream_record['eventName']
-        new_image = stream_record.dig('dynamodb', 'NewImage')
-        old_image = stream_record.dig('dynamodb', 'OldImage')
-
-        owner_id = extract_owner(new_image || old_image)
-        next unless owner_id
-
-        change = build_change(event_name, new_image, old_image)
-        grouped[owner_id] << change if change
-      end
-
-      grouped
-    end
-
-    def build_change(event_name, new_image, old_image)
-      case event_name
-      when 'INSERT'
-        tokens = extract_tokens(new_image)
-        record_id = extract_record_id(new_image)
-        return nil unless tokens && record_id && passes_filter?(new_image)
-
-        { action: :insert, record_id: record_id, tokens: tokens, meta: extract_meta(new_image) }
-      when 'REMOVE'
-        tokens = extract_tokens(old_image)
-        record_id = extract_record_id(old_image)
-        return nil unless tokens && record_id
-
-        { action: :delete, record_id: record_id, tokens: tokens }
-      when 'MODIFY'
-        old_tokens = extract_tokens(old_image)
-        new_tokens = extract_tokens(new_image)
-        record_id = extract_record_id(new_image)
-        return nil unless record_id
-
-        # If item no longer passes filter, treat as delete
-        unless passes_filter?(new_image)
-          return old_tokens ? { action: :delete, record_id: record_id, tokens: old_tokens } : nil
-        end
-
-        # If item previously didn't pass filter (no old tokens), treat as insert
-        unless old_tokens
-          return new_tokens ? { action: :insert, record_id: record_id, tokens: new_tokens,
-                                meta: extract_meta(new_image) } : nil
-        end
-
-        return nil unless new_tokens
-
-        { action: :update, record_id: record_id, old_tokens: old_tokens, new_tokens: new_tokens,
-          meta: extract_meta(new_image) }
-      end
-    end
-
-    def extract_owner(image)
-      return nil unless image
-
-      val = image[@config.owner_key]
-      val.is_a?(Hash) ? val['S'] : val
-    end
-
-    def extract_record_id(image)
-      return nil unless image
-
-      val = image['id']
-      val.is_a?(Hash) ? val['S'] : val
-    end
-
-    def extract_tokens(image)
-      return nil unless image
-
-      val = image[@config.token_field]
-      return nil unless val
-
-      # Token field is a DynamoDB Map: { "M": { "name": { "S": "..." }, "description": { "S": "..." } } }
-      if val.is_a?(Hash) && val.key?('M')
-        val['M'].transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
-      elsif val.is_a?(Hash) && !val.key?('S')
-        val.transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
-      end
-    end
-
-    def extract_meta(image)
-      @config.metadata_fields.each_with_object({}) do |field, meta|
-        val = image[field]
-        meta[field] = val.is_a?(Hash) ? (val['S'] || val['N'] || '') : val.to_s
-      end
-    end
-
-    def passes_filter?(image)
-      # Convert DynamoDB image to plain hash for filter
-      plain = image.transform_values { |v| v.is_a?(Hash) ? (v['S'] || v['N'] || v['BOOL']&.to_s || '') : v }
-      @config.record_filter.call(plain)
-    end
-
     def apply_change(db, change)
       case change[:action]
       when :insert
@@ -252,7 +158,7 @@ module S3arch
         result.items.each do |item|
           next unless @config.record_filter.call(item)
 
-          tokens = item[@config.token_field]
+          tokens = extract_tokens_from_item(item)
           next unless tokens.is_a?(Hash) && tokens.any?
 
           records << { 'id' => item['id'], 'tokens' => tokens, 'meta' => extract_meta_from_item(item) }
@@ -265,9 +171,21 @@ module S3arch
       records
     end
 
+    def extract_tokens_from_item(item)
+      # Try the dedicated token field first
+      tokens = item[@config.token_field]
+      return tokens if tokens.is_a?(Hash) && tokens.any?
+
+      # Fall back: synthesize tokens from searchable_fields directly
+      @config.searchable_fields.each_with_object({}) do |field, map|
+        val = item[field]
+        map[field] = val.to_s if val
+      end
+    end
+
     def build_query_params(owner_id)
       fields = (['id', @config.token_field, @config.owner_key] +
-                @config.metadata_fields + @config.filter_fields).uniq
+                @config.searchable_fields + @config.metadata_fields + @config.filter_fields).compact.uniq
       expression_names = {}
       projected = fields.map { |f| reserved_word?(f) ? "##{f}".tap { |p| expression_names[p] = f } : f }
 

data/lib/s3arch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3arch
-  VERSION = '0.0.6'
+  VERSION = '0.0.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3arch
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.8
 platform: ruby
 authors:
 - Adam Dalton
@@ -111,6 +111,7 @@ files:
 - lib/s3arch/configuration.rb
 - lib/s3arch/handler.rb
 - lib/s3arch/indexer.rb
+- lib/s3arch/indexer/stream_parser.rb
 - lib/s3arch/routes.rb
 - lib/s3arch/searcher.rb
 - lib/s3arch/tokenizer.rb