s3arch 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d5bd8c8d47da22b8feb308ff8fbc96205d554ae915ef8e4e2831d5ba534f786
-  data.tar.gz: 4741157e9c522074a08c3478dc4cfddd16445590cfdb107100121083ff34a29a
+  metadata.gz: cd57db198f9598324792e16e5b84e9c22914ebfdfd4fddeb09fc99641458322c
+  data.tar.gz: e107d35cd4b7084d3cf90e3d014bb4fe3c6089475b26419524d9132867565592
 SHA512:
-  metadata.gz: 54b5a57df4c767d23eac10d7984ea5ec796f60cfe9047b24619cb3d67600af2b67e71e2f0639704a55939a057f7351f7778a0587b9db6b80fa768eb2398c9e18
-  data.tar.gz: c4a2ff82ac4985b24376ccacab235e9a56132e1ee10b77202eff9defc62d0b1765ab9da31d89c695b42c3560984d1098df476a6b73b6a4561ad6c1cbefaa4749
+  metadata.gz: 65aba03e3064bf597dc8bb4e53d7552c07f0b89ebb7ca24b9e6c74a7f5c22f9c95746cec237d1e100fce63cb8f285f782487300abd7f4d1b6e1c76553fdad5ff
+  data.tar.gz: 28f5d435ce384c0c1c8e05d17d478ad9fdb9e0a949506350384886fdfcedbfaa2cbd9308dfd08d2faf91dfe4bf02ac07ac6d6749470143914a977f29ab0aef04

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [0.0.2] - 2025-06-08
+
+### Added
+
+- `S3arch::Tokenizer` — pure tokenization class for pre-computing search tokens
+- Token-based indexing — indexer reads pre-computed tokens, never raw content
+- Incremental updates via `Indexer#apply_changes` (DynamoDB Stream INSERT/MODIFY/REMOVE)
+- `Indexer#process_event` — processes SQS events containing DynamoDB stream records
+- `Configuration#token_field` — configurable DynamoDB attribute for stored tokens
+- `Configuration#searchable_fields` — configurable list of fields to tokenize
+
+### Changed
+
+- `S3arch::Indexer` rewritten to support both full rebuild and incremental updates
+- FTS5 contentless DELETE support using stored token values from stream OLD_IMAGE
+
 ## [0.0.1] - 2025-06-08
 
 ### Added

data/certs/stowzilla.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEdDCCAtygAwIBAgIBATANBgkqhkiG9w0BAQsFADBAMQ4wDAYDVQQDDAVhZ2Vu
+dDEZMBcGCgmSJomT8ixkARkWCXN0b3d6aWxsYTETMBEGCgmSJomT8ixkARkWA2Nv
+bTAeFw0yNjA2MDgxOTExNTlaFw0yNzA2MDgxOTExNTlaMEAxDjAMBgNVBAMMBWFn
+ZW50MRkwFwYKCZImiZPyLGQBGRYJc3Rvd3ppbGxhMRMwEQYKCZImiZPyLGQBGRYD
+Y29tMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAupBquKI/4WvXOgND
+pXyqH2GllZs1wG4TWWdn/DoMg45UoCwD+AWEuGrIdInBCpPN8vEJNJWPoM/RrU+b
+xRBZT4uUk00bnZRW2SYh5GJSqBoBR+rWc2DGkXyGfdRU2sQvkB0+is6ChgQ61WMM
+33LE9+loBlVsZ6EVtrc18Uh2OW0mJpe0hN2nmBrxZqqOZigxC4DKRMFHvpRkxSb6
+mD4kit1AcwX9NEWJsXxrPaetL/SB/VbXaEZX93XAvp6USaXvCWt4slkDS2mIvqtn
+9DtGC43LFC7SDGbnsG9PVenQgVCi8UWFPUAab0PqZSlmi3Qlbhw8qTGPp5Cbv4vz
+qjC2UGPOQigA/7lbbGRhCohMrjOVHMAQwkcgiIqtolUoYlnvPMIy+m3pdvgDv/PH
+bsZGvXQ7i0458xsmp1vaKthZocVAR+GboHbuIiYPUnO45ccXUQ00x6365tTe7mZi
+NvmUYdAGbQmVvFqyxF7IYA6sF74L2Lstu0knSfss557bAe1HAgMBAAGjeTB3MAkG
+A1UdEwQCMAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQWBBSnxTL/lNBCeLqpeVIX6AUY
+kel4zjAeBgNVHREEFzAVgRNhZ2VudEBzdG93emlsbGEuY29tMB4GA1UdEgQXMBWB
+E2FnZW50QHN0b3d6aWxsYS5jb20wDQYJKoZIhvcNAQELBQADggGBACm9Fjit/UCv
+FxlKqeiCTIG94cIx+QrWAOJSx9knKydwUec1u04D/DbfZjTn3C2Bj227QgxeUn+6
+if3e2v7zAk1896hLmGYzML0+nxQPb0vmtdLR7HETUlSKTVabcv1fbwLyjsuGrBvk
+y51vOEzUEZ508a9yepLYqrQu1kOju4d57c9oA5l3H0mMKWz7av9tFj0B+STvuaWk
+HRYDWc5HgOEVTyV+w0uFt2Kw4OCb8C42uSvC5RfYYtw78MSP+5Ru+LXJ7XOtmuN0
+E6GVmofQ17ig9O3rgfFbMendSInrRmvPIGswvM1yivq9NOllFbdck2OJKPx6FCJF
+7SJIkXQfc9P4B5iASIV1d1FsE0YX+g3jHXPJK/4mGL5bAyBKzpMfQB/mg6vQBzkh
+aOKPwcreFj7TznBl89R5tNS9wZQfPVR98zgPyocddWhK18eQNMSBUnv4eeJ8PPbk
+DovL+G8ajHDZ9fjH/+GVYHEMuiVdLarXrKJpHC1VfGTTUAp4NSEpUQ==
+-----END CERTIFICATE-----

data/lib/s3arch/configuration.rb

@@ -2,16 +2,50 @@
 
 module S3arch
   class Configuration
-    attr_accessor :source_table, :source_index, :owner_key, :index_bucket, :version_table,
-                  :searchable_fields, :metadata_fields, :record_filter, :owner_extractor,
-                  :logger, :version_ttl, :max_results, :max_cached_dbs, :ephemeral_storage_mb
+    # DynamoDB table that contains the source records to index
+    attr_accessor :source_table
+
+    # DynamoDB index to query records by owner (e.g., 'UserIndex')
+    attr_accessor :source_index
+
+    # Partition key field on the source table for owner lookup
+    attr_accessor :owner_key
+
+    # S3 bucket for storing SQLite index files
+    attr_accessor :index_bucket
+
+    # DynamoDB table for version tracking
+    attr_accessor :version_table
+
+    # FTS5 searchable fields — array of field names from the source record
+    attr_accessor :searchable_fields
+
+    # DynamoDB attribute name where pre-computed tokens are stored (Map type)
+    # e.g., { "searchTokens": { "name": "blue jacket", "description": "warm winter coat" } }
+    attr_accessor :token_field
+
+    # Metadata fields stored alongside FTS5 for filtering (not searched)
+    attr_accessor :metadata_fields
+
+    # Filter proc — receives a record hash, returns true to include in index
+    attr_accessor :record_filter
+
+    # Owner extractor — proc that extracts owner_id from a DynamoDB stream record
+    attr_accessor :owner_extractor
+
+    # Logger (defaults to $stdout)
+    attr_accessor :logger
+
+    # Searcher settings
+    attr_accessor :version_ttl, :max_results, :max_cached_dbs, :ephemeral_storage_mb
 
     def initialize
       @owner_key = 'user_id'
       @searchable_fields = %w[name description]
+      @token_field = 'searchTokens'
       @metadata_fields = %w[status created_at]
       @record_filter = ->(_record) { true }
-      @owner_extractor = ->(stream_record) {
+      @owner_extractor = lambda { |stream_record|
         image = stream_record.dig('dynamodb', 'NewImage') || stream_record.dig('dynamodb', 'OldImage') || {}
         image.dig(owner_key, 'S')
       }
@@ -22,11 +56,12 @@ module S3arch
       @logger = nil
     end
 
+    # Convenience: env-based configuration (reads from Lambda environment variables)
     def from_env!
-      @source_table = ENV['S3ARCH_SOURCE_TABLE']
+      @source_table = ENV['S3ARCH_SOURCE_TABLE'] || ENV.fetch('INVENTORY_TABLE', nil)
       @source_index = ENV['S3ARCH_SOURCE_INDEX'] || 'UserIndex'
-      @index_bucket = ENV['S3ARCH_INDEX_BUCKET']
-      @version_table = ENV['S3ARCH_VERSION_TABLE']
+      @index_bucket = ENV['S3ARCH_INDEX_BUCKET'] || ENV.fetch('SEARCH_INDEX_BUCKET', nil)
+      @version_table = ENV['S3ARCH_VERSION_TABLE'] || ENV.fetch('SEARCH_INDEX_TABLE', nil)
       self
     end
 

data/lib/s3arch/indexer.rb

@@ -6,6 +6,9 @@ require 'sqlite3'
 require 'json'
 
 module S3arch
+  # Builds SQLite FTS5 databases per owner from pre-computed tokens stored in DynamoDB.
+  # The indexer never sees raw content — only tokens. Supports incremental updates via
+  # DynamoDB Stream events (INSERT/MODIFY/REMOVE).
   class Indexer
     def initialize(config: S3arch.configuration)
       config.validate!
@@ -14,6 +17,8 @@ module S3arch
       @s3 = Aws::S3::Client.new
     end
 
+    # Full rebuild — pulls all tokens from DynamoDB for an owner.
+    # Used for initial backfill or when incremental isn't possible.
     def rebuild(owner_id)
       records = fetch_records(owner_id)
       db_path = "/tmp/s3arch_#{owner_id}.sqlite3"
@@ -27,38 +32,255 @@ module S3arch
       File.delete(db_path) if db_path && File.exist?(db_path)
     end
 
+    # Incremental update — applies INSERT/DELETE/UPDATE to an existing index.
+    # Downloads current DB from S3, applies changes, re-uploads.
+    def apply_changes(owner_id, changes)
+      db_path = "/tmp/s3arch_#{owner_id}.sqlite3"
+      download_existing(owner_id, db_path)
+
+      unless File.exist?(db_path)
+        log(:info, 'No existing index, doing full rebuild', owner_id: owner_id)
+        return rebuild(owner_id)
+      end
+
+      db = SQLite3::Database.new(db_path)
+      db.results_as_hash = true
+
+      db.transaction do
+        changes.each { |change| apply_change(db, change) }
+      end
+
+      record_count = db.get_first_value('SELECT COUNT(*) FROM records_meta')
+      db.close
+
+      upload(owner_id, db_path)
+      increment_version(owner_id, record_count)
+
+      log(:info, 'Index updated incrementally', owner_id: owner_id, changes: changes.size, record_count: record_count)
+    ensure
+      File.delete(db_path) if db_path && File.exist?(db_path)
+    end
+
+    # Process SQS event containing DynamoDB stream records.
+    # Groups by owner and applies incremental changes.
     def process_event(event)
-      records = event['Records'] || []
-      owner_ids = records.filter_map { |r|
-        body = JSON.parse(r['body'])
-        @config.owner_extractor.call(body)
-      }.uniq
+      sqs_records = event['Records'] || []
+      grouped = group_changes(sqs_records)
 
-      log(:info, 'Rebuilding indexes', owner_ids: owner_ids, record_count: records.size)
-      owner_ids.each { |id| rebuild(id) }
-      { statusCode: 200, body: JSON.generate(rebuilt: owner_ids.size) }
+      log(:info, 'Processing stream events', owner_count: grouped.size, record_count: sqs_records.size)
+
+      grouped.each do |owner_id, changes|
+        if changes.any? { |c| c[:action] == :rebuild }
+          rebuild(owner_id)
+        else
+          apply_changes(owner_id, changes)
+        end
+      end
+
+      { statusCode: 200, body: JSON.generate(rebuilt: grouped.size) }
     end
 
     private
 
+    def group_changes(sqs_records)
+      grouped = Hash.new { |h, k| h[k] = [] }
+
+      sqs_records.each do |sqs_record|
+        stream_record = JSON.parse(sqs_record['body'])
+        event_name = stream_record['eventName']
+        new_image = stream_record.dig('dynamodb', 'NewImage')
+        old_image = stream_record.dig('dynamodb', 'OldImage')
+
+        owner_id = extract_owner(new_image || old_image)
+        next unless owner_id
+
+        change = build_change(event_name, new_image, old_image)
+        grouped[owner_id] << change if change
+      end
+
+      grouped
+    end
+
+    def build_change(event_name, new_image, old_image)
+      case event_name
+      when 'INSERT'
+        tokens = extract_tokens(new_image)
+        record_id = extract_record_id(new_image)
+        return nil unless tokens && record_id && passes_filter?(new_image)
+
+        { action: :insert, record_id: record_id, tokens: tokens, meta: extract_meta(new_image) }
+      when 'REMOVE'
+        tokens = extract_tokens(old_image)
+        record_id = extract_record_id(old_image)
+        return nil unless tokens && record_id
+
+        { action: :delete, record_id: record_id, tokens: tokens }
+      when 'MODIFY'
+        old_tokens = extract_tokens(old_image)
+        new_tokens = extract_tokens(new_image)
+        record_id = extract_record_id(new_image)
+        return nil unless record_id
+
+        # If item no longer passes filter, treat as delete
+        unless passes_filter?(new_image)
+          return old_tokens ? { action: :delete, record_id: record_id, tokens: old_tokens } : nil
+        end
+
+        # If item previously didn't pass filter (no old tokens), treat as insert
+        unless old_tokens
+          return new_tokens ? { action: :insert, record_id: record_id, tokens: new_tokens,
+                                meta: extract_meta(new_image) } : nil
+        end
+
+        return nil unless new_tokens
+
+        { action: :update, record_id: record_id, old_tokens: old_tokens, new_tokens: new_tokens,
+          meta: extract_meta(new_image) }
+      end
+    end
+
+    def extract_owner(image)
+      return nil unless image
+
+      val = image[@config.owner_key]
+      val.is_a?(Hash) ? val['S'] : val
+    end
+
+    def extract_record_id(image)
+      return nil unless image
+
+      val = image['id']
+      val.is_a?(Hash) ? val['S'] : val
+    end
+
+    def extract_tokens(image)
+      return nil unless image
+
+      val = image[@config.token_field]
+      return nil unless val
+
+      # Token field is a DynamoDB Map: { "M": { "name": { "S": "..." }, "description": { "S": "..." } } }
+      if val.is_a?(Hash) && val.key?('M')
+        val['M'].transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
+      elsif val.is_a?(Hash) && !val.key?('S')
+        val.transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
+      end
+    end
+
+    def extract_meta(image)
+      @config.metadata_fields.each_with_object({}) do |field, meta|
+        val = image[field]
+        meta[field] = val.is_a?(Hash) ? (val['S'] || val['N'] || '') : val.to_s
+      end
+    end
+
+    def passes_filter?(image)
+      # Convert DynamoDB image to plain hash for filter
+      plain = image.transform_values { |v| v.is_a?(Hash) ? (v['S'] || v['N'] || v['BOOL']&.to_s || '') : v }
+      @config.record_filter.call(plain)
+    end
+
+    def apply_change(db, change)
+      case change[:action]
+      when :insert
+        rowid = next_rowid(db)
+        insert_row(db, rowid, change[:record_id], change[:tokens], change[:meta])
+      when :delete
+        rowid = find_rowid(db, change[:record_id])
+        delete_row(db, rowid, change[:tokens]) if rowid
+      when :update
+        rowid = find_rowid(db, change[:record_id])
+        if rowid
+          delete_row(db, rowid, change[:old_tokens])
+          insert_row(db, rowid, change[:record_id], change[:new_tokens], change[:meta])
+        else
+          # Record wasn't in index yet, just insert
+          new_rowid = next_rowid(db)
+          insert_row(db, new_rowid, change[:record_id], change[:new_tokens], change[:meta])
+        end
+      end
+    end
+
+    def insert_row(db, rowid, record_id, tokens, meta)
+      fts_cols = @config.searchable_fields
+      fts_values = fts_cols.map { |f| tokens[f] || '' }
+      placeholders = (['?'] * (fts_values.size + 1)).join(', ')
+      db.execute("INSERT INTO records_fts(rowid, #{fts_cols.join(', ')}) VALUES (#{placeholders})",
+                 [rowid] + fts_values)
+
+      meta ||= {}
+      meta_values = [rowid, record_id] + @config.metadata_fields.map { |f| meta[f] || '' }
+      meta_placeholders = (['?'] * meta_values.size).join(', ')
+      meta_cols = "rowid, record_id, #{@config.metadata_fields.join(', ')}"
+      db.execute("INSERT INTO records_meta(#{meta_cols}) VALUES (#{meta_placeholders})", meta_values)
+    end
+
+    def delete_row(db, rowid, tokens)
+      fts_cols = @config.searchable_fields
+      fts_values = fts_cols.map { |f| tokens[f] || '' }
+      placeholders = (['?'] * (fts_values.size + 1)).join(', ')
+      # FTS5 contentless delete: INSERT with special 'delete' command
+      fts_delete_sql = "INSERT INTO records_fts(records_fts, rowid, #{fts_cols.join(', ')}) " \
+                       "VALUES ('delete', #{placeholders})"
+      db.execute(fts_delete_sql, [rowid] + fts_values)
+      db.execute('DELETE FROM records_meta WHERE rowid = ?', [rowid])
+    end
+
+    def find_rowid(db, record_id)
+      db.get_first_value('SELECT rowid FROM records_meta WHERE record_id = ?', [record_id])
+    end
+
+    def next_rowid(db)
+      max = db.get_first_value('SELECT MAX(rowid) FROM records_meta')
+      (max || 0) + 1
+    end
+
+    def download_existing(owner_id, db_path)
+      @s3.get_object(bucket: @config.index_bucket, key: "#{owner_id}/index.sqlite3", response_target: db_path)
+    rescue Aws::S3::Errors::NoSuchKey
+      # No existing index — caller will fall back to rebuild
+    end
+
+    # Full rebuild: fetches token field from DynamoDB (never reads content)
     def fetch_records(owner_id)
       records = []
+      fields_to_project = ['id', @config.token_field, @config.owner_key] + @config.metadata_fields + filter_fields
       params = { table_name: @config.source_table, index_name: @config.source_index,
                  key_condition_expression: "#{@config.owner_key} = :owner",
-                 expression_attribute_values: { ':owner' => owner_id } }
+                 expression_attribute_values: { ':owner' => owner_id },
+                 projection_expression: fields_to_project.uniq.join(', ') }
 
       loop do
         result = @dynamodb.query(params)
-        result.items.each { |item| records << item if @config.record_filter.call(item) }
+        result.items.each do |item|
+          next unless @config.record_filter.call(item)
+
+          tokens = item[@config.token_field]
+          next unless tokens.is_a?(Hash) && tokens.any?
+
+          records << { 'id' => item['id'], 'tokens' => tokens, 'meta' => extract_meta_from_item(item) }
+        end
         break unless result.last_evaluated_key
+
         params[:exclusive_start_key] = result.last_evaluated_key
       end
 
       records
     end
 
+    def extract_meta_from_item(item)
+      @config.metadata_fields.to_h do |field|
+        [field, item[field].to_s]
+      end
+    end
+
+    def filter_fields
+      # Fields needed by the record_filter (best-effort — add status/bin_id for default filter)
+      %w[status bin_id]
+    end
+
     def build_database(db_path, records)
-      File.delete(db_path) if File.exist?(db_path)
+      FileUtils.rm_f(db_path)
       db = SQLite3::Database.new(db_path)
 
       fts_cols = @config.searchable_fields.join(', ')
@@ -73,35 +295,21 @@ module S3arch
       db.transaction do
         records.each_with_index do |record, idx|
           rowid = idx + 1
-          fts_values = @config.searchable_fields.map { |f| normalize_field(record[f]) }
-          db.execute("INSERT INTO records_fts(rowid, #{fts_cols}) VALUES (#{(['?'] * (fts_values.size + 1)).join(', ')})",
-                     [rowid] + fts_values)
-
-          meta_values = [rowid, normalize_field(record['id'])] + @config.metadata_fields.map { |f| normalize_field(record[f]) }
-          placeholders = (['?'] * meta_values.size).join(', ')
-          db.execute("INSERT INTO records_meta(rowid, record_id, #{@config.metadata_fields.join(', ')}) VALUES (#{placeholders})",
-                     meta_values)
+          tokens = record['tokens']
+          fts_values = @config.searchable_fields.map { |f| tokens[f] || '' }
+          placeholders = (['?'] * (fts_values.size + 1)).join(', ')
+          db.execute("INSERT INTO records_fts(rowid, #{fts_cols}) VALUES (#{placeholders})", [rowid] + fts_values)
+
+          meta_values = [rowid, record['id']] + @config.metadata_fields.map { |f| record['meta'][f] || '' }
+          meta_placeholders = (['?'] * meta_values.size).join(', ')
+          meta_cols = "rowid, record_id, #{@config.metadata_fields.join(', ')}"
+          db.execute("INSERT INTO records_meta(#{meta_cols}) VALUES (#{meta_placeholders})", meta_values)
         end
       end
 
       db.close
     end
 
-    def normalize_field(value)
-      case value
-      when Hash
-        if value.key?('S') then value['S'].to_s
-        elsif value.key?('L') then value['L'].map { |v| v['S'] || v.to_s }.join(' ')
-        elsif value.key?('N') then value['N'].to_s
-        elsif value.key?('SS') then value['SS'].join(' ')
-        else value.values.first.to_s
-        end
-      when Array then value.map { |v| v.is_a?(Hash) ? (v['S'] || v.to_s) : v.to_s }.join(' ')
-      when nil then ''
-      else value.to_s
-      end
-    end
-
     def upload(owner_id, db_path)
       @s3.put_object(bucket: @config.index_bucket, key: "#{owner_id}/index.sqlite3", body: File.open(db_path, 'rb'))
     end
@@ -110,13 +318,15 @@ module S3arch
       @dynamodb.update_item(
         table_name: @config.version_table,
         key: { @config.owner_key => owner_id },
-        update_expression: 'SET version = if_not_exists(version, :zero) + :one, updated_at = :now, record_count = :count',
+        update_expression: 'SET version = if_not_exists(version, :zero) + :one, ' \
+                           'updated_at = :now, record_count = :count',
         expression_attribute_values: { ':zero' => 0, ':one' => 1, ':now' => Time.now.iso8601, ':count' => record_count }
       )
     end
 
     def log(level, message, **data)
       return unless @config.logger
+
       @config.logger.send(level, message, **data)
     end
   end

data/lib/s3arch/searcher.rb

@@ -72,9 +72,7 @@ module S3arch
 
     def fetch_version(owner_id)
       cached = self.class.version_cache[owner_id]
-      if cached && (Time.now - cached[:checked_at]) < @config.version_ttl
-        return cached[:version]
-      end
+      return cached[:version] if cached && (Time.now - cached[:checked_at]) < @config.version_ttl
 
       result = @dynamodb.get_item(table_name: @config.version_table,
                                   key: { @config.owner_key => owner_id },
@@ -112,6 +110,7 @@ module S3arch
       rows = db.execute(sql, [match_expr])
       rows.filter_map do |row|
         next if filters.any? { |field, value| row[field.to_s] != value }
+
         { record_id: row['record_id'], rank: row['rank'] }
       end
     rescue SQLite3::Exception => e
@@ -131,6 +130,7 @@ module S3arch
 
     def log(level, message, **data)
       return unless @config.logger
+
       @config.logger.send(level, message, **data)
     end
   end

data/lib/s3arch/tokenizer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module S3arch
+  # Pure tokenization — generates the token string stored in DynamoDB.
+  # At query time, FTS5 tokenizes the same way internally, so prefix matches work.
+  class Tokenizer
+    def initialize(fields: S3arch.configuration.searchable_fields)
+      @fields = fields
+    end
+
+    # Accepts a record hash, returns a hash of { field => tokenized_string }
+    # This is what gets stored in DynamoDB and fed directly into FTS5.
+    def tokenize(record)
+      @fields.to_h do |field|
+        [field, normalize(record[field])]
+      end
+    end
+
+    # Flattened single-string version for simple storage (all fields concatenated)
+    def tokenize_flat(record)
+      @fields.map { |f| normalize(record[f]) }.reject(&:empty?).join(' ')
+    end
+
+    private
+
+    def normalize(value)
+      case value
+      when Hash
+        if value.key?('S') then value['S'].to_s
+        elsif value.key?('L') then value['L'].map { |v| v['S'] || v.to_s }.join(' ')
+        elsif value.key?('N') then value['N'].to_s
+        elsif value.key?('SS') then value['SS'].join(' ')
+        else value.values.first.to_s
+        end
+      when Array then value.map { |v| v.is_a?(Hash) ? (v['S'] || v.to_s) : v.to_s }.join(' ')
+      when nil then ''
+      else value.to_s
+      end
+    end
+  end
+end

data/lib/s3arch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3arch
-  VERSION = '0.0.1'
+  VERSION = '0.0.3'
 end

data/lib/s3arch.rb

@@ -2,6 +2,7 @@
 
 require_relative 's3arch/version'
 require_relative 's3arch/configuration'
+require_relative 's3arch/tokenizer'
 require_relative 's3arch/indexer'
 require_relative 's3arch/searcher'
 require_relative 's3arch/handler'

data.tar.gz.sig

@@ -0,0 +1,2 @@
+%d��Ox+B<,�Jׂ����>:	��P۬��`�3?�N���6��7�7=�	��[�=j�o���#��.eO�ä�k}�fG�m$�|!�5�*�����%.	1�M�a��$��
+�Q.�:��Yo�Ǥvׇ��pI�s�w��i�d� {�ȡ��'q��Ǿz6���xPK���%v?$�\�O�BT��OƭF�Cq�2 �Y$�'�I�з����!Z���KH�F�~�S�Rˁc&8R�����d{U�TZo�:���S��2

metadata

@@ -1,12 +1,39 @@
 --- !ruby/object:Gem::Specification
 name: s3arch
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Adam Dalton
 bindir: bin
-cert_chain: []
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEdDCCAtygAwIBAgIBATANBgkqhkiG9w0BAQsFADBAMQ4wDAYDVQQDDAVhZ2Vu
+  dDEZMBcGCgmSJomT8ixkARkWCXN0b3d6aWxsYTETMBEGCgmSJomT8ixkARkWA2Nv
+  bTAeFw0yNjA2MDgxOTExNTlaFw0yNzA2MDgxOTExNTlaMEAxDjAMBgNVBAMMBWFn
+  ZW50MRkwFwYKCZImiZPyLGQBGRYJc3Rvd3ppbGxhMRMwEQYKCZImiZPyLGQBGRYD
+  Y29tMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAupBquKI/4WvXOgND
+  pXyqH2GllZs1wG4TWWdn/DoMg45UoCwD+AWEuGrIdInBCpPN8vEJNJWPoM/RrU+b
+  xRBZT4uUk00bnZRW2SYh5GJSqBoBR+rWc2DGkXyGfdRU2sQvkB0+is6ChgQ61WMM
+  33LE9+loBlVsZ6EVtrc18Uh2OW0mJpe0hN2nmBrxZqqOZigxC4DKRMFHvpRkxSb6
+  mD4kit1AcwX9NEWJsXxrPaetL/SB/VbXaEZX93XAvp6USaXvCWt4slkDS2mIvqtn
+  9DtGC43LFC7SDGbnsG9PVenQgVCi8UWFPUAab0PqZSlmi3Qlbhw8qTGPp5Cbv4vz
+  qjC2UGPOQigA/7lbbGRhCohMrjOVHMAQwkcgiIqtolUoYlnvPMIy+m3pdvgDv/PH
+  bsZGvXQ7i0458xsmp1vaKthZocVAR+GboHbuIiYPUnO45ccXUQ00x6365tTe7mZi
+  NvmUYdAGbQmVvFqyxF7IYA6sF74L2Lstu0knSfss557bAe1HAgMBAAGjeTB3MAkG
+  A1UdEwQCMAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQWBBSnxTL/lNBCeLqpeVIX6AUY
+  kel4zjAeBgNVHREEFzAVgRNhZ2VudEBzdG93emlsbGEuY29tMB4GA1UdEgQXMBWB
+  E2FnZW50QHN0b3d6aWxsYS5jb20wDQYJKoZIhvcNAQELBQADggGBACm9Fjit/UCv
+  FxlKqeiCTIG94cIx+QrWAOJSx9knKydwUec1u04D/DbfZjTn3C2Bj227QgxeUn+6
+  if3e2v7zAk1896hLmGYzML0+nxQPb0vmtdLR7HETUlSKTVabcv1fbwLyjsuGrBvk
+  y51vOEzUEZ508a9yepLYqrQu1kOju4d57c9oA5l3H0mMKWz7av9tFj0B+STvuaWk
+  HRYDWc5HgOEVTyV+w0uFt2Kw4OCb8C42uSvC5RfYYtw78MSP+5Ru+LXJ7XOtmuN0
+  E6GVmofQ17ig9O3rgfFbMendSInrRmvPIGswvM1yivq9NOllFbdck2OJKPx6FCJF
+  7SJIkXQfc9P4B5iASIV1d1FsE0YX+g3jHXPJK/4mGL5bAyBKzpMfQB/mg6vQBzkh
+  aOKPwcreFj7TznBl89R5tNS9wZQfPVR98zgPyocddWhK18eQNMSBUnv4eeJ8PPbk
+  DovL+G8ajHDZ9fjH/+GVYHEMuiVdLarXrKJpHC1VfGTTUAp4NSEpUQ==
+  -----END CERTIFICATE-----
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
@@ -62,11 +89,13 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- certs/stowzilla.pem
 - lib/s3arch.rb
 - lib/s3arch/configuration.rb
 - lib/s3arch/handler.rb
 - lib/s3arch/indexer.rb
 - lib/s3arch/searcher.rb
+- lib/s3arch/tokenizer.rb
 - lib/s3arch/version.rb
 homepage: https://github.com/stowzilla/s3arch
 licenses:
@@ -74,7 +103,7 @@ licenses:
 metadata:
   rubygems_mfa_required: 'true'
   homepage_uri: https://github.com/stowzilla/s3arch
-  source_code_uri: https://github.com/stowzilla/s3arch
+  source_code_uri: https://github.com/stowzilla/s3arch/tree/main
   changelog_uri: https://github.com/stowzilla/s3arch/blob/main/CHANGELOG.md
 rdoc_options: []
 require_paths:
@@ -90,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.11
+rubygems_version: 3.6.9
 specification_version: 4
 summary: SQLite FTS5 full-text search for DynamoDB on AWS Lambda
 test_files: []

metadata.gz.sig

@@ -0,0 +1 @@
+Oco�
2��I�?D����0�̌��\�z,V��i���جԻX���f���f��$^�o������+"3�R�04q$�#`7���?���E�MF�A�	�x���f�����]���~j�]�/�IK�Dz#ni��B[XS������m��%��jQ+�a��_Ghf�a�J�"(o�~|I��y�&��kXڈL �Ϟ�j�Y�<��C���e�:�Pø�A��2X�P��)����nxL�y�<�L��je�L7���|������_�pG��p�nfPw���P_+�$�N�e�8�!����2�ա�̕g0#����WEdp�8��0�v�#_\G�^lU���(2jǨ��@�o^)��x�R%3���x���|��=���Ǻ