s3arch 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e0ba51260081a3de075c6e14332909892693b64dce6eb1a3e52f7b1f145b537
-  data.tar.gz: 7553e90741ef8a1cc6c72b55bb9239c0e238da2a09916d34e8edf18780942a24
+  metadata.gz: 154d95c7316c2400be5958f3722bf84175559c0e0a6febf739e661852de955b8
+  data.tar.gz: d869f08480bd60443e7de00cb82042143a571134c7b2f1c14e4d2e56586be20c
 SHA512:
-  metadata.gz: 26e0cd3ff75ea9aff46dfd28a5978db557b26cef4780d966345b14fdb0b023bc9a6d22809c30c1422ac967aea3cbcc7bf18a75e94ea07d56af13d0f96d2d12e2
-  data.tar.gz: bf5c5b19de20e235f95427218b8beb0ee5ab03b4270bf31cbe38ca2f6f420c5665eb5b2a44e65461997a12412d4256099edaf3bc5be4ddb9f53f73105aaa3e79
+  metadata.gz: 0ee5a7e1426fad5be655a050b466ddb18dd34f3c61694d7fba9cbd7262ee87da867b1669493773a1c4ff803605bf70da74f8cef8ff6cbafff86c54fe1822d175
+  data.tar.gz: 0e35593c714d9bb102076026bb4d64a1bc0bf56bc11489f8864b1fd5b3cf941a5f0c4aa34e88dc1cb4df412a85871ea0acb48002c15243d6536ec3f477f52e4b

data/lambda/controllers/s3arch_controller.rb

@@ -1,20 +1,13 @@
 # frozen_string_literal: true
 
-require 'erb'
 require 'aws-sdk-dynamodb'
 
 class S3archController < BeltController::Base
-  VIEWS_PATH = File.expand_path('../../lib/s3arch/dashboard/views', __dir__)
-
   def index
-    @owners = fetch_owners
-    html = render_erb('index')
-    html_response(html)
+    owners = fetch_owners
+    success_response(owners: owners)
   rescue StandardError => e
-    @error = e.message
-    @owners = []
-    html = render_erb('index')
-    html_response(html, 500)
+    error_response("Failed to load owners: #{e.message}", 500)
   end
 
   def rebuild
@@ -58,9 +51,4 @@ class S3archController < BeltController::Base
     end
     owners.sort_by { |o| o[:updated_at].to_s }.reverse
   end
-
-  def render_erb(template)
-    path = File.join(VIEWS_PATH, "#{template}.html.erb")
-    ERB.new(File.read(path), trim_mode: '-').result(binding)
-  end
 end

data/lib/s3arch/indexer/stream_parser.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module S3arch
+  class Indexer
+    # Parses DynamoDB Stream events from SQS records into change operations.
+    module StreamParser
+      private
+
+      def group_changes(sqs_records)
+        grouped = Hash.new { |h, k| h[k] = [] }
+
+        sqs_records.each do |sqs_record|
+          stream_record = JSON.parse(sqs_record['body'])
+          event_name = stream_record['eventName']
+          new_image = stream_record.dig('dynamodb', 'NewImage')
+          old_image = stream_record.dig('dynamodb', 'OldImage')
+
+          owner_id = extract_owner(new_image || old_image)
+          next unless owner_id
+
+          change = build_change(event_name, new_image, old_image)
+          grouped[owner_id] << change if change
+        end
+
+        grouped
+      end
+
+      def build_change(event_name, new_image, old_image)
+        case event_name
+        when 'INSERT'
+          tokens = extract_tokens(new_image)
+          record_id = extract_record_id(new_image)
+          return nil unless tokens && record_id && passes_filter?(new_image)
+
+          { action: :insert, record_id: record_id, tokens: tokens, meta: extract_meta(new_image) }
+        when 'REMOVE'
+          tokens = extract_tokens(old_image)
+          record_id = extract_record_id(old_image)
+          return nil unless tokens && record_id
+
+          { action: :delete, record_id: record_id, tokens: tokens }
+        when 'MODIFY'
+          build_modify_change(new_image, old_image)
+        end
+      end
+
+      def build_modify_change(new_image, old_image)
+        old_tokens = extract_tokens(old_image)
+        new_tokens = extract_tokens(new_image)
+        record_id = extract_record_id(new_image)
+        return nil unless record_id
+
+        unless passes_filter?(new_image)
+          return old_tokens ? { action: :delete, record_id: record_id, tokens: old_tokens } : nil
+        end
+
+        unless old_tokens
+          return new_tokens ? { action: :insert, record_id: record_id, tokens: new_tokens,
+                                meta: extract_meta(new_image) } : nil
+        end
+
+        return nil unless new_tokens
+
+        { action: :update, record_id: record_id, old_tokens: old_tokens, new_tokens: new_tokens,
+          meta: extract_meta(new_image) }
+      end
+
+      def extract_owner(image)
+        return nil unless image
+
+        val = image[@config.owner_key]
+        val.is_a?(Hash) ? val['S'] : val
+      end
+
+      def extract_record_id(image)
+        return nil unless image
+
+        val = image['id']
+        val.is_a?(Hash) ? val['S'] : val
+      end
+
+      def extract_tokens(image)
+        return nil unless image
+
+        val = image[@config.token_field]
+        return nil unless val
+
+        if val.is_a?(Hash) && val.key?('M')
+          val['M'].transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
+        elsif val.is_a?(Hash) && !val.key?('S')
+          val.transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
+        end
+      end
+
+      def extract_meta(image)
+        @config.metadata_fields.each_with_object({}) do |field, meta|
+          val = image[field]
+          meta[field] = val.is_a?(Hash) ? (val['S'] || val['N'] || '') : val.to_s
+        end
+      end
+
+      def passes_filter?(image)
+        plain = image.transform_values { |v| v.is_a?(Hash) ? (v['S'] || v['N'] || v['BOOL']&.to_s || '') : v }
+        @config.record_filter.call(plain)
+      end
+    end
+  end
+end

data/lib/s3arch/indexer.rb

@@ -4,11 +4,15 @@ require 'aws-sdk-dynamodb'
 require 'aws-sdk-s3'
 require 'sqlite3'
 require 'json'
+require_relative 'indexer/stream_parser'
+
 module S3arch
   # Builds SQLite FTS5 databases per owner from pre-computed tokens stored in DynamoDB.
   # The indexer never sees raw content — only tokens. Supports incremental updates via
   # DynamoDB Stream events (INSERT/MODIFY/REMOVE).
   class Indexer
+    include StreamParser
+
     def initialize(config: S3arch.configuration)
       config.validate!
       @config = config
@@ -83,104 +87,6 @@ module S3arch
 
     private
 
-    def group_changes(sqs_records)
-      grouped = Hash.new { |h, k| h[k] = [] }
-
-      sqs_records.each do |sqs_record|
-        stream_record = JSON.parse(sqs_record['body'])
-        event_name = stream_record['eventName']
-        new_image = stream_record.dig('dynamodb', 'NewImage')
-        old_image = stream_record.dig('dynamodb', 'OldImage')
-
-        owner_id = extract_owner(new_image || old_image)
-        next unless owner_id
-
-        change = build_change(event_name, new_image, old_image)
-        grouped[owner_id] << change if change
-      end
-
-      grouped
-    end
-
-    def build_change(event_name, new_image, old_image)
-      case event_name
-      when 'INSERT'
-        tokens = extract_tokens(new_image)
-        record_id = extract_record_id(new_image)
-        return nil unless tokens && record_id && passes_filter?(new_image)
-
-        { action: :insert, record_id: record_id, tokens: tokens, meta: extract_meta(new_image) }
-      when 'REMOVE'
-        tokens = extract_tokens(old_image)
-        record_id = extract_record_id(old_image)
-        return nil unless tokens && record_id
-
-        { action: :delete, record_id: record_id, tokens: tokens }
-      when 'MODIFY'
-        old_tokens = extract_tokens(old_image)
-        new_tokens = extract_tokens(new_image)
-        record_id = extract_record_id(new_image)
-        return nil unless record_id
-
-        # If item no longer passes filter, treat as delete
-        unless passes_filter?(new_image)
-          return old_tokens ? { action: :delete, record_id: record_id, tokens: old_tokens } : nil
-        end
-
-        # If item previously didn't pass filter (no old tokens), treat as insert
-        unless old_tokens
-          return new_tokens ? { action: :insert, record_id: record_id, tokens: new_tokens,
-                                meta: extract_meta(new_image) } : nil
-        end
-
-        return nil unless new_tokens
-
-        { action: :update, record_id: record_id, old_tokens: old_tokens, new_tokens: new_tokens,
-          meta: extract_meta(new_image) }
-      end
-    end
-
-    def extract_owner(image)
-      return nil unless image
-
-      val = image[@config.owner_key]
-      val.is_a?(Hash) ? val['S'] : val
-    end
-
-    def extract_record_id(image)
-      return nil unless image
-
-      val = image['id']
-      val.is_a?(Hash) ? val['S'] : val
-    end
-
-    def extract_tokens(image)
-      return nil unless image
-
-      val = image[@config.token_field]
-      return nil unless val
-
-      # Token field is a DynamoDB Map: { "M": { "name": { "S": "..." }, "description": { "S": "..." } } }
-      if val.is_a?(Hash) && val.key?('M')
-        val['M'].transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
-      elsif val.is_a?(Hash) && !val.key?('S')
-        val.transform_values { |v| v.is_a?(Hash) ? (v['S'] || '') : v.to_s }
-      end
-    end
-
-    def extract_meta(image)
-      @config.metadata_fields.each_with_object({}) do |field, meta|
-        val = image[field]
-        meta[field] = val.is_a?(Hash) ? (val['S'] || val['N'] || '') : val.to_s
-      end
-    end
-
-    def passes_filter?(image)
-      # Convert DynamoDB image to plain hash for filter
-      plain = image.transform_values { |v| v.is_a?(Hash) ? (v['S'] || v['N'] || v['BOOL']&.to_s || '') : v }
-      @config.record_filter.call(plain)
-    end
-
     def apply_change(db, change)
       case change[:action]
       when :insert
@@ -252,7 +158,7 @@ module S3arch
         result.items.each do |item|
           next unless @config.record_filter.call(item)
 
-          tokens = item[@config.token_field]
+          tokens = extract_tokens_from_item(item)
           next unless tokens.is_a?(Hash) && tokens.any?
 
           records << { 'id' => item['id'], 'tokens' => tokens, 'meta' => extract_meta_from_item(item) }
@@ -265,9 +171,21 @@ module S3arch
       records
     end
 
+    def extract_tokens_from_item(item)
+      # Try the dedicated token field first
+      tokens = item[@config.token_field]
+      return tokens if tokens.is_a?(Hash) && tokens.any?
+
+      # Fall back: synthesize tokens from searchable_fields directly
+      @config.searchable_fields.each_with_object({}) do |field, map|
+        val = item[field]
+        map[field] = val.to_s if val
+      end
+    end
+
     def build_query_params(owner_id)
       fields = (['id', @config.token_field, @config.owner_key] +
-                @config.metadata_fields + @config.filter_fields).uniq
+                @config.searchable_fields + @config.metadata_fields + @config.filter_fields).compact.uniq
       expression_names = {}
       projected = fields.map { |f| reserved_word?(f) ? "##{f}".tap { |p| expression_names[p] = f } : f }
 

data/lib/s3arch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3arch
-  VERSION = '0.0.5'
+  VERSION = '0.0.7'
 end

data/lib/s3arch/web.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 # Lightweight S3arch load for web/API contexts (no SQLite dependency).
-# Provides configuration, holster registration, and routes only.
+# Provides configuration and routes only.
 require_relative 'version'
 require_relative 'configuration'
 require_relative 'routes'
-require_relative 'holster'
 
 module S3arch
   class Error < StandardError; end

data/lib/s3arch.rb

@@ -7,10 +7,6 @@ require_relative 's3arch/indexer'
 require_relative 's3arch/searcher'
 require_relative 's3arch/handler'
 require_relative 's3arch/routes'
-require_relative 's3arch/dashboard'
-
-# Register as a Belt holster when Belt is loaded
-require_relative 's3arch/holster' if defined?(Belt::Holster)
 
 module S3arch
   class Error < StandardError; end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3arch
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Adam Dalton
@@ -109,12 +109,9 @@ files:
 - lambda/controllers/s3arch_controller.rb
 - lib/s3arch.rb
 - lib/s3arch/configuration.rb
-- lib/s3arch/dashboard.rb
-- lib/s3arch/dashboard/application.rb
-- lib/s3arch/dashboard/views/index.html.erb
 - lib/s3arch/handler.rb
-- lib/s3arch/holster.rb
 - lib/s3arch/indexer.rb
+- lib/s3arch/indexer/stream_parser.rb
 - lib/s3arch/routes.rb
 - lib/s3arch/searcher.rb
 - lib/s3arch/tokenizer.rb

data/lib/s3arch/dashboard/application.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-require 'erb'
-require 'json'
-require 'aws-sdk-dynamodb'
-
-module S3arch
-  module Dashboard
-    class Application
-      VIEWS_PATH = File.expand_path('views', __dir__)
-
-      # Dispatcher mount DSL contract — keep in sync with #call dispatch below.
-      def routes
-        [
-          { method: :get, path: '/' },
-          { method: :post, path: '/rebuild' }
-        ]
-      end
-
-      def call(env)
-        req = Rack::Request.new(env)
-        path = req.path_info.sub(%r{^/}, '')
-
-        case [req.request_method, path]
-        when ['GET', ''], %w[GET index]
-          index(req)
-        when %w[POST rebuild]
-          rebuild(req)
-        else
-          [404, { 'content-type' => 'text/plain' }, ['Not Found']]
-        end
-      end
-
-      private
-
-      def index(req)
-        @env = req.env
-        @owners = fetch_owners
-        html = render('index')
-        [200, { 'content-type' => 'text/html' }, [html]]
-      end
-
-      def rebuild(req)
-        owner_id = req.params['owner_id']&.strip
-        if owner_id && !owner_id.empty?
-          indexer = S3arch::Indexer.new
-          indexer.rebuild(owner_id)
-        end
-        [303, { 'location' => "#{req.script_name}/" }, []]
-      end
-
-      def fetch_owners
-        config = S3arch.configuration
-        dynamodb = Aws::DynamoDB::Client.new
-        items = []
-        params = { table_name: config.version_table }
-
-        loop do
-          result = dynamodb.scan(params)
-          items.concat(result.items)
-          break unless result.last_evaluated_key
-
-          params[:exclusive_start_key] = result.last_evaluated_key
-        end
-
-        owners = items.map do |item|
-          {
-            owner_id: item[config.owner_key] || item.values.first,
-            version: item['version'],
-            record_count: item['record_count'],
-            updated_at: item['updated_at']
-          }
-        end
-        owners.sort_by { |o| o[:updated_at].to_s }.reverse
-      rescue StandardError => e
-        @error = e.message
-        []
-      end
-
-      def render(template)
-        path = File.join(VIEWS_PATH, "#{template}.html.erb")
-        ERB.new(File.read(path), trim_mode: '-').result(binding)
-      end
-    end
-  end
-end

data/lib/s3arch/dashboard/views/index.html.erb

@@ -1,150 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <title>S3arch Dashboard</title>
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <script>
-    (function() {
-      function getIdToken() {
-        try {
-          for (var i = 0; i < localStorage.length; i++) {
-            var key = localStorage.key(i);
-            if (key && key.indexOf('.idToken') !== -1) return localStorage.getItem(key);
-          }
-        } catch(e) {}
-        return null;
-      }
-      function isTokenValid(token) {
-        try {
-          var payload = JSON.parse(atob(token.split('.')[1].replace(/-/g,'+').replace(/_/g,'/')));
-          return payload.exp && payload.exp > Date.now() / 1000;
-        } catch(e) { return false; }
-      }
-      var token = getIdToken();
-      if (!token || !isTokenValid(token)) {
-        window.location.replace('/login');
-      }
-    })();
-  </script>
-  <style>
-    * { box-sizing: border-box; margin: 0; padding: 0; }
-    body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f5f5f5; color: #333; padding: 1.5rem; }
-    h1 { margin-bottom: 1rem; font-size: 1.5rem; }
-    .error { background: #fee; border: 1px solid #fcc; padding: 0.75rem; border-radius: 4px; margin-bottom: 1rem; color: #c00; }
-    .success { background: #efe; border: 1px solid #cfc; padding: 0.75rem; border-radius: 4px; margin-bottom: 1rem; color: #060; display: none; }
-    .meta { font-size: 0.85rem; color: #666; margin-bottom: 1rem; }
-    .empty { padding: 2rem; text-align: center; color: #666; }
-
-    /* Desktop: table */
-    table { width: 100%; border-collapse: collapse; background: #fff; border-radius: 6px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
-    th, td { padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid #eee; }
-    th { background: #fafafa; font-weight: 600; font-size: 0.85rem; text-transform: uppercase; letter-spacing: 0.03em; color: #555; }
-    tr:last-child td { border-bottom: none; }
-    tr:hover td { background: #f9fafb; }
-    .btn { border: none; padding: 0.4rem 0.8rem; border-radius: 4px; cursor: pointer; font-size: 0.85rem; font-weight: 500; transition: background 0.15s; }
-    .btn-rebuild { background: #2563eb; color: #fff; }
-    .btn-rebuild:hover { background: #1d4ed8; }
-    .btn-rebuild:disabled { background: #93c5fd; cursor: not-allowed; }
-    .owner-id { font-family: "SF Mono", SFMono-Regular, Consolas, monospace; font-size: 0.8rem; }
-
-    /* Mobile: card layout */
-    @media (max-width: 640px) {
-      body { padding: 1rem; }
-      table, thead, tbody, th, tr, td { display: block; }
-      thead { display: none; }
-      tr { background: #fff; border-radius: 6px; box-shadow: 0 1px 3px rgba(0,0,0,0.1); margin-bottom: 0.75rem; padding: 0.75rem; }
-      tr:hover td { background: transparent; }
-      td { padding: 0.25rem 0; border: none; display: flex; justify-content: space-between; align-items: center; }
-      td::before { content: attr(data-label); font-weight: 600; font-size: 0.75rem; text-transform: uppercase; color: #555; margin-right: 0.5rem; }
-      td:last-child { margin-top: 0.5rem; justify-content: flex-start; }
-      td:last-child::before { display: none; }
-      .btn { width: 100%; text-align: center; padding: 0.6rem; }
-    }
-  </style>
-</head>
-<body>
-  <h1>S3arch Dashboard</h1>
-  <p class="meta"><%= @owners.size %> indexed owner<%= @owners.size == 1 ? '' : 's' %></p>
-
-  <div id="flash" class="success"></div>
-
-  <%- if @error -%>
-  <div class="error"><strong>Error:</strong> <%= @error %></div>
-  <%- end -%>
-
-  <%- if @owners.any? -%>
-  <table>
-    <thead>
-      <tr>
-        <th>Owner</th>
-        <th>Version</th>
-        <th>Records</th>
-        <th>Last Updated</th>
-        <th>Actions</th>
-      </tr>
-    </thead>
-    <tbody>
-      <%- @owners.each do |owner| -%>
-      <tr>
-        <td class="owner-id" data-label="Owner"><%= owner[:owner_id] %></td>
-        <td data-label="Version"><%= owner[:version].to_i %></td>
-        <td data-label="Records"><%= owner[:record_count] ? owner[:record_count].to_i.to_s.gsub(/(\d)(?=(\d{3})+$)/, '\1,') : '—' %></td>
-        <td data-label="Updated"><%= owner[:updated_at] || '—' %></td>
-        <td>
-          <button class="btn btn-rebuild" onclick="rebuild('<%= owner[:owner_id] %>', this)">Recreate SQLite</button>
-        </td>
-      </tr>
-      <%- end -%>
-    </tbody>
-  </table>
-  <%- else -%>
-  <div class="empty">No indexed owners found.</div>
-  <%- end -%>
-
-  <script>
-    function getIdToken() {
-      try {
-        for (var i = 0; i < localStorage.length; i++) {
-          var key = localStorage.key(i);
-          if (key && key.indexOf('.idToken') !== -1) return localStorage.getItem(key);
-        }
-      } catch(e) {}
-      return null;
-    }
-
-    function rebuild(ownerId, btn) {
-      btn.disabled = true;
-      btn.textContent = 'Rebuilding\u2026';
-      var headers = { 'Content-Type': 'application/json' };
-      var token = getIdToken();
-      if (token) headers['Authorization'] = 'Bearer ' + token;
-
-      fetch(window.location.pathname.replace(/\/$/, '') + '/rebuild', {
-        method: 'POST',
-        headers: headers,
-        body: JSON.stringify({ owner_id: ownerId })
-      }).then(function(r) { return r.json(); }).then(function(data) {
-        btn.textContent = 'Recreate SQLite';
-        btn.disabled = false;
-        var flash = document.getElementById('flash');
-        if (data.status === 'ok') {
-          flash.className = 'success';
-          flash.textContent = 'Rebuild triggered for ' + ownerId;
-          flash.style.display = 'block';
-        } else {
-          flash.className = 'error';
-          flash.textContent = data.error || 'Rebuild failed';
-          flash.style.display = 'block';
-        }
-      }).catch(function(e) {
-        btn.textContent = 'Recreate SQLite';
-        btn.disabled = false;
-        var flash = document.getElementById('flash');
-        flash.className = 'error';
-        flash.textContent = 'Request failed: ' + e.message;
-        flash.style.display = 'block';
-      });
-    }
-  </script>
-</body>
-</html>

data/lib/s3arch/dashboard.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-require 'erb'
-require_relative 'dashboard/application'
-
-module S3arch
-  module Dashboard
-    def self.app
-      Application.new
-    end
-  end
-end

data/lib/s3arch/holster.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module S3arch
-  class Holster < Belt::Holster
-    self.gem_root = File.expand_path('../..', __dir__)
-  end
-end