s3arch 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd57db198f9598324792e16e5b84e9c22914ebfdfd4fddeb09fc99641458322c
-  data.tar.gz: e107d35cd4b7084d3cf90e3d014bb4fe3c6089475b26419524d9132867565592
+  metadata.gz: 6e0ba51260081a3de075c6e14332909892693b64dce6eb1a3e52f7b1f145b537
+  data.tar.gz: 7553e90741ef8a1cc6c72b55bb9239c0e238da2a09916d34e8edf18780942a24
 SHA512:
-  metadata.gz: 65aba03e3064bf597dc8bb4e53d7552c07f0b89ebb7ca24b9e6c74a7f5c22f9c95746cec237d1e100fce63cb8f285f782487300abd7f4d1b6e1c76553fdad5ff
-  data.tar.gz: 28f5d435ce384c0c1c8e05d17d478ad9fdb9e0a949506350384886fdfcedbfaa2cbd9308dfd08d2faf91dfe4bf02ac07ac6d6749470143914a977f29ab0aef04
+  metadata.gz: 26e0cd3ff75ea9aff46dfd28a5978db557b26cef4780d966345b14fdb0b023bc9a6d22809c30c1422ac967aea3cbcc7bf18a75e94ea07d56af13d0f96d2d12e2
+  data.tar.gz: bf5c5b19de20e235f95427218b8beb0ee5ab03b4270bf31cbe38ca2f6f420c5665eb5b2a44e65461997a12412d4256099edaf3bc5be4ddb9f53f73105aaa3e79

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## [0.0.5] - 2025-06-12
+
+### Added
+
+- `S3arch::Dashboard` — mountable Rack application for viewing index stats (records, versions, owner counts)
+- `S3arch::Dashboard::Application` — ERB-rendered HTML dashboard with mobile-friendly layout
+- `S3arch::Holster` — Belt holster integration for convention-over-configuration mounting
+- `S3arch::Routes` — Dispatcher mount DSL compatibility (`#routes` method)
+- `S3arch::Web` — lightweight Rack entry point for Lambda integration
+- `S3archController` — Belt controller for dashboard with Cognito auth gate
+- `Configuration#dashboard_auth` — configurable authentication for dashboard access
+- localStorage-based auth for CloudFront same-origin serving
+
+### Changed
+
+- Dashboard renders HTML via ERB instead of JSON
+- Scientific notation fix for large record/version numbers in dashboard
+
+## [0.0.4] - 2025-06-09
+
+### Fixed
+
+- `Indexer#fetch_records` now uses `expression_attribute_names` for DynamoDB reserved words (e.g., `status`), fixing `ValidationException` on `rebuild()`
+
+### Added
+
+- `Configuration#filter_fields` — declare which fields the `record_filter` proc needs projected from DynamoDB
+
+### Removed
+
+- Hardcoded `filter_fields` method (`%w[status bin_id]`) — replaced by configurable `filter_fields`
+
 ## [0.0.2] - 2025-06-08
 
 ### Added

data/infrastructure/routes.tf.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+TerraDispatch.routes.draw do
+  namespace :s3arch, auth: :cognito, tables: [:search_indexes] do
+    get '/', action: 'index'
+    post '/rebuild', action: 'rebuild'
+  end
+end

data/infrastructure/schema.tf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+TerraDispatch.schema.define do
+  # S3arch does not define request/response models yet.
+  # The search_indexes table is provisioned by s3arch's own Terraform module.
+end

data/lambda/controllers/s3arch_controller.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'aws-sdk-dynamodb'
+
+class S3archController < BeltController::Base
+  VIEWS_PATH = File.expand_path('../../lib/s3arch/dashboard/views', __dir__)
+
+  def index
+    @owners = fetch_owners
+    html = render_erb('index')
+    html_response(html)
+  rescue StandardError => e
+    @error = e.message
+    @owners = []
+    html = render_erb('index')
+    html_response(html, 500)
+  end
+
+  def rebuild
+    owner_id = params['owner_id']&.strip
+    return error_response('owner_id is required', 400) if owner_id.nil? || owner_id.empty?
+
+    handler = S3arch.configuration.rebuild_handler
+    if handler
+      handler.call(owner_id)
+    else
+      S3arch::Indexer.new.rebuild(owner_id)
+    end
+    success_response(status: 'ok', owner_id: owner_id)
+  rescue StandardError => e
+    error_response("Failed to rebuild index: #{e.message}", 500)
+  end
+
+  private
+
+  def fetch_owners
+    dynamodb = Aws::DynamoDB::Client.new
+    config = S3arch.configuration
+    items = []
+    scan_params = { table_name: config.version_table }
+
+    loop do
+      result = dynamodb.scan(scan_params)
+      items.concat(result.items)
+      break unless result.last_evaluated_key
+
+      scan_params[:exclusive_start_key] = result.last_evaluated_key
+    end
+
+    owners = items.map do |item|
+      {
+        owner_id: item[config.owner_key] || item.values.first,
+        version: item['version'],
+        record_count: item['record_count'],
+        updated_at: item['updated_at']
+      }
+    end
+    owners.sort_by { |o| o[:updated_at].to_s }.reverse
+  end
+
+  def render_erb(template)
+    path = File.join(VIEWS_PATH, "#{template}.html.erb")
+    ERB.new(File.read(path), trim_mode: '-').result(binding)
+  end
+end

data/lib/s3arch/configuration.rb

@@ -30,6 +30,9 @@ module S3arch
     # Filter proc — receives a record hash, returns true to include in index
     attr_accessor :record_filter
 
+    # Fields needed by the record_filter for projection (e.g., %w[status])
+    attr_accessor :filter_fields
+
     # Owner extractor — proc that extracts owner_id from a DynamoDB stream record
     attr_accessor :owner_extractor
 
@@ -39,12 +42,18 @@ module S3arch
     # Searcher settings
     attr_accessor :version_ttl, :max_results, :max_cached_dbs, :ephemeral_storage_mb
 
+    # Custom rebuild handler — proc/lambda that receives owner_id.
+    # When set, the controller uses this instead of calling Indexer.new.rebuild directly.
+    # Useful for triggering rebuilds via Lambda invocation instead of in-process.
+    attr_accessor :rebuild_handler
+
     def initialize
       @owner_key = 'user_id'
       @searchable_fields = %w[name description]
       @token_field = 'searchTokens'
       @metadata_fields = %w[status created_at]
       @record_filter = ->(_record) { true }
+      @filter_fields = %w[status]
       @owner_extractor = lambda { |stream_record|
         image = stream_record.dig('dynamodb', 'NewImage') || stream_record.dig('dynamodb', 'OldImage') || {}
         image.dig(owner_key, 'S')

data/lib/s3arch/dashboard/application.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'json'
+require 'aws-sdk-dynamodb'
+
+module S3arch
+  module Dashboard
+    class Application
+      VIEWS_PATH = File.expand_path('views', __dir__)
+
+      # Dispatcher mount DSL contract — keep in sync with #call dispatch below.
+      def routes
+        [
+          { method: :get, path: '/' },
+          { method: :post, path: '/rebuild' }
+        ]
+      end
+
+      def call(env)
+        req = Rack::Request.new(env)
+        path = req.path_info.sub(%r{^/}, '')
+
+        case [req.request_method, path]
+        when ['GET', ''], %w[GET index]
+          index(req)
+        when %w[POST rebuild]
+          rebuild(req)
+        else
+          [404, { 'content-type' => 'text/plain' }, ['Not Found']]
+        end
+      end
+
+      private
+
+      def index(req)
+        @env = req.env
+        @owners = fetch_owners
+        html = render('index')
+        [200, { 'content-type' => 'text/html' }, [html]]
+      end
+
+      def rebuild(req)
+        owner_id = req.params['owner_id']&.strip
+        if owner_id && !owner_id.empty?
+          indexer = S3arch::Indexer.new
+          indexer.rebuild(owner_id)
+        end
+        [303, { 'location' => "#{req.script_name}/" }, []]
+      end
+
+      def fetch_owners
+        config = S3arch.configuration
+        dynamodb = Aws::DynamoDB::Client.new
+        items = []
+        params = { table_name: config.version_table }
+
+        loop do
+          result = dynamodb.scan(params)
+          items.concat(result.items)
+          break unless result.last_evaluated_key
+
+          params[:exclusive_start_key] = result.last_evaluated_key
+        end
+
+        owners = items.map do |item|
+          {
+            owner_id: item[config.owner_key] || item.values.first,
+            version: item['version'],
+            record_count: item['record_count'],
+            updated_at: item['updated_at']
+          }
+        end
+        owners.sort_by { |o| o[:updated_at].to_s }.reverse
+      rescue StandardError => e
+        @error = e.message
+        []
+      end
+
+      def render(template)
+        path = File.join(VIEWS_PATH, "#{template}.html.erb")
+        ERB.new(File.read(path), trim_mode: '-').result(binding)
+      end
+    end
+  end
+end

data/lib/s3arch/dashboard/views/index.html.erb

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>S3arch Dashboard</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <script>
+    (function() {
+      function getIdToken() {
+        try {
+          for (var i = 0; i < localStorage.length; i++) {
+            var key = localStorage.key(i);
+            if (key && key.indexOf('.idToken') !== -1) return localStorage.getItem(key);
+          }
+        } catch(e) {}
+        return null;
+      }
+      function isTokenValid(token) {
+        try {
+          var payload = JSON.parse(atob(token.split('.')[1].replace(/-/g,'+').replace(/_/g,'/')));
+          return payload.exp && payload.exp > Date.now() / 1000;
+        } catch(e) { return false; }
+      }
+      var token = getIdToken();
+      if (!token || !isTokenValid(token)) {
+        window.location.replace('/login');
+      }
+    })();
+  </script>
+  <style>
+    * { box-sizing: border-box; margin: 0; padding: 0; }
+    body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f5f5f5; color: #333; padding: 1.5rem; }
+    h1 { margin-bottom: 1rem; font-size: 1.5rem; }
+    .error { background: #fee; border: 1px solid #fcc; padding: 0.75rem; border-radius: 4px; margin-bottom: 1rem; color: #c00; }
+    .success { background: #efe; border: 1px solid #cfc; padding: 0.75rem; border-radius: 4px; margin-bottom: 1rem; color: #060; display: none; }
+    .meta { font-size: 0.85rem; color: #666; margin-bottom: 1rem; }
+    .empty { padding: 2rem; text-align: center; color: #666; }
+
+    /* Desktop: table */
+    table { width: 100%; border-collapse: collapse; background: #fff; border-radius: 6px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
+    th, td { padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid #eee; }
+    th { background: #fafafa; font-weight: 600; font-size: 0.85rem; text-transform: uppercase; letter-spacing: 0.03em; color: #555; }
+    tr:last-child td { border-bottom: none; }
+    tr:hover td { background: #f9fafb; }
+    .btn { border: none; padding: 0.4rem 0.8rem; border-radius: 4px; cursor: pointer; font-size: 0.85rem; font-weight: 500; transition: background 0.15s; }
+    .btn-rebuild { background: #2563eb; color: #fff; }
+    .btn-rebuild:hover { background: #1d4ed8; }
+    .btn-rebuild:disabled { background: #93c5fd; cursor: not-allowed; }
+    .owner-id { font-family: "SF Mono", SFMono-Regular, Consolas, monospace; font-size: 0.8rem; }
+
+    /* Mobile: card layout */
+    @media (max-width: 640px) {
+      body { padding: 1rem; }
+      table, thead, tbody, th, tr, td { display: block; }
+      thead { display: none; }
+      tr { background: #fff; border-radius: 6px; box-shadow: 0 1px 3px rgba(0,0,0,0.1); margin-bottom: 0.75rem; padding: 0.75rem; }
+      tr:hover td { background: transparent; }
+      td { padding: 0.25rem 0; border: none; display: flex; justify-content: space-between; align-items: center; }
+      td::before { content: attr(data-label); font-weight: 600; font-size: 0.75rem; text-transform: uppercase; color: #555; margin-right: 0.5rem; }
+      td:last-child { margin-top: 0.5rem; justify-content: flex-start; }
+      td:last-child::before { display: none; }
+      .btn { width: 100%; text-align: center; padding: 0.6rem; }
+    }
+  </style>
+</head>
+<body>
+  <h1>S3arch Dashboard</h1>
+  <p class="meta"><%= @owners.size %> indexed owner<%= @owners.size == 1 ? '' : 's' %></p>
+
+  <div id="flash" class="success"></div>
+
+  <%- if @error -%>
+  <div class="error"><strong>Error:</strong> <%= @error %></div>
+  <%- end -%>
+
+  <%- if @owners.any? -%>
+  <table>
+    <thead>
+      <tr>
+        <th>Owner</th>
+        <th>Version</th>
+        <th>Records</th>
+        <th>Last Updated</th>
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      <%- @owners.each do |owner| -%>
+      <tr>
+        <td class="owner-id" data-label="Owner"><%= owner[:owner_id] %></td>
+        <td data-label="Version"><%= owner[:version].to_i %></td>
+        <td data-label="Records"><%= owner[:record_count] ? owner[:record_count].to_i.to_s.gsub(/(\d)(?=(\d{3})+$)/, '\1,') : '—' %></td>
+        <td data-label="Updated"><%= owner[:updated_at] || '—' %></td>
+        <td>
+          <button class="btn btn-rebuild" onclick="rebuild('<%= owner[:owner_id] %>', this)">Recreate SQLite</button>
+        </td>
+      </tr>
+      <%- end -%>
+    </tbody>
+  </table>
+  <%- else -%>
+  <div class="empty">No indexed owners found.</div>
+  <%- end -%>
+
+  <script>
+    function getIdToken() {
+      try {
+        for (var i = 0; i < localStorage.length; i++) {
+          var key = localStorage.key(i);
+          if (key && key.indexOf('.idToken') !== -1) return localStorage.getItem(key);
+        }
+      } catch(e) {}
+      return null;
+    }
+
+    function rebuild(ownerId, btn) {
+      btn.disabled = true;
+      btn.textContent = 'Rebuilding\u2026';
+      var headers = { 'Content-Type': 'application/json' };
+      var token = getIdToken();
+      if (token) headers['Authorization'] = 'Bearer ' + token;
+
+      fetch(window.location.pathname.replace(/\/$/, '') + '/rebuild', {
+        method: 'POST',
+        headers: headers,
+        body: JSON.stringify({ owner_id: ownerId })
+      }).then(function(r) { return r.json(); }).then(function(data) {
+        btn.textContent = 'Recreate SQLite';
+        btn.disabled = false;
+        var flash = document.getElementById('flash');
+        if (data.status === 'ok') {
+          flash.className = 'success';
+          flash.textContent = 'Rebuild triggered for ' + ownerId;
+          flash.style.display = 'block';
+        } else {
+          flash.className = 'error';
+          flash.textContent = data.error || 'Rebuild failed';
+          flash.style.display = 'block';
+        }
+      }).catch(function(e) {
+        btn.textContent = 'Recreate SQLite';
+        btn.disabled = false;
+        var flash = document.getElementById('flash');
+        flash.className = 'error';
+        flash.textContent = 'Request failed: ' + e.message;
+        flash.style.display = 'block';
+      });
+    }
+  </script>
+</body>
+</html>

data/lib/s3arch/dashboard.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'erb'
+require_relative 'dashboard/application'
+
+module S3arch
+  module Dashboard
+    def self.app
+      Application.new
+    end
+  end
+end

data/lib/s3arch/holster.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module S3arch
+  class Holster < Belt::Holster
+    self.gem_root = File.expand_path('../..', __dir__)
+  end
+end

data/lib/s3arch/indexer.rb

@@ -4,7 +4,6 @@ require 'aws-sdk-dynamodb'
 require 'aws-sdk-s3'
 require 'sqlite3'
 require 'json'
-
 module S3arch
   # Builds SQLite FTS5 databases per owner from pre-computed tokens stored in DynamoDB.
   # The indexer never sees raw content — only tokens. Supports incremental updates via
@@ -80,6 +79,8 @@ module S3arch
       { statusCode: 200, body: JSON.generate(rebuilt: grouped.size) }
     end
 
+    RESERVED_WORDS = Set.new(%w[status name comment count size type]).freeze
+
     private
 
     def group_changes(sqs_records)
@@ -244,11 +245,7 @@ module S3arch
     # Full rebuild: fetches token field from DynamoDB (never reads content)
     def fetch_records(owner_id)
       records = []
-      fields_to_project = ['id', @config.token_field, @config.owner_key] + @config.metadata_fields + filter_fields
-      params = { table_name: @config.source_table, index_name: @config.source_index,
-                 key_condition_expression: "#{@config.owner_key} = :owner",
-                 expression_attribute_values: { ':owner' => owner_id },
-                 projection_expression: fields_to_project.uniq.join(', ') }
+      params = build_query_params(owner_id)
 
       loop do
         result = @dynamodb.query(params)
@@ -268,17 +265,29 @@ module S3arch
       records
     end
 
-    def extract_meta_from_item(item)
-      @config.metadata_fields.to_h do |field|
-        [field, item[field].to_s]
-      end
+    def build_query_params(owner_id)
+      fields = (['id', @config.token_field, @config.owner_key] +
+                @config.metadata_fields + @config.filter_fields).uniq
+      expression_names = {}
+      projected = fields.map { |f| reserved_word?(f) ? "##{f}".tap { |p| expression_names[p] = f } : f }
+
+      owner_placeholder = reserved_word?(@config.owner_key) ? "##{@config.owner_key}" : @config.owner_key
+      expression_names["##{@config.owner_key}"] = @config.owner_key if reserved_word?(@config.owner_key)
+
+      params = { table_name: @config.source_table, index_name: @config.source_index,
+                 key_condition_expression: "#{owner_placeholder} = :owner",
+                 expression_attribute_values: { ':owner' => owner_id },
+                 projection_expression: projected.join(', ') }
+      params[:expression_attribute_names] = expression_names if expression_names.any?
+      params
     end
 
-    def filter_fields
-      # Fields needed by the record_filter (best-effort — add status/bin_id for default filter)
-      %w[status bin_id]
+    def extract_meta_from_item(item)
+      @config.metadata_fields.to_h { |field| [field, item[field].to_s] }
     end
 
+    def reserved_word?(field) = RESERVED_WORDS.include?(field.downcase)
+
     def build_database(db_path, records)
       FileUtils.rm_f(db_path)
       db = SQLite3::Database.new(db_path)

data/lib/s3arch/routes.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module S3arch
+  # Lightweight route manifest for Dispatcher mount DSL.
+  # No heavy dependencies — safe to load in parse-only contexts.
+  module Routes
+    def self.routes
+      [
+        { method: :get, path: '/' },
+        { method: :post, path: '/rebuild' }
+      ]
+    end
+  end
+end

data/lib/s3arch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3arch
-  VERSION = '0.0.3'
+  VERSION = '0.0.5'
 end

data/lib/s3arch/web.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Lightweight S3arch load for web/API contexts (no SQLite dependency).
+# Provides configuration, holster registration, and routes only.
+require_relative 'version'
+require_relative 'configuration'
+require_relative 'routes'
+require_relative 'holster'
+
+module S3arch
+  class Error < StandardError; end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+  end
+end

data/lib/s3arch.rb

@@ -6,6 +6,11 @@ require_relative 's3arch/tokenizer'
 require_relative 's3arch/indexer'
 require_relative 's3arch/searcher'
 require_relative 's3arch/handler'
+require_relative 's3arch/routes'
+require_relative 's3arch/dashboard'
+
+# Register as a Belt holster when Belt is loaded
+require_relative 's3arch/holster' if defined?(Belt::Holster)
 
 module S3arch
   class Error < StandardError; end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3arch
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Adam Dalton
@@ -64,6 +64,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -78,8 +92,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-description: Per-owner SQLite FTS5 indexes stored on S3, queried from Lambda /tmp.
-  DynamoDB source records are indexed via streams, with version tracking and LRU caching.
+description: Per-owner SQLite FTS5 indexes stored on S3, queried from Lambda /tmp
+  with version tracking.
 email:
 - adam@stowzilla.com
 executables: []
@@ -90,13 +104,22 @@ files:
 - LICENSE.txt
 - README.md
 - certs/stowzilla.pem
+- infrastructure/routes.tf.rb
+- infrastructure/schema.tf.rb
+- lambda/controllers/s3arch_controller.rb
 - lib/s3arch.rb
 - lib/s3arch/configuration.rb
+- lib/s3arch/dashboard.rb
+- lib/s3arch/dashboard/application.rb
+- lib/s3arch/dashboard/views/index.html.erb
 - lib/s3arch/handler.rb
+- lib/s3arch/holster.rb
 - lib/s3arch/indexer.rb
+- lib/s3arch/routes.rb
 - lib/s3arch/searcher.rb
 - lib/s3arch/tokenizer.rb
 - lib/s3arch/version.rb
+- lib/s3arch/web.rb
 homepage: https://github.com/stowzilla/s3arch
 licenses:
 - MIT