s3arch 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89f4fce79a53db6b2890bf9034c5702e556ce27a1f399e8d4c605b8953fcd033
-  data.tar.gz: 13b9092d968a9aa1a44dcacbe9bd9ede1304e8a9e67623219245e0f601a9d467
+  metadata.gz: 86ab9ff26fb027515006bee668f41d7d5f264fc301110d57f6ac4d365b426588
+  data.tar.gz: bb2a513c4f7252620668c7292357d0e314a60d9efae70738c5ffd127f386d4b2
 SHA512:
-  metadata.gz: 1c2093cac0b0e12bc0a9fe679d218fa13aa6b1dfeb35b1bd69bef03dbf1699eaba32084931330e3322993981d541787966e270803e566716ea4b61de9edaddfe
-  data.tar.gz: c6d81a49d383c03c03638a0d6548acc7df697b3fa3f1dc6bc26f8347bf03bb6f29a63707af68baa8920f45dceea74f3fe886533a4914c3af281149271f0b00e1
+  metadata.gz: 6def4b186fc609041e90c7246d1a9fb0453aeb01ae7c880040d9c8154a94eb514505adeab291191094c8e5e755e9fb753ccf699dd04cabccc672b0c7f74ad4cc
+  data.tar.gz: d1e054232f1d30443340ddd06f2a264458166b030a69b823327f82a61025f4065c000fadba2bbfc82d45949ee69b00e25d5be350d12a09309e4efab133d49503

checksums.yaml.gz.sig

@@ -1,2 +1,2 @@
-s���%�1�Ro[h�
�bIo��
-}�S�̭��	Y錣�E�����I�1�$|,lư�ˇ��O?��\���Vm���!���ov����e�:�r,�JO*��70�������<��!�G�BAA�����Ý5��4?\$ߴD�{ܭw��G��ǧK�U���M����q�E��>�l�����I�(%@С8KϷ�T(
+j��YC�����㯬`�Q����[��M~��Ҫ�q\Tr���Ih/4Z
�]~�Қ�p�X�|;s��B��nP����R���
�X��Ꙁ�V -S6Ev�X�!ͦo���W9Y6�H$�04�6q[�4�E�P�(��HÊ���+$��[	��L�~��!T~��A���
+N����ʽ˛[�O�C�k����*���Q��

data/README.md

@@ -26,7 +26,7 @@ Each owner (user, tenant, account) gets their own SQLite database. The indexer r
 gem 's3arch'
 ```
 
-Requires the `sqlite3` native extension available at runtime. On Lambda, use a layer that provides it (e.g., [stowzilla-sqlite3-ruby](https://github.com/stowzilla/stowzilla-sqlite3-ruby)).
+Requires the `sqlite3` native extension available at runtime. On Lambda, build the layer with `rake lambda:build_layer` (see [Building the Lambda Layer](#building-the-lambda-layer)).
 
 ## Usage
 
@@ -139,6 +139,51 @@ Outputs include `indexer_env_vars`, `searcher_env_vars`, `indexer_permissions`,
 5. Apply metadata filters, sort by rank, return record IDs
 6. LRU eviction when `/tmp` fills up
 
+## Building the Lambda Layer
+
+S3arch requires the `sqlite3` native extension at runtime. A Rake task is included to build and publish the Lambda layer:
+
+```bash
+# Build the layer zip (outputs to pkg/sqlite-layer.zip)
+rake s3arch:layer:build
+
+# Build and publish in one step
+rake s3arch:layer:publish PROFILE=devzilla
+
+# Customize the build
+rake s3arch:layer:build RUBY_VERSION=3.4 ARCHITECTURE=x86_64
+
+# Publish to a specific region/account
+rake s3arch:layer:publish PROFILE=production REGION=us-west-2 LAYER_NAME=stowzilla-sqlite3-ruby
+```
+
+| Environment Variable | Default | Description |
+|---------------------|---------|-------------|
+| `RUBY_VERSION` | `3.4` | Ruby runtime version |
+| `ARCHITECTURE` | `x86_64` | `x86_64` or `arm64` |
+| `OUTPUT` | `pkg/sqlite-layer.zip` | Output path for the zip |
+| `PROFILE` | (none) | AWS CLI profile for publishing |
+| `REGION` | `us-east-1` | AWS region to publish to |
+| `LAYER_NAME` | `stowzilla-sqlite3-ruby` | Layer name in AWS |
+| `S3ARCH_VERSION` | `~> current minor` | Version constraint for s3arch in the layer |
+
+Requires Docker to be running (uses the official AWS SAM build images).
+
+## Rake Tasks
+
+```bash
+rake s3arch:layer:build    # Build the Lambda layer zip
+rake s3arch:layer:publish  # Build and publish to AWS
+rake s3arch:rebuild        # Rebuild index for an owner (OWNER_ID=xxx)
+rake s3arch:info           # Show config and version info
+```
+
+To make these tasks available in your app, add to your Rakefile:
+
+```ruby
+require 's3arch/tasks'
+```
+
 ## Requirements
 
 - Ruby >= 3.2
@@ -146,6 +191,7 @@ Outputs include `indexer_env_vars`, `searcher_env_vars`, `indexer_permissions`,
 - SQLite3 native extension (via Lambda layer)
 - DynamoDB table with streams enabled
 - S3 bucket for index storage
+- Docker (for building the Lambda layer)
 
 ## License
 

data/lib/s3arch/tasks.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Load s3arch Rake tasks into the consuming application.
+#
+# Usage (in your Rakefile):
+#   require 's3arch/tasks'
+#
+# Or let your framework (e.g., belt) auto-require this.
+
+Dir.glob(File.join(__dir__, '../../tasks/**/*.rake')).each { |r| load r }

data/lib/s3arch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3arch
-  VERSION = '0.1.1'
+  VERSION = '0.1.3'
 end

data/lib/tasks/s3arch.rake

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative '../s3arch/version'
+
+namespace :s3arch do
+  namespace :layer do
+    desc 'Build the Lambda layer zip (defaults: RUBY_VERSION=3.4, ARCHITECTURE=x86_64)'
+    task :build do
+      require 'tmpdir'
+      require 'fileutils'
+
+      ruby_version   = ENV.fetch('RUBY_VERSION', '3.4')
+      architecture   = ENV.fetch('ARCHITECTURE', 'x86_64')
+      output_file    = File.expand_path(ENV.fetch('OUTPUT', 'pkg/sqlite-layer.zip'))
+      s3arch_version = ENV.fetch('S3ARCH_VERSION', "~> #{S3arch::VERSION.sub(/\.\d+$/, '.0')}")
+
+      FileUtils.mkdir_p(File.dirname(output_file))
+
+      puts "🔨 Building Lambda layer (Ruby #{ruby_version} / #{architecture}, s3arch #{s3arch_version})..."
+
+      tmpdir = Dir.mktmpdir('s3arch-layer')
+      begin
+        File.write("#{tmpdir}/Gemfile", <<~GEMFILE)
+          source 'https://rubygems.org'
+          gem 'sqlite3', '~> 2.0'
+          gem 's3arch', '#{s3arch_version}'
+        GEMFILE
+
+        docker_image = "public.ecr.aws/sam/build-ruby#{ruby_version}:latest-#{architecture}"
+        platform = architecture == 'x86_64' ? 'linux/amd64' : 'linux/arm64'
+
+        build_script = <<~BASH
+          yum install -y sqlite-devel 2>/dev/null || dnf install -y sqlite-devel 2>/dev/null
+          bundle config set --local path /build/layer/ruby/gems/#{ruby_version}.0
+          bundle config set --local without "development test"
+          bundle config set --local deployment false
+          bundle install
+          NESTED="/build/layer/ruby/gems/#{ruby_version}.0/ruby/#{ruby_version}.0"
+          if [ -d "$NESTED" ]; then
+            mkdir -p /build/layer-final/ruby/gems/#{ruby_version}.0
+            cp -a "$NESTED"/* /build/layer-final/ruby/gems/#{ruby_version}.0/
+            rm -rf /build/layer
+            mv /build/layer-final /build/layer
+          fi
+        BASH
+
+        docker_cmd = [
+          'docker', 'run', '--rm',
+          '--platform', platform,
+          '-v', "#{tmpdir}:/build",
+          '-w', '/build',
+          docker_image,
+          '/bin/bash', '-c', build_script
+        ]
+
+        puts "🐳 Building in Docker (#{docker_image})..."
+        abort '❌ Docker build failed' unless system(*docker_cmd)
+
+        layer_dir = "#{tmpdir}/layer"
+
+        puts '🧹 Stripping layer...'
+        Dir.glob("#{layer_dir}/ruby/gems/*/gems/*/").each do |gem_dir|
+          %w[spec test tests doc docs examples benchmarks].each do |fat_dir|
+            FileUtils.rm_rf("#{gem_dir}#{fat_dir}")
+          end
+        end
+
+        junk_patterns = %w[
+          *.md *.rdoc *.txt *.c *.h *.o Makefile *.log
+          CHANGELOG* HISTORY* LICENSE* README* .gitignore Rakefile
+        ]
+        junk_patterns.each do |pattern|
+          Dir.glob("#{layer_dir}/**/#{pattern}").each { |f| FileUtils.rm_f(f) }
+        end
+
+        Dir.glob("#{layer_dir}/**/*.so").each do |so|
+          system('strip', '--strip-debug', so, err: File::NULL)
+        end
+
+        sqlite_so = Dir.glob("#{layer_dir}/**/sqlite3_native.so") +
+                    Dir.glob("#{layer_dir}/**/sqlite3.so")
+        abort '❌ sqlite3 native extension NOT found in layer' if sqlite_so.empty?
+        puts '✅ sqlite3 native extension included'
+
+        s3arch_lib = Dir.glob("#{layer_dir}/**/gems/s3arch-*/lib/s3arch.rb")
+        abort '❌ s3arch gem NOT found in layer' if s3arch_lib.empty?
+        puts '✅ s3arch gem included'
+
+        FileUtils.rm_f(output_file)
+        abort '❌ Failed to create zip' unless system('zip', '-qr', output_file, 'ruby/', chdir: layer_dir)
+
+        size = `du -h #{output_file}`.split("\t").first
+        puts "✅ Layer built: #{output_file} (#{size})"
+      ensure
+        system('sudo', 'rm', '-rf', tmpdir) || FileUtils.rm_rf(tmpdir)
+      end
+    end
+
+    desc 'Build and publish the Lambda layer (set AWS_PROFILE, LAYER_NAME=stowzilla-sqlite3-ruby)'
+    task publish: :build do
+      layer_name   = ENV.fetch('LAYER_NAME', 'stowzilla-sqlite3-ruby')
+      ruby_version = ENV.fetch('RUBY_VERSION', '3.4')
+      architecture = ENV.fetch('ARCHITECTURE', 'x86_64')
+      output_file  = File.expand_path(ENV.fetch('OUTPUT', 'pkg/sqlite-layer.zip'))
+
+      abort "❌ Layer zip not found at #{output_file}" unless File.exist?(output_file)
+
+      cmd = %w[aws lambda publish-layer-version]
+      cmd += ['--layer-name', layer_name]
+      cmd += ['--zip-file', "fileb://#{output_file}"]
+      cmd += ['--compatible-runtimes', "ruby#{ruby_version}"]
+      cmd += ['--compatible-architectures', architecture]
+
+      puts "📤 Publishing layer '#{layer_name}'..."
+      puts "   #{cmd.join(' ')}"
+
+      abort '❌ Failed to publish layer' unless system(*cmd)
+
+      puts '✅ Layer published successfully'
+    end
+  end
+
+  desc 'Rebuild the search index for a given owner: rake s3arch:rebuild[owner-123]'
+  task :rebuild, [:owner_id] do |_t, args|
+    require 's3arch'
+
+    owner_id = args.owner_id || ENV.fetch('OWNER_ID', nil)
+    abort '❌ Usage: rake s3arch:rebuild[OWNER_ID]' unless owner_id
+
+    S3arch.configure { |c| c.from_env! }
+
+    puts "🔄 Rebuilding index for #{owner_id}..."
+    S3arch::Indexer.new.rebuild(owner_id)
+    puts '✅ Done'
+  end
+
+  desc 'Show current s3arch configuration and version info'
+  task :info do
+    require 's3arch'
+
+    puts "s3arch #{S3arch::VERSION}"
+    puts ''
+    puts 'Environment:'
+    %w[S3ARCH_SOURCE_TABLE S3ARCH_SOURCE_INDEX S3ARCH_INDEX_BUCKET S3ARCH_VERSION_TABLE].each do |var|
+      puts "  #{var}=#{ENV.fetch(var, '(not set)')}"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3arch
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Adam Dalton
@@ -130,9 +130,11 @@ files:
 - lib/s3arch/routes.rb
 - lib/s3arch/searcher.rb
 - lib/s3arch/store.rb
+- lib/s3arch/tasks.rb
 - lib/s3arch/tokenizer.rb
 - lib/s3arch/version.rb
 - lib/s3arch/web.rb
+- lib/tasks/s3arch.rake
 homepage: https://github.com/stowzilla/s3arch
 licenses:
 - MIT