s3arch 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +32 -1
- data/lib/s3arch/tasks.rb +10 -0
- data/lib/s3arch/version.rb +1 -1
- data/lib/tasks/lambda_layer.rake +134 -0
- data.tar.gz.sig +2 -4
- metadata +17 -1
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b51661d90b5b65c15611e53f316b542cb6032dc34007adae6f8ee735baf6f0a3
|
|
4
|
+
data.tar.gz: cacd1cc9bcab08dcb1bae6dc9e0f757bc7a014a1dc92d907a1e90e7d1e99eab3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c4bafc88643be123cfaf047d245679feae7881a3e2b75b0c8c90c445dc8be01a51be7aabb96d19a0760b47dee9f92f851c5c35f1547e806c9bb0b21a749a4009
|
|
7
|
+
data.tar.gz: 7b137e16651e582d3b1a4582949133c91b992ea96b98659b11d415410b9bbf20edc5653b0614455c51b6b2f5a019292232fabb0732924e292130d6bfc59aeb8a
|
checksums.yaml.gz.sig
CHANGED
|
Binary file
|
data/README.md
CHANGED
|
@@ -26,7 +26,7 @@ Each owner (user, tenant, account) gets their own SQLite database. The indexer r
|
|
|
26
26
|
gem 's3arch'
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
Requires the `sqlite3` native extension available at runtime. On Lambda,
|
|
29
|
+
Requires the `sqlite3` native extension available at runtime. On Lambda, build the layer with `rake lambda:build_layer` (see [Building the Lambda Layer](#building-the-lambda-layer)).
|
|
30
30
|
|
|
31
31
|
## Usage
|
|
32
32
|
|
|
@@ -139,6 +139,36 @@ Outputs include `indexer_env_vars`, `searcher_env_vars`, `indexer_permissions`,
|
|
|
139
139
|
5. Apply metadata filters, sort by rank, return record IDs
|
|
140
140
|
6. LRU eviction when `/tmp` fills up
|
|
141
141
|
|
|
142
|
+
## Building the Lambda Layer
|
|
143
|
+
|
|
144
|
+
S3arch requires the `sqlite3` native extension at runtime. A Rake task is included to build and publish the Lambda layer:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
# Build the layer zip (outputs to pkg/sqlite-layer.zip)
|
|
148
|
+
rake lambda:build_layer
|
|
149
|
+
|
|
150
|
+
# Build and publish in one step
|
|
151
|
+
rake lambda:publish_layer PROFILE=devzilla
|
|
152
|
+
|
|
153
|
+
# Customize the build
|
|
154
|
+
rake lambda:build_layer RUBY_VERSION=3.4 ARCHITECTURE=x86_64
|
|
155
|
+
|
|
156
|
+
# Publish to a specific region/account
|
|
157
|
+
rake lambda:publish_layer PROFILE=production REGION=us-west-2 LAYER_NAME=stowzilla-sqlite3-ruby
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
| Environment Variable | Default | Description |
|
|
161
|
+
|---------------------|---------|-------------|
|
|
162
|
+
| `RUBY_VERSION` | `3.4` | Ruby runtime version |
|
|
163
|
+
| `ARCHITECTURE` | `x86_64` | `x86_64` or `arm64` |
|
|
164
|
+
| `OUTPUT` | `pkg/sqlite-layer.zip` | Output path for the zip |
|
|
165
|
+
| `PROFILE` | (none) | AWS CLI profile for publishing |
|
|
166
|
+
| `REGION` | `us-east-1` | AWS region to publish to |
|
|
167
|
+
| `LAYER_NAME` | `stowzilla-sqlite3-ruby` | Layer name in AWS |
|
|
168
|
+
| `S3ARCH_VERSION` | `~> current minor` | Version constraint for s3arch in the layer |
|
|
169
|
+
|
|
170
|
+
Requires Docker to be running (uses the official AWS SAM build images).
|
|
171
|
+
|
|
142
172
|
## Requirements
|
|
143
173
|
|
|
144
174
|
- Ruby >= 3.2
|
|
@@ -146,6 +176,7 @@ Outputs include `indexer_env_vars`, `searcher_env_vars`, `indexer_permissions`,
|
|
|
146
176
|
- SQLite3 native extension (via Lambda layer)
|
|
147
177
|
- DynamoDB table with streams enabled
|
|
148
178
|
- S3 bucket for index storage
|
|
179
|
+
- Docker (for building the Lambda layer)
|
|
149
180
|
|
|
150
181
|
## License
|
|
151
182
|
|
data/lib/s3arch/tasks.rb
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Load s3arch Rake tasks into the consuming application.
|
|
4
|
+
#
|
|
5
|
+
# Usage (in your Rakefile):
|
|
6
|
+
# require 's3arch/tasks'
|
|
7
|
+
#
|
|
8
|
+
# Or let your framework (e.g., belt) auto-require this.
|
|
9
|
+
|
|
10
|
+
Dir.glob(File.join(__dir__, '../../tasks/**/*.rake')).each { |r| load r }
|
data/lib/s3arch/version.rb
CHANGED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../s3arch/version'
|
|
4
|
+
|
|
5
|
+
namespace :lambda do
|
|
6
|
+
desc 'Build the Lambda layer zip (sqlite3 + s3arch) for deployment'
|
|
7
|
+
task :build_layer do
|
|
8
|
+
require 'tmpdir'
|
|
9
|
+
require 'fileutils'
|
|
10
|
+
|
|
11
|
+
ruby_version = ENV.fetch('RUBY_VERSION', '3.4')
|
|
12
|
+
architecture = ENV.fetch('ARCHITECTURE', 'x86_64')
|
|
13
|
+
output_file = File.expand_path(ENV.fetch('OUTPUT', 'pkg/sqlite-layer.zip'))
|
|
14
|
+
s3arch_version = ENV.fetch('S3ARCH_VERSION', "~> #{S3arch::VERSION.sub(/\.\d+$/, '.0')}")
|
|
15
|
+
|
|
16
|
+
FileUtils.mkdir_p(File.dirname(output_file))
|
|
17
|
+
|
|
18
|
+
puts "🔨 Building Lambda layer (Ruby #{ruby_version} / #{architecture}, s3arch #{s3arch_version})..."
|
|
19
|
+
|
|
20
|
+
tmpdir = Dir.mktmpdir('s3arch-layer')
|
|
21
|
+
begin
|
|
22
|
+
# Write a minimal Gemfile for the layer
|
|
23
|
+
File.write("#{tmpdir}/Gemfile", <<~GEMFILE)
|
|
24
|
+
source 'https://rubygems.org'
|
|
25
|
+
gem 'sqlite3', '~> 2.0'
|
|
26
|
+
gem 's3arch', '#{s3arch_version}'
|
|
27
|
+
GEMFILE
|
|
28
|
+
|
|
29
|
+
docker_image = "public.ecr.aws/sam/build-ruby#{ruby_version}:latest-#{architecture}"
|
|
30
|
+
platform = architecture == 'x86_64' ? 'linux/amd64' : 'linux/arm64'
|
|
31
|
+
|
|
32
|
+
build_script = <<~BASH
|
|
33
|
+
yum install -y sqlite-devel 2>/dev/null || dnf install -y sqlite-devel 2>/dev/null
|
|
34
|
+
bundle config set --local path /build/layer/ruby/gems/#{ruby_version}.0
|
|
35
|
+
bundle config set --local without "development test"
|
|
36
|
+
bundle config set --local deployment false
|
|
37
|
+
bundle install
|
|
38
|
+
# Flatten bundler's nested ruby/X.Y.0 structure into the layer root
|
|
39
|
+
NESTED="/build/layer/ruby/gems/#{ruby_version}.0/ruby/#{ruby_version}.0"
|
|
40
|
+
if [ -d "$NESTED" ]; then
|
|
41
|
+
mkdir -p /build/layer-final/ruby/gems/#{ruby_version}.0
|
|
42
|
+
cp -a "$NESTED"/* /build/layer-final/ruby/gems/#{ruby_version}.0/
|
|
43
|
+
rm -rf /build/layer
|
|
44
|
+
mv /build/layer-final /build/layer
|
|
45
|
+
fi
|
|
46
|
+
BASH
|
|
47
|
+
|
|
48
|
+
docker_cmd = [
|
|
49
|
+
'docker', 'run', '--rm',
|
|
50
|
+
'--platform', platform,
|
|
51
|
+
'-v', "#{tmpdir}:/build",
|
|
52
|
+
'-w', '/build',
|
|
53
|
+
docker_image,
|
|
54
|
+
'/bin/bash', '-c', build_script
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
puts "🐳 Building in Docker (#{docker_image})..."
|
|
58
|
+
abort '❌ Docker build failed' unless system(*docker_cmd)
|
|
59
|
+
|
|
60
|
+
layer_dir = "#{tmpdir}/layer"
|
|
61
|
+
|
|
62
|
+
# Strip unnecessary files to reduce layer size
|
|
63
|
+
puts '🧹 Stripping layer...'
|
|
64
|
+
Dir.glob("#{layer_dir}/ruby/gems/*/gems/*/").each do |gem_dir|
|
|
65
|
+
%w[spec test tests doc docs examples benchmarks].each do |fat_dir|
|
|
66
|
+
FileUtils.rm_rf("#{gem_dir}#{fat_dir}")
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
junk_patterns = %w[
|
|
71
|
+
*.md *.rdoc *.txt *.c *.h *.o Makefile *.log
|
|
72
|
+
CHANGELOG* HISTORY* LICENSE* README* .gitignore Rakefile
|
|
73
|
+
]
|
|
74
|
+
junk_patterns.each do |pattern|
|
|
75
|
+
Dir.glob("#{layer_dir}/**/#{pattern}").each { |f| FileUtils.rm_f(f) }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Strip debug symbols from shared objects
|
|
79
|
+
Dir.glob("#{layer_dir}/**/*.so").each do |so|
|
|
80
|
+
system('strip', '--strip-debug', so, err: File::NULL)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Verify native extension is present
|
|
84
|
+
sqlite_so = Dir.glob("#{layer_dir}/**/sqlite3_native.so") +
|
|
85
|
+
Dir.glob("#{layer_dir}/**/sqlite3.so")
|
|
86
|
+
abort '❌ sqlite3 native extension NOT found in layer' if sqlite_so.empty?
|
|
87
|
+
puts '✅ sqlite3 native extension included'
|
|
88
|
+
|
|
89
|
+
# Verify s3arch gem is present
|
|
90
|
+
s3arch_lib = Dir.glob("#{layer_dir}/**/gems/s3arch-*/lib/s3arch.rb")
|
|
91
|
+
abort '❌ s3arch gem NOT found in layer' if s3arch_lib.empty?
|
|
92
|
+
puts '✅ s3arch gem included'
|
|
93
|
+
|
|
94
|
+
# Package the zip
|
|
95
|
+
FileUtils.rm_f(output_file)
|
|
96
|
+
abort '❌ Failed to create zip' unless system('zip', '-qr', output_file, 'ruby/', chdir: layer_dir)
|
|
97
|
+
|
|
98
|
+
size = `du -h #{output_file}`.split("\t").first
|
|
99
|
+
puts "✅ Layer built: #{output_file} (#{size})"
|
|
100
|
+
ensure
|
|
101
|
+
# Docker creates files as root; use sudo rm to clean up
|
|
102
|
+
system('sudo', 'rm', '-rf', tmpdir) || FileUtils.rm_rf(tmpdir)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
desc 'Publish the Lambda layer to AWS (set PROFILE and REGION env vars)'
|
|
107
|
+
task publish_layer: :build_layer do
|
|
108
|
+
profile = ENV.fetch('PROFILE', nil)
|
|
109
|
+
region = ENV.fetch('REGION', 'us-east-1')
|
|
110
|
+
layer_name = ENV.fetch('LAYER_NAME', 'stowzilla-sqlite3-ruby')
|
|
111
|
+
ruby_version = ENV.fetch('RUBY_VERSION', '3.4')
|
|
112
|
+
architecture = ENV.fetch('ARCHITECTURE', 'x86_64')
|
|
113
|
+
output_file = File.expand_path(ENV.fetch('OUTPUT', 'pkg/sqlite-layer.zip'))
|
|
114
|
+
|
|
115
|
+
unless File.exist?(output_file)
|
|
116
|
+
abort "❌ Layer zip not found at #{output_file}. Run `rake lambda:build_layer` first."
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
cmd = %w[aws lambda publish-layer-version]
|
|
120
|
+
cmd += ['--layer-name', layer_name]
|
|
121
|
+
cmd += ['--zip-file', "fileb://#{output_file}"]
|
|
122
|
+
cmd += ['--compatible-runtimes', "ruby#{ruby_version}"]
|
|
123
|
+
cmd += ['--compatible-architectures', architecture]
|
|
124
|
+
cmd += ['--region', region]
|
|
125
|
+
cmd += ['--profile', profile] if profile
|
|
126
|
+
|
|
127
|
+
puts "📤 Publishing layer '#{layer_name}' to #{region}..."
|
|
128
|
+
puts " #{cmd.join(' ')}"
|
|
129
|
+
|
|
130
|
+
abort '❌ Failed to publish layer' unless system(*cmd)
|
|
131
|
+
|
|
132
|
+
puts '✅ Layer published successfully'
|
|
133
|
+
end
|
|
134
|
+
end
|
data.tar.gz.sig
CHANGED
|
@@ -1,4 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
k��<��^1��R
|
|
4
|
-
�W���^ b���$��;2f"�>�g{���Ĭ�� �G`�Pf�[�� �B���&ͽp�x{�[�k���u&
|
|
1
|
+
9Q�{�e[��a$)[���%Zb#s�1$�ؖ�j�p�7"��y ;� X��$y�*�H������яd�L���M�y H2ZZD���`��8*n�@tj3(��g����^a��@gV�{�q� 6t�>����3B�>�v��.F W�g��-ͤ< jgT ;�;u�H�&��H�D�$��E���8 �
|
|
2
|
+
�ݻ O�+悹m2���'��z��:��pV���&r�W��� q�`?/� @r�5���r� �p=J��3���֠�Fr��r��&t� T��qG��������A�"��0�&�g�Yo��ˮ��l����*S����Pd��!S�od[�{%������:��&
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: s3arch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Adam Dalton
|
|
@@ -64,6 +64,20 @@ dependencies:
|
|
|
64
64
|
- - "~>"
|
|
65
65
|
- !ruby/object:Gem::Version
|
|
66
66
|
version: '1.0'
|
|
67
|
+
- !ruby/object:Gem::Dependency
|
|
68
|
+
name: belt
|
|
69
|
+
requirement: !ruby/object:Gem::Requirement
|
|
70
|
+
requirements:
|
|
71
|
+
- - "~>"
|
|
72
|
+
- !ruby/object:Gem::Version
|
|
73
|
+
version: '0.0'
|
|
74
|
+
type: :runtime
|
|
75
|
+
prerelease: false
|
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
77
|
+
requirements:
|
|
78
|
+
- - "~>"
|
|
79
|
+
- !ruby/object:Gem::Version
|
|
80
|
+
version: '0.0'
|
|
67
81
|
- !ruby/object:Gem::Dependency
|
|
68
82
|
name: rack
|
|
69
83
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -116,9 +130,11 @@ files:
|
|
|
116
130
|
- lib/s3arch/routes.rb
|
|
117
131
|
- lib/s3arch/searcher.rb
|
|
118
132
|
- lib/s3arch/store.rb
|
|
133
|
+
- lib/s3arch/tasks.rb
|
|
119
134
|
- lib/s3arch/tokenizer.rb
|
|
120
135
|
- lib/s3arch/version.rb
|
|
121
136
|
- lib/s3arch/web.rb
|
|
137
|
+
- lib/tasks/lambda_layer.rake
|
|
122
138
|
homepage: https://github.com/stowzilla/s3arch
|
|
123
139
|
licenses:
|
|
124
140
|
- MIT
|
metadata.gz.sig
CHANGED
|
Binary file
|