rails-sqlite-extras 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a885ac19f93d301c20fa64d0a49f1f742ec5e99fdc2c1640560a0292e6f3d69d
-  data.tar.gz: 8326db26cf1606a796aec10edd886d424cc85a8e1e582aa53ccd333d32008997
+  metadata.gz: e6d98e686e16d43b33d634d912d20a24bbe265a212ca1a634a497b577d3f13bf
+  data.tar.gz: 5f8e86e60d4e9ac302edf9f7857293c242d03c5622a852b30bd038e20aaa6adf
 SHA512:
-  metadata.gz: 853ba689f98e3bd9ea863a3ed84250e0e6e483847d3e3f71494c252eb6b97759aa62a7c6de233b71ad577ac3d62d6303e52b92243961a5032ead192a03702eca
-  data.tar.gz: b740b5b22380c5e229f98c8bb23c4bd0855e0c6bc43c39d004a75dc488b3a356b49858583444f945aece0174d251880e5bd4989956dbaea7642db526bb60cbad
+  metadata.gz: 1e2a85241730e526e6fb4926786904b825a797aa56a7354ad6a095f0b2e7577da1a2e7539f84985b61c38fdeb62f0f30f18df5f7ec52781b5dbf20ce71d6a6e4
+  data.tar.gz: f85072769f7c50592258f87b71fd2ef25901cdf2d310bcc4b3bf40d6174639102b4869dc5d4d06d3a3c63881f94125a511e0d9aeaa30fc2f90df2c2737362de2

data/.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.3', '3.2', '3.1', '3.0', '2.7', '2.6']
+        ruby-version: ['3.3', '3.2', '3.1']
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby ${{ matrix.ruby-version }}
@@ -24,6 +24,8 @@ jobs:
           gem install bundler -v 2.4.22
           sudo apt-get update --allow-releaseinfo-change
           bundle config set --local path 'vendor/bundle'
+          bundle config set force_ruby_platform true
+          bundle config set build.sqlite3 "--with-sqlite-cflags='-DSQLITE_ENABLE_DBSTAT_VTAB=1'"
           bundle install
           sleep 2
       - name: Run tests 

data/.gitignore

@@ -1,4 +1,5 @@
 Gemfile.lock
+.bundle/
 .ruby-version
 pkg/
 *.gem

data/README.md

@@ -1,3 +1,140 @@
 # Rails Sqlite Extras [![Gem Version](https://badge.fury.io/rb/rails-sqlite-extras.svg)](https://badge.fury.io/rb/rails-sqlite-extras) [![GH Actions](https://github.com/pawurb/rails-sqlite-extras/actions/workflows/ci.yml/badge.svg)](https://github.com/pawurb/rails-sqlite-extras/actions)
 
-WIP
+Copy/paste of [ecto_sqlite3_extras](https://github.com/orsinium-labs/ecto_sqlite3_extras). Helper queries available in Ruby and rake tasks providing insights into the Sqlite database.
+
+## Installation
+
+Some queries use `dbstat` table which has to be enabled via a `SQLITE_ENABLE_DBSTAT_VTAB` compile flag:
+
+```bash
+bundle config set force_ruby_platform true
+bundle config set build.sqlite3 "--with-sqlite-cflags='-DSQLITE_ENABLE_DBSTAT_VTAB=1'"
+```
+
+`Gemfile`
+```bash
+gem 'rails-sqlite-extras'
+```
+
+## Available queries
+
+### `total_size`
+
+```bash
+rake sqlite_extras:total_size
+```
+
+```ruby
+RailsSqliteExtras.total_size
+```
+
+The total size of all tables and indices. It's a summary table, it has only 2 columns: `name` and `value`. Rows:
+
+- `cells`: The number of cells in the DB. Each value stored in the DB is represented as at least one cell. So, the number of cells correlates with the number of records in the DB.
+- `payload_size`: How much space the actual useful payload takes in the DB.
+- `unused_size`: How much space in the DB is reserved, not used yet, and can be used later to store more data. This is a surplus that occurs because SQLite allocates space for data in chunks ("pages").
+- `vacuum_size`: How much space is unused and cannot be used for future data. You can run [VACUUM](https://www.sqlite.org/lang_vacuum.html) command to reduce it.
+- `page_size`: The total space occupied by all pages. Each page is a single chunk of space allocated by SQLite. This number is the sum of `payload_size`, `unused_size`, and `vacuum_size`.
+- `pages`: The total number of pages.
+- `pages: leaf`: The pages that store the actual data. Read [SQLite Internals: Pages & B-trees](https://fly.io/blog/sqlite-internals-btree/) to learn more.
+- `pages: internal`: The pages that store ranges for leaf pages for a faster lookup. Sometimes also called "interior pages".
+- `pages: overflow`: The pages that store chunks of big data that don't fit in a single leaf page.
+- `pages: table`: The pages used for storing data for tables.
+- `pages: index`: The pages used for storing indices.
+
+### `table_size`
+
+```bash
+rake sqlite_extras:table_size
+```
+
+```ruby
+RailsSqliteExtras.table_size
+```
+
+Information about the space used (and unused) by all tables. Based on the [dbstat](https://www.sqlite.org/dbstat.html) virtual table.
+
+- `name`: The table name.
+- `payload_size`.
+- `unused_size`.
+- `vacuum_size`.
+- `page_size`.
+- `cells`.
+- `pages`.
+- `max_payload_size`: The size of the biggest payload in the table.
+
+### `index_size`
+
+```bash
+rake sqlite_extras:index_size
+```
+
+```ruby
+RailsSqliteExtras.index_size
+```
+
+Size of all indices.
+
+- `name`: The index name.
+- `table_name`: The table where the index is defined.
+- `column_name`: The name of the column being indexed. This column is NULL if the column is the rowid or an expression.
+- `payload_size`.
+- `unused_size`.
+- `page_size`.
+- `cells`.
+- `pages`.
+- `max_payload_size`.
+
+### `sequence_number`
+
+```bash
+rake sqlite_extras:sequence_number
+```
+
+```ruby
+RailsSqliteExtras.sequence_number
+```
+
+Sequence numbers of autoincrement columns. Generated based on the [sqlite_sequence](https://renenyffenegger.ch/notes/development/databases/SQLite/internals/schema-objects/sqlite_sequence) table. The query will fail if there are no autoincrement columns in the DB yet.
+
+- `table_name`.
+- `sequence_number`.
+
+### `pragma`
+
+```bash
+rake sqlite_extras:pragma
+```
+
+```ruby
+RailsSqliteExtras.pragma
+```
+
+List values of PRAGMAs (settings). Only includes the ones that have an integer or a boolean value. For brevity, the ones with the `0` (`false`) value are excluded from the output (based on the observation that this is the default value for most of the PRAGMAs). Check out the SQLite documentation to learn more about what each PRAGMA means: [PRAGMA Statements](https://www.sqlite.org/pragma.html).
+
+- `name`: the name of the PRAGMA as listed in the SQLite documentation.
+- `value`: the value of the PRAGMA. The `true` value is converted into `1` (and `false` is simply excluded).
+
+### `compile_options` 
+
+```bash
+rake sqlite_extras:compile_options
+```
+
+```ruby
+RailsSqliteExtras.compile_options
+```
+
+List the [compile-time options](https://www.sqlite.org/compile.html) used when building SQLite, one option per row. The "SQLITE_" prefix is omitted from the returned option names. See [exqlite docs](https://github.com/elixir-sqlite/exqlite#defining-extra-compile-flags) on how to change these options.
+
+### `integrity_check`
+
+```bash
+rake sqlite_extras:integrity_check
+```
+
+```ruby
+RailsSqliteExtras.integrity_check
+```
+
+Run integrity checks on the database. Executes [PRAGMA integrity_check](https://www.sqlite.org/pragma.html#pragma_integrity_check) and returns the resulting messages.

data/lib/rails-sqlite-extras.rb

@@ -1,7 +1,97 @@
 # frozen_string_literal: true
 
 module RailsSqliteExtras
-  def self.hello
-    "there"
+  @@database_url = nil
+
+  QUERIES = %i(
+    compile_options
+    index_size
+    integrity_check
+    pragma
+    sequence_number
+    table_size
+    total_size
+  )
+
+  QUERIES.each do |query_name|
+    define_singleton_method query_name do |options = {}|
+      run_query(
+        query_name: query_name,
+        in_format: options.fetch(:in_format, :display_table),
+      )
+    end
+  end
+
+  def self.run_query(query_name:, in_format:)
+    sql = sql_for(query_name: query_name)
+
+    result = connection.execute(sql)
+
+    self.display_result(
+      result,
+      title: self.description_for(query_name: query_name),
+      in_format: in_format,
+    )
+  end
+
+  def self.display_result(result, title:, in_format:)
+    case in_format
+    when :array
+      result.values
+    when :hash
+      result.to_a
+    when :raw
+      result
+    when :display_table
+      headings = if result.count > 0
+          result[0].keys
+        else
+          ["No results"]
+        end
+
+      puts Terminal::Table.new(
+        title: title,
+        headings: headings,
+        rows: result.values,
+      )
+    else
+      raise "Invalid in_format option"
+    end
+  end
+
+  def self.sql_for(query_name:)
+    File.read(
+      sql_path_for(query_name: query_name)
+    )
+  end
+
+  def self.description_for(query_name:)
+    first_line = File.open(
+      sql_path_for(query_name: query_name)
+    ) { |f| f.readline }
+
+    first_line[/\/\*(.*?)\*\//m, 1].strip
+  end
+
+  def self.sql_path_for(query_name:)
+    File.join(File.dirname(__FILE__), "/rails_sqlite_extras/queries/#{query_name}.sql")
+  end
+
+  def self.connection
+    if (db_url = ENV["RAILS_SQLITE_EXTRAS_DATABASE_URL"])
+      ActiveRecord::Base.establish_connection(db_url).lease_connection
+    else
+      ActiveRecord::Base.connection
+    end
+  end
+
+  def self.database_url=(value)
+    @@database_url = value
+  end
+
+  def self.database_url
+    @@database_url || ENV["RUBY_PG_EXTRAS_DATABASE_URL"] || ENV.fetch("DATABASE_URL")
   end
 end
+
+require "rails_sqlite_extras/railtie" if defined?(Rails)

data/lib/rails_sqlite_extras/queries/compile_options.sql

@@ -0,0 +1,3 @@
+/* List the compile-time options used when building SQLite. */
+
+SELECT compile_options AS 'value' FROM pragma_compile_options;

data/lib/rails_sqlite_extras/queries/index_size.sql

@@ -0,0 +1,21 @@
+/* Metadata of the indexes, descending by size. */
+
+SELECT
+    d.name        AS name,
+    s.tbl_name    AS table_name,
+    i.name        AS column_name,
+    d.payload     AS payload_size,
+    d.unused      AS unused_size,
+    d.pgsize      AS page_size,
+    d.ncell       AS cells,
+    d.pageno      AS pages,
+    d.mx_payload  AS max_payload_size
+FROM
+    dbstat AS d,
+    sqlite_schema AS s,
+    pragma_index_info(d.name) AS i
+WHERE
+        d.name      = s.name
+    AND s.type      = 'index'
+    AND d.aggregate = TRUE
+ORDER BY page_size DESC;

data/lib/rails_sqlite_extras/queries/integrity_check.sql

@@ -0,0 +1,4 @@
+/* Run integrity checks on the database. */
+
+SELECT integrity_check as message
+FROM pragma_integrity_check;

data/lib/rails_sqlite_extras/queries/pragma.sql

@@ -0,0 +1,35 @@
+/* List values of PRAGMAs (settings). */
+
+SELECT *
+FROM (
+    SELECT 'analysis_limit' AS name, analysis_limit AS value FROM pragma_analysis_limit
+    UNION ALL SELECT 'application_id', *          FROM pragma_application_id
+    UNION ALL SELECT 'auto_vacuum', *             FROM pragma_auto_vacuum
+    UNION ALL SELECT 'automatic_index', *         FROM pragma_automatic_index
+    UNION ALL SELECT 'busy_timeout', *            FROM pragma_busy_timeout
+    UNION ALL SELECT 'cache_size', *              FROM pragma_cache_size
+    UNION ALL SELECT 'cache_spill', *             FROM pragma_cache_spill
+    UNION ALL SELECT 'cell_size_check', *         FROM pragma_cell_size_check
+    UNION ALL SELECT 'checkpoint_fullfsync', *    FROM pragma_checkpoint_fullfsync
+    UNION ALL SELECT 'data_version', *            FROM pragma_data_version
+    UNION ALL SELECT 'defer_foreign_keys', *      FROM pragma_defer_foreign_keys
+    UNION ALL SELECT 'freelist_count', *          FROM pragma_freelist_count
+    UNION ALL SELECT 'fullfsync', *               FROM pragma_fullfsync
+    UNION ALL SELECT 'hard_heap_limit', *         FROM pragma_hard_heap_limit
+    UNION ALL SELECT 'ignore_check_constraints', * FROM pragma_ignore_check_constraints
+    UNION ALL SELECT 'journal_size_limit', *      FROM pragma_journal_size_limit
+    UNION ALL SELECT 'legacy_alter_table', *      FROM pragma_legacy_alter_table
+    UNION ALL SELECT 'max_page_count', *          FROM pragma_max_page_count
+    UNION ALL SELECT 'page_size', *               FROM pragma_page_size
+    UNION ALL SELECT 'query_only', *              FROM pragma_query_only
+    UNION ALL SELECT 'read_uncommitted', *        FROM pragma_read_uncommitted
+    UNION ALL SELECT 'recursive_triggers', *      FROM pragma_recursive_triggers
+    UNION ALL SELECT 'reverse_unordered_selects', * FROM pragma_reverse_unordered_selects
+    UNION ALL SELECT 'soft_heap_limit', *         FROM pragma_soft_heap_limit
+    UNION ALL SELECT 'synchronous', *             FROM pragma_synchronous
+    UNION ALL SELECT 'temp_store', *              FROM pragma_temp_store
+    UNION ALL SELECT 'threads', *                 FROM pragma_threads
+    UNION ALL SELECT 'trusted_schema', *          FROM pragma_trusted_schema
+    UNION ALL SELECT 'user_version', *            FROM pragma_user_version
+)
+WHERE value != 0

data/lib/rails_sqlite_extras/queries/sequence_number.sql

@@ -0,0 +1,5 @@
+/* Sequence numbers of autoincrement columns. */
+
+SELECT name AS table_name, seq AS sequence_number
+FROM sqlite_sequence
+ORDER BY sequence_number DESC

data/lib/rails_sqlite_extras/queries/table_size.sql

@@ -0,0 +1,15 @@
+/* Metadata of the tables (excluding indexes), descending by size. */
+SELECT
+    name,
+    payload AS payload_size,
+    unused as unused_size,
+    pgsize - payload - unused as vacuum_size,
+    pgsize as page_size,
+    ncell as cells,
+    pageno as pages,
+    mx_payload as max_payload_size
+FROM dbstat
+WHERE
+    name IN (SELECT name FROM sqlite_schema WHERE type='table')
+    AND aggregate=TRUE
+ORDER BY page_size DESC;

data/lib/rails_sqlite_extras/queries/total_size.sql

@@ -0,0 +1,18 @@
+/* The total size of all tables and indexes. */
+SELECT 'pages' AS name,      MAX(pageno) as value FROM dbstat
+UNION ALL SELECT 'cells',    SUM(ncell)    FROM dbstat
+UNION ALL SELECT 'payload_size',  SUM(payload)  FROM dbstat
+UNION ALL SELECT 'unused_size',   SUM(unused)   FROM dbstat
+UNION ALL SELECT 'vacuum_size',   SUM(pgsize) - SUM(payload) - SUM(unused) FROM dbstat
+UNION ALL SELECT 'page_size',     SUM(pgsize)   FROM dbstat
+
+UNION ALL SELECT 'pages: leaf', COUNT(*)
+    FROM dbstat WHERE pagetype = 'leaf'
+UNION ALL SELECT 'pages: internal', COUNT(*)
+    FROM dbstat WHERE pagetype = 'internal'
+UNION ALL SELECT 'pages: overflow', COUNT(*)
+    FROM dbstat WHERE pagetype = 'overflow'
+UNION ALL SELECT 'pages: table', COUNT(*)
+    FROM dbstat WHERE name IN (SELECT name FROM sqlite_schema WHERE type='table')
+UNION ALL SELECT 'pages: index', COUNT(*)
+    FROM dbstat WHERE name IN (SELECT name FROM sqlite_schema WHERE type='index')

data/lib/rails_sqlite_extras/railtie.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RailsSqliteExtras::Railtie < Rails::Railtie
+  rake_tasks do
+    load "rails_sqlite_extras/tasks/all.rake"
+  end
+end

data/lib/rails_sqlite_extras/tasks/all.rake

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "rails-sqlite-extras"
+
+namespace :sqlite_extras do
+  RailsSqliteExtras::QUERIES.each do |query_name|
+    desc RailsSqliteExtras.description_for(query_name: query_name)
+    task query_name.to_sym => :environment do
+      RailsSqliteExtras.public_send(query_name)
+    end
+  end
+end

data/lib/rails_sqlite_extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsSqliteExtras
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/rails-sqlite-extras.gemspec

@@ -11,11 +11,12 @@ Gem::Specification.new do |s|
   s.summary = %q{ Rails Sqlite database insights. }
   s.description = %q{ Helper queries providing insights into the Sqlite database for Rails apps. }
   s.homepage = "http://github.com/pawurb/rails-sqlite-extras"
-  s.files = `git ls-files`.split("\n")
+  s.files = `git ls-files`.split("\n").reject { |f| f.match?(/\.db$/) }
   s.test_files = s.files.grep(%r{^(spec)/})
   s.require_paths = ["lib"]
   s.license = "MIT"
   s.add_dependency "rails"
+  s.add_dependency "sqlite3"
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "rufo"

data/spec/smoke_spec.rb

@@ -4,7 +4,24 @@ require "spec_helper"
 require "rails-sqlite-extras"
 
 describe RailsSqliteExtras do
-  it "works" do
-    expect(RailsSqliteExtras.hello).to eq("there")
+  RailsSqliteExtras::QUERIES.each do |query_name|
+    it "#{query_name} description can be read" do
+      expect do
+        RailsSqliteExtras.description_for(
+          query_name: query_name,
+        )
+      end.not_to raise_error
+    end
+  end
+
+  RailsSqliteExtras::QUERIES.reject { |q| q == :kill_all }.each do |query_name|
+    it "#{query_name} query can be executed" do
+      expect do
+        RailsSqliteExtras.run_query(
+          query_name: query_name,
+          in_format: :hash,
+        )
+      end.not_to raise_error
+    end
   end
 end

data/spec/spec_helper.rb

@@ -7,6 +7,7 @@ require_relative "../lib/rails-sqlite-extras"
 
 RSpec.configure do |config|
   config.before :suite do
+    ENV["RAILS_SQLITE_EXTRAS_DATABASE_URL"] = "sqlite3://#{Dir.pwd}/test.db"
   end
 
   config.after :suite do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails-sqlite-extras
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-14 00:00:00.000000000 Z
+date: 2024-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -81,6 +95,15 @@ files:
 - README.md
 - Rakefile
 - lib/rails-sqlite-extras.rb
+- lib/rails_sqlite_extras/queries/compile_options.sql
+- lib/rails_sqlite_extras/queries/index_size.sql
+- lib/rails_sqlite_extras/queries/integrity_check.sql
+- lib/rails_sqlite_extras/queries/pragma.sql
+- lib/rails_sqlite_extras/queries/sequence_number.sql
+- lib/rails_sqlite_extras/queries/table_size.sql
+- lib/rails_sqlite_extras/queries/total_size.sql
+- lib/rails_sqlite_extras/railtie.rb
+- lib/rails_sqlite_extras/tasks/all.rake
 - lib/rails_sqlite_extras/version.rb
 - rails-sqlite-extras.gemspec
 - spec/smoke_spec.rb