rails-sqlite-extras 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a885ac19f93d301c20fa64d0a49f1f742ec5e99fdc2c1640560a0292e6f3d69d
-  data.tar.gz: 8326db26cf1606a796aec10edd886d424cc85a8e1e582aa53ccd333d32008997
+  metadata.gz: 61ce5e28ddc420085b0afa20233dee41e6ae2e4201734d40f41c34945dcb6e73
+  data.tar.gz: 722883255ff6ad48e88da3bdf555ca432e1681e79ae472ff297500e83d5d7e9d
 SHA512:
-  metadata.gz: 853ba689f98e3bd9ea863a3ed84250e0e6e483847d3e3f71494c252eb6b97759aa62a7c6de233b71ad577ac3d62d6303e52b92243961a5032ead192a03702eca
-  data.tar.gz: b740b5b22380c5e229f98c8bb23c4bd0855e0c6bc43c39d004a75dc488b3a356b49858583444f945aece0174d251880e5bd4989956dbaea7642db526bb60cbad
+  metadata.gz: 64b83a0a0a7b00c8575ac8672b9ebc4a6d158a3720b81b707038b9e04b890f588775d516237f2aef1ce97600ae65590238c05ee6c02e73854d0aef4609dc4931
+  data.tar.gz: 74cbff0f00743a51186f214faee86b011be071001bcb386ce869517f0b598ad8e3bce8bc666eadb71fe473c39e7fd6ddaab8bf041e9f152744f14600855db0a8

data/.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.3', '3.2', '3.1', '3.0', '2.7', '2.6']
+        ruby-version: ['3.3', '3.2', '3.1']
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby ${{ matrix.ruby-version }}
@@ -24,6 +24,8 @@ jobs:
           gem install bundler -v 2.4.22
           sudo apt-get update --allow-releaseinfo-change
           bundle config set --local path 'vendor/bundle'
+          bundle config set force_ruby_platform true
+          bundle config set build.sqlite3 "--with-sqlite-cflags='-DSQLITE_ENABLE_DBSTAT_VTAB=1'"
           bundle install
           sleep 2
       - name: Run tests 

data/.gitignore

@@ -1,4 +1,5 @@
 Gemfile.lock
+.bundle/
 .ruby-version
 pkg/
 *.gem

data/README.md

@@ -1,3 +1,112 @@
 # Rails Sqlite Extras [![Gem Version](https://badge.fury.io/rb/rails-sqlite-extras.svg)](https://badge.fury.io/rb/rails-sqlite-extras) [![GH Actions](https://github.com/pawurb/rails-sqlite-extras/actions/workflows/ci.yml/badge.svg)](https://github.com/pawurb/rails-sqlite-extras/actions)
 
-WIP
+Copy/paste of [ecto_sqlite3_extras](https://github.com/orsinium-labs/ecto_sqlite3_extras). Helper queries providing insights into the Sqlite database.
+
+## Installation
+
+Some queries use `dbstat` table which has to be enabled via a `SQLITE_ENABLE_DBSTAT_VTAB` compile flag:
+
+```bash
+bundle config set force_ruby_platform true
+bundle config set build.sqlite3 "--with-sqlite-cflags='-DSQLITE_ENABLE_DBSTAT_VTAB=1'"
+```
+
+`Gemfile`
+```bash
+gem 'rails-sqlite-extras'
+```
+
+## Available queries
+
+### `total_size`
+
+```ruby
+RailsSqliteExtras.total_size
+```
+
+The total size of all tables and indices. It's a summary table, it has only 2 columns: `name` and `value`. Rows:
+
+- `cells`: The number of cells in the DB. Each value stored in the DB is represented as at least one cell. So, the number of cells correlates with the number of records in the DB.
+- `payload_size`: How much space the actual useful payload takes in the DB.
+- `unused_size`: How much space in the DB is reserved, not used yet, and can be used later to store more data. This is a surplus that occurs because SQLite allocates space for data in chunks ("pages").
+- `vacuum_size`: How much space is unused and cannot be used for future data. You can run [VACUUM](https://www.sqlite.org/lang_vacuum.html) command to reduce it.
+- `page_size`: The total space occupied by all pages. Each page is a single chunk of space allocated by SQLite. This number is the sum of `payload_size`, `unused_size`, and `vacuum_size`.
+- `pages`: The total number of pages.
+- `pages: leaf`: The pages that store the actual data. Read [SQLite Internals: Pages & B-trees](https://fly.io/blog/sqlite-internals-btree/) to learn more.
+- `pages: internal`: The pages that store ranges for leaf pages for a faster lookup. Sometimes also called "interior pages".
+- `pages: overflow`: The pages that store chunks of big data that don't fit in a single leaf page.
+- `pages: table`: The pages used for storing data for tables.
+- `pages: index`: The pages used for storing indices.
+
+### `table_size`
+
+```ruby
+RailsSqliteExtras.table_size
+```
+
+Information about the space used (and unused) by all tables. Based on the [dbstat](https://www.sqlite.org/dbstat.html) virtual table.
+
+- `name`: The table name.
+- `payload_size`.
+- `unused_size`.
+- `vacuum_size`.
+- `page_size`.
+- `cells`.
+- `pages`.
+- `max_payload_size`: The size of the biggest payload in the table.
+
+### `index_size`
+
+```ruby
+RailsSqliteExtras.index_size
+```
+
+Size of all indices.
+
+- `name`: The index name.
+- `table_name`: The table where the index is defined.
+- `column_name`: The name of the column being indexed. This column is NULL if the column is the rowid or an expression.
+- `payload_size`.
+- `unused_size`.
+- `page_size`.
+- `cells`.
+- `pages`.
+- `max_payload_size`.
+
+### `sequence_number`
+
+```ruby
+RailsSqliteExtras.sequence_number
+```
+
+Sequence numbers of autoincrement columns. Generated based on the [sqlite_sequence](https://renenyffenegger.ch/notes/development/databases/SQLite/internals/schema-objects/sqlite_sequence) table. The query will fail if there are no autoincrement columns in the DB yet.
+
+- `table_name`.
+- `sequence_number`.
+
+### `pragma`
+
+```ruby
+RailsSqliteExtras.pragma
+```
+
+List values of PRAGMAs (settings). Only includes the ones that have an integer or a boolean value. For brevity, the ones with the `0` (`false`) value are excluded from the output (based on the observation that this is the default value for most of the PRAGMAs). Check out the SQLite documentation to learn more about what each PRAGMA means: [PRAGMA Statements](https://www.sqlite.org/pragma.html).
+
+- `name`: the name of the PRAGMA as listed in the SQLite documentation.
+- `value`: the value of the PRAGMA. The `true` value is converted into `1` (and `false` is simply excluded).
+
+### `compile_options` 
+
+```ruby
+RailsSqliteExtras.compile_options
+```
+
+List the [compile-time options](https://www.sqlite.org/compile.html) used when building SQLite, one option per row. The "SQLITE_" prefix is omitted from the returned option names. See [exqlite docs](https://github.com/elixir-sqlite/exqlite#defining-extra-compile-flags) on how to change these options.
+
+### `integrity_check`
+
+```ruby
+RailsSqliteExtras.integrity_check
+```
+
+Run integrity checks on the database. Executes [PRAGMA integrity_check](https://www.sqlite.org/pragma.html#pragma_integrity_check) and returns the resulting messages.

data/lib/rails-sqlite-extras.rb

@@ -1,7 +1,95 @@
 # frozen_string_literal: true
 
 module RailsSqliteExtras
-  def self.hello
-    "there"
+  @@database_url = nil
+
+  QUERIES = %i(
+    compile_options
+    index_size
+    integrity_check
+    pragma
+    sequence_number
+    table_size
+    total_size
+  )
+
+  QUERIES.each do |query_name|
+    define_singleton_method query_name do |options = {}|
+      run_query(
+        query_name: query_name,
+        in_format: options.fetch(:in_format, :display_table),
+      )
+    end
+  end
+
+  def self.run_query(query_name:, in_format:)
+    sql = sql_for(query_name: query_name)
+
+    result = connection.execute(sql)
+
+    self.display_result(
+      result,
+      title: self.description_for(query_name: query_name),
+      in_format: in_format,
+    )
+  end
+
+  def self.display_result(result, title:, in_format:)
+    case in_format
+    when :array
+      result.values
+    when :hash
+      result.to_a
+    when :raw
+      result
+    when :display_table
+      headings = if result.count > 0
+          result[0].keys
+        else
+          ["No results"]
+        end
+
+      puts Terminal::Table.new(
+        title: title,
+        headings: headings,
+        rows: result.values,
+      )
+    else
+      raise "Invalid in_format option"
+    end
+  end
+
+  def self.sql_for(query_name:)
+    File.read(
+      sql_path_for(query_name: query_name)
+    )
+  end
+
+  def self.description_for(query_name:)
+    first_line = File.open(
+      sql_path_for(query_name: query_name)
+    ) { |f| f.readline }
+
+    first_line[/\/\*(.*?)\*\//m, 1].strip
+  end
+
+  def self.sql_path_for(query_name:)
+    File.join(File.dirname(__FILE__), "/rails_sqlite_extras/queries/#{query_name}.sql")
+  end
+
+  def self.connection
+    if (db_url = ENV["RAILS_SQLITE_EXTRAS_DATABASE_URL"])
+      ActiveRecord::Base.establish_connection(db_url).lease_connection
+    else
+      ActiveRecord::Base.connection
+    end
+  end
+
+  def self.database_url=(value)
+    @@database_url = value
+  end
+
+  def self.database_url
+    @@database_url || ENV["RUBY_PG_EXTRAS_DATABASE_URL"] || ENV.fetch("DATABASE_URL")
   end
 end

data/lib/rails_sqlite_extras/queries/compile_options.sql

@@ -0,0 +1,3 @@
+/* List the compile-time options used when building SQLite. */
+
+SELECT compile_options AS 'value' FROM pragma_compile_options;

data/lib/rails_sqlite_extras/queries/index_size.sql

@@ -0,0 +1,21 @@
+/* Metadata of the indexes, descending by size. */
+
+SELECT
+    d.name        AS name,
+    s.tbl_name    AS table_name,
+    i.name        AS column_name,
+    d.payload     AS payload_size,
+    d.unused      AS unused_size,
+    d.pgsize      AS page_size,
+    d.ncell       AS cells,
+    d.pageno      AS pages,
+    d.mx_payload  AS max_payload_size
+FROM
+    dbstat AS d,
+    sqlite_schema AS s,
+    pragma_index_info(d.name) AS i
+WHERE
+        d.name      = s.name
+    AND s.type      = 'index'
+    AND d.aggregate = TRUE
+ORDER BY page_size DESC;

data/lib/rails_sqlite_extras/queries/integrity_check.sql

@@ -0,0 +1,4 @@
+/* Run integrity checks on the database. */
+
+SELECT integrity_check as message
+FROM pragma_integrity_check;

data/lib/rails_sqlite_extras/queries/pragma.sql

@@ -0,0 +1,35 @@
+/* List values of PRAGMAs (settings). */
+
+SELECT *
+FROM (
+    SELECT 'analysis_limit' AS name, analysis_limit AS value FROM pragma_analysis_limit
+    UNION ALL SELECT 'application_id', *          FROM pragma_application_id
+    UNION ALL SELECT 'auto_vacuum', *             FROM pragma_auto_vacuum
+    UNION ALL SELECT 'automatic_index', *         FROM pragma_automatic_index
+    UNION ALL SELECT 'busy_timeout', *            FROM pragma_busy_timeout
+    UNION ALL SELECT 'cache_size', *              FROM pragma_cache_size
+    UNION ALL SELECT 'cache_spill', *             FROM pragma_cache_spill
+    UNION ALL SELECT 'cell_size_check', *         FROM pragma_cell_size_check
+    UNION ALL SELECT 'checkpoint_fullfsync', *    FROM pragma_checkpoint_fullfsync
+    UNION ALL SELECT 'data_version', *            FROM pragma_data_version
+    UNION ALL SELECT 'defer_foreign_keys', *      FROM pragma_defer_foreign_keys
+    UNION ALL SELECT 'freelist_count', *          FROM pragma_freelist_count
+    UNION ALL SELECT 'fullfsync', *               FROM pragma_fullfsync
+    UNION ALL SELECT 'hard_heap_limit', *         FROM pragma_hard_heap_limit
+    UNION ALL SELECT 'ignore_check_constraints', * FROM pragma_ignore_check_constraints
+    UNION ALL SELECT 'journal_size_limit', *      FROM pragma_journal_size_limit
+    UNION ALL SELECT 'legacy_alter_table', *      FROM pragma_legacy_alter_table
+    UNION ALL SELECT 'max_page_count', *          FROM pragma_max_page_count
+    UNION ALL SELECT 'page_size', *               FROM pragma_page_size
+    UNION ALL SELECT 'query_only', *              FROM pragma_query_only
+    UNION ALL SELECT 'read_uncommitted', *        FROM pragma_read_uncommitted
+    UNION ALL SELECT 'recursive_triggers', *      FROM pragma_recursive_triggers
+    UNION ALL SELECT 'reverse_unordered_selects', * FROM pragma_reverse_unordered_selects
+    UNION ALL SELECT 'soft_heap_limit', *         FROM pragma_soft_heap_limit
+    UNION ALL SELECT 'synchronous', *             FROM pragma_synchronous
+    UNION ALL SELECT 'temp_store', *              FROM pragma_temp_store
+    UNION ALL SELECT 'threads', *                 FROM pragma_threads
+    UNION ALL SELECT 'trusted_schema', *          FROM pragma_trusted_schema
+    UNION ALL SELECT 'user_version', *            FROM pragma_user_version
+)
+WHERE value != 0

data/lib/rails_sqlite_extras/queries/sequence_number.sql

@@ -0,0 +1,5 @@
+/* Sequence numbers of autoincrement columns. */
+
+SELECT name AS table_name, seq AS sequence_number
+FROM sqlite_sequence
+ORDER BY sequence_number DESC

data/lib/rails_sqlite_extras/queries/table_size.sql

@@ -0,0 +1,15 @@
+/* Metadata of the tables (excluding indexes), descending by size. */
+SELECT
+    name,
+    payload AS payload_size,
+    unused as unused_size,
+    pgsize - payload - unused as vacuum_size,
+    pgsize as page_size,
+    ncell as cells,
+    pageno as pages,
+    mx_payload as max_payload_size
+FROM dbstat
+WHERE
+    name IN (SELECT name FROM sqlite_schema WHERE type='table')
+    AND aggregate=TRUE
+ORDER BY page_size DESC;

data/lib/rails_sqlite_extras/queries/total_size.sql

@@ -0,0 +1,18 @@
+/* The total size of all tables and indexes. */
+SELECT 'pages' AS name,      MAX(pageno) as value FROM dbstat
+UNION ALL SELECT 'cells',    SUM(ncell)    FROM dbstat
+UNION ALL SELECT 'payload_size',  SUM(payload)  FROM dbstat
+UNION ALL SELECT 'unused_size',   SUM(unused)   FROM dbstat
+UNION ALL SELECT 'vacuum_size',   SUM(pgsize) - SUM(payload) - SUM(unused) FROM dbstat
+UNION ALL SELECT 'page_size',     SUM(pgsize)   FROM dbstat
+
+UNION ALL SELECT 'pages: leaf', COUNT(*)
+    FROM dbstat WHERE pagetype = 'leaf'
+UNION ALL SELECT 'pages: internal', COUNT(*)
+    FROM dbstat WHERE pagetype = 'internal'
+UNION ALL SELECT 'pages: overflow', COUNT(*)
+    FROM dbstat WHERE pagetype = 'overflow'
+UNION ALL SELECT 'pages: table', COUNT(*)
+    FROM dbstat WHERE name IN (SELECT name FROM sqlite_schema WHERE type='table')
+UNION ALL SELECT 'pages: index', COUNT(*)
+    FROM dbstat WHERE name IN (SELECT name FROM sqlite_schema WHERE type='index')

data/lib/rails_sqlite_extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsSqliteExtras
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/rails-sqlite-extras.gemspec

@@ -11,11 +11,12 @@ Gem::Specification.new do |s|
   s.summary = %q{ Rails Sqlite database insights. }
   s.description = %q{ Helper queries providing insights into the Sqlite database for Rails apps. }
   s.homepage = "http://github.com/pawurb/rails-sqlite-extras"
-  s.files = `git ls-files`.split("\n")
+  s.files = `git ls-files`.split("\n").reject { |f| f.match?(/\.db$/) }
   s.test_files = s.files.grep(%r{^(spec)/})
   s.require_paths = ["lib"]
   s.license = "MIT"
   s.add_dependency "rails"
+  s.add_dependency "sqlite3"
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "rufo"

data/spec/smoke_spec.rb

@@ -4,7 +4,24 @@ require "spec_helper"
 require "rails-sqlite-extras"
 
 describe RailsSqliteExtras do
-  it "works" do
-    expect(RailsSqliteExtras.hello).to eq("there")
+  RailsSqliteExtras::QUERIES.each do |query_name|
+    it "#{query_name} description can be read" do
+      expect do
+        RailsSqliteExtras.description_for(
+          query_name: query_name,
+        )
+      end.not_to raise_error
+    end
+  end
+
+  RailsSqliteExtras::QUERIES.reject { |q| q == :kill_all }.each do |query_name|
+    it "#{query_name} query can be executed" do
+      expect do
+        RailsSqliteExtras.run_query(
+          query_name: query_name,
+          in_format: :hash,
+        )
+      end.not_to raise_error
+    end
   end
 end

data/spec/spec_helper.rb

@@ -7,6 +7,7 @@ require_relative "../lib/rails-sqlite-extras"
 
 RSpec.configure do |config|
   config.before :suite do
+    ENV["RAILS_SQLITE_EXTRAS_DATABASE_URL"] = "sqlite3://#{Dir.pwd}/test.db"
   end
 
   config.after :suite do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails-sqlite-extras
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-14 00:00:00.000000000 Z
+date: 2024-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -81,6 +95,13 @@ files:
 - README.md
 - Rakefile
 - lib/rails-sqlite-extras.rb
+- lib/rails_sqlite_extras/queries/compile_options.sql
+- lib/rails_sqlite_extras/queries/index_size.sql
+- lib/rails_sqlite_extras/queries/integrity_check.sql
+- lib/rails_sqlite_extras/queries/pragma.sql
+- lib/rails_sqlite_extras/queries/sequence_number.sql
+- lib/rails_sqlite_extras/queries/table_size.sql
+- lib/rails_sqlite_extras/queries/total_size.sql
 - lib/rails_sqlite_extras/version.rb
 - rails-sqlite-extras.gemspec
 - spec/smoke_spec.rb