solid_log-core 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '09b479494d3ac70c18b6d0611a5ee0879ff01b90c17108dcffe1eaa951a47f00'
+  data.tar.gz: ca5f502ca1f8c7b722f1c26b31c898bf3c0e04b512fbde82d0b1e1e888006e32
+SHA512:
+  metadata.gz: 2da7bc4a35999dc410ef3951325e1c61c7e67ef4a7f5028d386bb397e676edc50528e2b46209a4999dc94f95cf6ff6b7025db7d7010683e2dd73b207dcac9333
+  data.tar.gz: dff9a33b078a75d0019bfba2d9371b9d4804ce6bcfb52ed6be75bc2ce9a3892ba57d679885fd64ac56c4904945d101078eedfbbb709114d662b0978b57218431

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright Dan Loman
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,176 @@
+# SolidLog::Core
+
+Core models, database adapters, parser, DirectLogger, and HTTP client for the SolidLog logging system.
+
+## Overview
+
+`solid_log-core` provides the shared foundation for `solid_log-service` and `solid_log-ui` gems:
+
+- **Models**: `RawEntry`, `Entry`, `Token`, `Field`, `FacetCache`
+- **Database Adapters**: SQLite, PostgreSQL, MySQL adapters with database-specific optimizations
+- **DirectLogger**: High-performance batched logging for parent app (50,000+ logs/sec)
+- **Parser**: Structured log parsing with field extraction
+- **Service Objects**: `RetentionService`, `FieldAnalyzer`, `SearchService`, `CorrelationService`
+- **HTTP Client**: Buffered log sender with retry logic for remote ingestion
+- **Migrations**: Database schema in `db/log_migrate/`
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'solid_log-core'
+
+# Also ensure you have the database adapter gem for your database:
+gem 'sqlite3', '>= 2.1'   # For SQLite
+# OR
+gem 'pg', '>= 1.1'        # For PostgreSQL
+# OR
+gem 'mysql2', '>= 0.5'    # For MySQL
+```
+
+**Dependencies:**
+- `solid_log-core` requires only specific Rails components: `activerecord`, `activesupport`, and `activejob`
+- It does **NOT** require the full `rails` gem
+- Database adapter gems are optional - install only what you need
+
+## Usage
+
+This gem is typically used as a dependency for:
+- **solid_log-service**: Standalone log ingestion service
+- **solid_log-ui**: Web interface for viewing logs
+
+### DirectLogger (Recommended for Parent App)
+
+**DirectLogger** writes logs directly to the database for maximum performance. Use this when your Rails app has direct database access.
+
+```ruby
+# config/initializers/lograge.rb
+Rails.application.configure do
+  config.lograge.enabled = true
+  config.lograge.formatter = Lograge::Formatters::Json.new
+
+  # Use DirectLogger for fast, batched database writes
+  config.lograge.logger = ActiveSupport::Logger.new(
+    SolidLog::DirectLogger.new(
+      batch_size: 100,                      # Batch size (default: 100)
+      flush_interval: 5,                    # Flush every 5 seconds
+      eager_flush_levels: [:error, :fatal]  # Flush immediately on errors (crash safety)
+    )
+  )
+end
+```
+
+**Performance:**
+- **16,882 logs/sec** with crash safety (SQLite + WAL mode)
+- **56,660 logs/sec** without eager flush (risky - may lose crash logs)
+- **9x faster** than individual database inserts
+- **67x faster** than HTTP logging
+
+**Configuration Options:**
+
+```ruby
+SolidLog::DirectLogger.new(
+  batch_size: 100,                      # Logs per batch (default: 100)
+  flush_interval: 5,                    # Flush interval in seconds (default: 5)
+  eager_flush_levels: [:error, :fatal], # Flush these levels immediately (default: [:error, :fatal])
+  token_id: nil                          # Optional token for audit trail (default: nil or ENV['SOLIDLOG_TOKEN_ID'])
+)
+```
+
+**Crash Safety:**
+
+By default, DirectLogger flushes error and fatal logs immediately to prevent losing the logs that explain WHY your app crashed:
+
+```ruby
+# Safe (recommended for production)
+logger = SolidLog::DirectLogger.new(
+  eager_flush_levels: [:error, :fatal]
+)
+
+# Maximum performance (risky - may lose crash context)
+logger = SolidLog::DirectLogger.new(
+  eager_flush_levels: []
+)
+
+# Maximum safety (slower)
+logger = SolidLog::DirectLogger.new(
+  batch_size: 10,
+  flush_interval: 1,
+  eager_flush_levels: [:debug, :info, :warn, :error, :fatal]
+)
+```
+
+**Environment Variables:**
+
+DirectLogger supports optional token configuration via environment:
+
+```bash
+export SOLIDLOG_TOKEN_ID=123  # Optional: for audit trail tracking
+```
+
+See [BENCHMARK_RESULTS.md](BENCHMARK_RESULTS.md) for detailed performance analysis.
+
+### HTTP Client (For External Services)
+
+For sending logs to a remote SolidLog service:
+
+```ruby
+SolidLog::Core.configure_client do |config|
+  config.service_url = ENV['SOLIDLOG_SERVICE_URL']
+  config.token = ENV['SOLIDLOG_TOKEN']
+  config.app_name = 'web'
+  config.environment = Rails.env
+end
+```
+
+## Database Setup
+
+**Option 1: Run migrations**
+
+```bash
+rails db:migrate
+```
+
+**Option 2: Use provided structure file**
+
+Choose the structure file for your database type:
+
+```bash
+# For SQLite
+rails db:schema:load SCHEMA=db/log_structure_sqlite.sql
+
+# For PostgreSQL
+rails db:schema:load SCHEMA=db/log_structure_postgresql.sql
+
+# For MySQL
+rails db:schema:load SCHEMA=db/log_structure_mysql.sql
+```
+
+The structure files are optimized for each database with appropriate FTS implementations:
+- **SQLite**: FTS5 virtual table
+- **PostgreSQL**: tsvector with GIN index
+- **MySQL**: FULLTEXT index
+
+### Enable WAL Mode for SQLite (Highly Recommended)
+
+WAL (Write-Ahead Logging) mode provides **243% better performance** for crash-safe logging:
+
+```ruby
+# config/initializers/solid_log.rb
+ActiveRecord::Base.connected_to(role: :log) do
+  ActiveRecord::Base.connection.execute("PRAGMA journal_mode=WAL")
+  ActiveRecord::Base.connection.execute("PRAGMA synchronous=NORMAL")
+end
+```
+
+**Benefits:**
+- **3.4x faster** eager flush (16,882 vs 4,923 logs/sec)
+- Better concurrency (readers don't block writers)
+- Recommended for all production deployments
+
+See [BENCHMARK_RESULTS.md](BENCHMARK_RESULTS.md) for performance comparison.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "lib"
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = false
+end
+
+task default: :test

data/db/log_migrate/20251222000001_create_solid_log_raw.rb

@@ -0,0 +1,15 @@
+class CreateSolidLogRaw < ActiveRecord::Migration[8.0]
+  def change
+    create_table :solid_log_raw do |t|
+      t.datetime :received_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+      t.integer :token_id  # Nullable - only required for HTTP API auth, not DirectLogger
+      t.text :payload, null: false
+      t.boolean :parsed, default: false, null: false
+      t.datetime :parsed_at
+    end
+
+    add_index :solid_log_raw, [:parsed, :received_at], name: "idx_raw_unparsed"
+    add_index :solid_log_raw, :token_id, name: "idx_raw_token"
+    add_index :solid_log_raw, :received_at, name: "idx_raw_received"
+  end
+end

data/db/log_migrate/20251222000002_create_solid_log_entries.rb

@@ -0,0 +1,29 @@
+class CreateSolidLogEntries < ActiveRecord::Migration[8.0]
+  def change
+    create_table :solid_log_entries do |t|
+      t.integer :raw_id, null: false
+      t.datetime :timestamp, null: false    # When the log event occurred
+      t.datetime :created_at, null: false   # When the entry was parsed/created
+      t.string :level, null: false
+      t.string :app
+      t.string :env
+      t.text :message
+      t.string :request_id
+      t.string :job_id
+      t.float :duration
+      t.integer :status_code
+      t.string :controller
+      t.string :action
+      t.string :path
+      t.string :method
+      t.text :extra_fields
+    end
+
+    add_index :solid_log_entries, :timestamp, order: { timestamp: :desc }, name: "idx_entries_timestamp"
+    add_index :solid_log_entries, :level, name: "idx_entries_level"
+    add_index :solid_log_entries, [:app, :env, :timestamp], order: { timestamp: :desc }, name: "idx_entries_app_env_time"
+    add_index :solid_log_entries, :request_id, name: "idx_entries_request"
+    add_index :solid_log_entries, :job_id, name: "idx_entries_job"
+    add_index :solid_log_entries, :raw_id, name: "idx_entries_raw"
+  end
+end

data/db/log_migrate/20251222000004_create_solid_log_fields.rb

@@ -0,0 +1,17 @@
+class CreateSolidLogFields < ActiveRecord::Migration[8.0]
+  def change
+    create_table :solid_log_fields do |t|
+      t.string :name, null: false
+      t.string :field_type, null: false
+      t.string :filter_type, default: "multiselect", null: false
+      t.integer :usage_count, default: 0, null: false
+      t.datetime :last_seen_at
+      t.boolean :promoted, default: false, null: false
+      t.timestamps
+    end
+
+    add_index :solid_log_fields, :name, unique: true, name: "idx_fields_name"
+    add_index :solid_log_fields, :promoted, name: "idx_fields_promoted"
+    add_index :solid_log_fields, :usage_count, name: "idx_fields_usage"
+  end
+end

data/db/log_migrate/20251222000005_create_solid_log_tokens.rb

@@ -0,0 +1,13 @@
+class CreateSolidLogTokens < ActiveRecord::Migration[8.0]
+  def change
+    create_table :solid_log_tokens do |t|
+      t.string :name, null: false
+      t.string :token_hash, null: false
+      t.datetime :last_used_at
+      t.timestamps
+    end
+
+    add_index :solid_log_tokens, :token_hash, unique: true, name: "idx_tokens_hash"
+    add_index :solid_log_tokens, :name, name: "idx_tokens_name"
+  end
+end

data/db/log_migrate/20251222000006_create_solid_log_facet_cache.rb

@@ -0,0 +1,13 @@
+class CreateSolidLogFacetCache < ActiveRecord::Migration[8.0]
+  def change
+    create_table :solid_log_facet_cache do |t|
+      t.string :key_name, null: false
+      t.text :cache_value, null: false
+      t.datetime :expires_at
+      t.timestamps
+    end
+
+    add_index :solid_log_facet_cache, :key_name, unique: true, name: "idx_facet_key_name"
+    add_index :solid_log_facet_cache, :expires_at, name: "idx_facet_expires"
+  end
+end

data/db/log_migrate/20251222000007_create_solid_log_fts_triggers.rb

@@ -0,0 +1,41 @@
+class CreateSolidLogFtsTriggers < ActiveRecord::Migration[8.0]
+  def up
+    # SQLite FTS5 implementation
+    execute <<~SQL
+      CREATE VIRTUAL TABLE solid_log_entries_fts USING fts5(
+        message,
+        extra_fields,
+        content='solid_log_entries',
+        content_rowid='id'
+      );
+    SQL
+
+    execute <<~SQL
+      CREATE TRIGGER solid_log_entries_fts_insert AFTER INSERT ON solid_log_entries BEGIN
+        INSERT INTO solid_log_entries_fts(rowid, message, extra_fields)
+        VALUES (new.id, new.message, new.extra_fields);
+      END;
+    SQL
+
+    execute <<~SQL
+      CREATE TRIGGER solid_log_entries_fts_update AFTER UPDATE ON solid_log_entries BEGIN
+        UPDATE solid_log_entries_fts
+        SET message = new.message, extra_fields = new.extra_fields
+        WHERE rowid = new.id;
+      END;
+    SQL
+
+    execute <<~SQL
+      CREATE TRIGGER solid_log_entries_fts_delete AFTER DELETE ON solid_log_entries BEGIN
+        DELETE FROM solid_log_entries_fts WHERE rowid = old.id;
+      END;
+    SQL
+  end
+
+  def down
+    execute "DROP TRIGGER IF EXISTS solid_log_entries_fts_delete"
+    execute "DROP TRIGGER IF EXISTS solid_log_entries_fts_update"
+    execute "DROP TRIGGER IF EXISTS solid_log_entries_fts_insert"
+    execute "DROP TABLE IF EXISTS solid_log_entries_fts"
+  end
+end

data/db/log_structure_mysql.sql

@@ -0,0 +1,96 @@
+-- SolidLog Database Structure for MySQL
+-- Generated from migrations in db/log_migrate/
+
+-- solid_log_raw: Fast ingestion table for raw log payloads
+CREATE TABLE solid_log_raw (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  received_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  token_id BIGINT NOT NULL,
+  payload TEXT NOT NULL,
+  parsed TINYINT(1) NOT NULL DEFAULT 0,
+  parsed_at DATETIME,
+  INDEX idx_raw_unparsed (parsed, received_at),
+  INDEX idx_raw_token (token_id),
+  INDEX idx_raw_received (received_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- solid_log_entries: Parsed and indexed log entries for querying
+CREATE TABLE solid_log_entries (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  raw_id BIGINT NOT NULL,
+  timestamp DATETIME NOT NULL,
+  created_at DATETIME NOT NULL,
+  level VARCHAR(255) NOT NULL,
+  app VARCHAR(255),
+  env VARCHAR(255),
+  message TEXT,
+  request_id VARCHAR(255),
+  job_id VARCHAR(255),
+  duration DOUBLE,
+  status_code INT,
+  controller VARCHAR(255),
+  action VARCHAR(255),
+  path VARCHAR(255),
+  method VARCHAR(255),
+  extra_fields TEXT,
+  INDEX idx_entries_timestamp (timestamp DESC),
+  INDEX idx_entries_level (level),
+  INDEX idx_entries_app_env_time (app, env, timestamp DESC),
+  INDEX idx_entries_request (request_id),
+  INDEX idx_entries_job (job_id),
+  INDEX idx_entries_raw (raw_id),
+  FULLTEXT INDEX idx_entries_fts (message, extra_fields)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- solid_log_fields: Field registry for tracking dynamic JSON fields
+CREATE TABLE solid_log_fields (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  name VARCHAR(255) NOT NULL,
+  field_type VARCHAR(255) NOT NULL,
+  filter_type VARCHAR(255) NOT NULL DEFAULT 'multiselect',
+  usage_count INT NOT NULL DEFAULT 0,
+  last_seen_at DATETIME,
+  promoted TINYINT(1) NOT NULL DEFAULT 0,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  UNIQUE INDEX idx_fields_name (name),
+  INDEX idx_fields_promoted (promoted),
+  INDEX idx_fields_usage (usage_count)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- solid_log_tokens: API tokens for log ingestion authentication
+CREATE TABLE solid_log_tokens (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  name VARCHAR(255) NOT NULL,
+  token_hash VARCHAR(255) NOT NULL,
+  last_used_at DATETIME,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  UNIQUE INDEX idx_tokens_hash (token_hash),
+  INDEX idx_tokens_name (name)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- solid_log_facet_cache: Cache for filter options to reduce DB load
+CREATE TABLE solid_log_facet_cache (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  key_name VARCHAR(255) NOT NULL,
+  cache_value TEXT NOT NULL,
+  expires_at DATETIME,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  UNIQUE INDEX idx_facet_key_name (key_name),
+  INDEX idx_facet_expires (expires_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Schema migrations
+CREATE TABLE IF NOT EXISTS schema_migrations (
+  version VARCHAR(255) PRIMARY KEY
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+INSERT INTO schema_migrations (version) VALUES
+  ('20251222000001'),
+  ('20251222000002'),
+  ('20251222000004'),
+  ('20251222000005'),
+  ('20251222000006'),
+  ('20251222000007');

data/db/log_structure_postgresql.sql

@@ -0,0 +1,118 @@
+-- SolidLog Database Structure for PostgreSQL
+-- Generated from migrations in db/log_migrate/
+
+-- solid_log_raw: Fast ingestion table for raw log payloads
+CREATE TABLE solid_log_raw (
+  id BIGSERIAL PRIMARY KEY,
+  received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  token_id BIGINT NOT NULL,
+  payload TEXT NOT NULL,
+  parsed BOOLEAN NOT NULL DEFAULT FALSE,
+  parsed_at TIMESTAMP
+);
+
+CREATE INDEX idx_raw_unparsed ON solid_log_raw(parsed, received_at);
+CREATE INDEX idx_raw_token ON solid_log_raw(token_id);
+CREATE INDEX idx_raw_received ON solid_log_raw(received_at);
+
+-- solid_log_entries: Parsed and indexed log entries for querying
+CREATE TABLE solid_log_entries (
+  id BIGSERIAL PRIMARY KEY,
+  raw_id BIGINT NOT NULL,
+  timestamp TIMESTAMP NOT NULL,
+  created_at TIMESTAMP NOT NULL,
+  level VARCHAR(255) NOT NULL,
+  app VARCHAR(255),
+  env VARCHAR(255),
+  message TEXT,
+  request_id VARCHAR(255),
+  job_id VARCHAR(255),
+  duration DOUBLE PRECISION,
+  status_code INTEGER,
+  controller VARCHAR(255),
+  action VARCHAR(255),
+  path VARCHAR(255),
+  method VARCHAR(255),
+  extra_fields TEXT,
+  fts_vector TSVECTOR
+);
+
+CREATE INDEX idx_entries_timestamp ON solid_log_entries(timestamp DESC);
+CREATE INDEX idx_entries_level ON solid_log_entries(level);
+CREATE INDEX idx_entries_app_env_time ON solid_log_entries(app, env, timestamp DESC);
+CREATE INDEX idx_entries_request ON solid_log_entries(request_id);
+CREATE INDEX idx_entries_job ON solid_log_entries(job_id);
+CREATE INDEX idx_entries_raw ON solid_log_entries(raw_id);
+
+-- Full-text search index using tsvector and GIN
+CREATE INDEX idx_entries_fts ON solid_log_entries USING GIN(fts_vector);
+
+-- Trigger to auto-update FTS vector
+CREATE OR REPLACE FUNCTION solid_log_entries_fts_trigger() RETURNS trigger AS $$
+BEGIN
+  NEW.fts_vector :=
+    setweight(to_tsvector('english', COALESCE(NEW.message, '')), 'A') ||
+    setweight(to_tsvector('english', COALESCE(NEW.extra_fields, '')), 'B');
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER solid_log_entries_fts_update
+  BEFORE INSERT OR UPDATE ON solid_log_entries
+  FOR EACH ROW EXECUTE FUNCTION solid_log_entries_fts_trigger();
+
+-- solid_log_fields: Field registry for tracking dynamic JSON fields
+CREATE TABLE solid_log_fields (
+  id BIGSERIAL PRIMARY KEY,
+  name VARCHAR(255) NOT NULL,
+  field_type VARCHAR(255) NOT NULL,
+  filter_type VARCHAR(255) NOT NULL DEFAULT 'multiselect',
+  usage_count INTEGER NOT NULL DEFAULT 0,
+  last_seen_at TIMESTAMP,
+  promoted BOOLEAN NOT NULL DEFAULT FALSE,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_fields_name ON solid_log_fields(name);
+CREATE INDEX idx_fields_promoted ON solid_log_fields(promoted);
+CREATE INDEX idx_fields_usage ON solid_log_fields(usage_count);
+
+-- solid_log_tokens: API tokens for log ingestion authentication
+CREATE TABLE solid_log_tokens (
+  id BIGSERIAL PRIMARY KEY,
+  name VARCHAR(255) NOT NULL,
+  token_hash VARCHAR(255) NOT NULL,
+  last_used_at TIMESTAMP,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_tokens_hash ON solid_log_tokens(token_hash);
+CREATE INDEX idx_tokens_name ON solid_log_tokens(name);
+
+-- solid_log_facet_cache: Cache for filter options to reduce DB load
+CREATE TABLE solid_log_facet_cache (
+  id BIGSERIAL PRIMARY KEY,
+  key_name VARCHAR(255) NOT NULL,
+  cache_value TEXT NOT NULL,
+  expires_at TIMESTAMP,
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_facet_key_name ON solid_log_facet_cache(key_name);
+CREATE INDEX idx_facet_expires ON solid_log_facet_cache(expires_at);
+
+-- Schema migrations
+CREATE TABLE IF NOT EXISTS schema_migrations (
+  version VARCHAR(255) PRIMARY KEY
+);
+
+INSERT INTO schema_migrations (version) VALUES
+  ('20251222000001'),
+  ('20251222000002'),
+  ('20251222000004'),
+  ('20251222000005'),
+  ('20251222000006'),
+  ('20251222000007');

data/db/log_structure_sqlite.sql

@@ -0,0 +1,123 @@
+-- SolidLog Database Structure for SQLite
+-- Generated from migrations in db/log_migrate/
+
+-- solid_log_raw: Fast ingestion table for raw log payloads
+CREATE TABLE solid_log_raw (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  received_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  token_id INTEGER NOT NULL,
+  payload TEXT NOT NULL,
+  parsed BOOLEAN NOT NULL DEFAULT 0,
+  parsed_at DATETIME
+);
+
+CREATE INDEX idx_raw_unparsed ON solid_log_raw(parsed, received_at);
+CREATE INDEX idx_raw_token ON solid_log_raw(token_id);
+CREATE INDEX idx_raw_received ON solid_log_raw(received_at);
+
+-- solid_log_entries: Parsed and indexed log entries for querying
+CREATE TABLE solid_log_entries (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  raw_id INTEGER NOT NULL,
+  timestamp DATETIME NOT NULL,
+  created_at DATETIME NOT NULL,
+  level VARCHAR(255) NOT NULL,
+  app VARCHAR(255),
+  env VARCHAR(255),
+  message TEXT,
+  request_id VARCHAR(255),
+  job_id VARCHAR(255),
+  duration REAL,
+  status_code INTEGER,
+  controller VARCHAR(255),
+  action VARCHAR(255),
+  path VARCHAR(255),
+  method VARCHAR(255),
+  extra_fields TEXT
+);
+
+CREATE INDEX idx_entries_timestamp ON solid_log_entries(timestamp DESC);
+CREATE INDEX idx_entries_level ON solid_log_entries(level);
+CREATE INDEX idx_entries_app_env_time ON solid_log_entries(app, env, timestamp DESC);
+CREATE INDEX idx_entries_request ON solid_log_entries(request_id);
+CREATE INDEX idx_entries_job ON solid_log_entries(job_id);
+CREATE INDEX idx_entries_raw ON solid_log_entries(raw_id);
+
+-- solid_log_fields: Field registry for tracking dynamic JSON fields
+CREATE TABLE solid_log_fields (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name VARCHAR(255) NOT NULL,
+  field_type VARCHAR(255) NOT NULL,
+  filter_type VARCHAR(255) NOT NULL DEFAULT 'multiselect',
+  usage_count INTEGER NOT NULL DEFAULT 0,
+  last_seen_at DATETIME,
+  promoted BOOLEAN NOT NULL DEFAULT 0,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_fields_name ON solid_log_fields(name);
+CREATE INDEX idx_fields_promoted ON solid_log_fields(promoted);
+CREATE INDEX idx_fields_usage ON solid_log_fields(usage_count);
+
+-- solid_log_tokens: API tokens for log ingestion authentication
+CREATE TABLE solid_log_tokens (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name VARCHAR(255) NOT NULL,
+  token_hash VARCHAR(255) NOT NULL,
+  last_used_at DATETIME,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_tokens_hash ON solid_log_tokens(token_hash);
+CREATE INDEX idx_tokens_name ON solid_log_tokens(name);
+
+-- solid_log_facet_cache: Cache for filter options to reduce DB load
+CREATE TABLE solid_log_facet_cache (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  key_name VARCHAR(255) NOT NULL,
+  cache_value TEXT NOT NULL,
+  expires_at DATETIME,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_facet_key_name ON solid_log_facet_cache(key_name);
+CREATE INDEX idx_facet_expires ON solid_log_facet_cache(expires_at);
+
+-- Full-Text Search (SQLite FTS5)
+CREATE VIRTUAL TABLE solid_log_entries_fts USING fts5(
+  message,
+  extra_fields,
+  content='solid_log_entries',
+  content_rowid='id'
+);
+
+CREATE TRIGGER solid_log_entries_fts_insert AFTER INSERT ON solid_log_entries BEGIN
+  INSERT INTO solid_log_entries_fts(rowid, message, extra_fields)
+  VALUES (new.id, new.message, new.extra_fields);
+END;
+
+CREATE TRIGGER solid_log_entries_fts_update AFTER UPDATE ON solid_log_entries BEGIN
+  UPDATE solid_log_entries_fts
+  SET message = new.message, extra_fields = new.extra_fields
+  WHERE rowid = new.id;
+END;
+
+CREATE TRIGGER solid_log_entries_fts_delete AFTER DELETE ON solid_log_entries BEGIN
+  DELETE FROM solid_log_entries_fts WHERE rowid = old.id;
+END;
+
+-- Schema migrations
+CREATE TABLE IF NOT EXISTS schema_migrations (
+  version VARCHAR(255) PRIMARY KEY
+);
+
+INSERT INTO schema_migrations (version) VALUES
+  ('20251222000001'),
+  ('20251222000002'),
+  ('20251222000004'),
+  ('20251222000005'),
+  ('20251222000006'),
+  ('20251222000007');