whodunit-chronicles 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +50 -0
- data/.rubocop.yml +3 -3
- data/.yardopts +7 -5
- data/CHANGELOG.md +125 -1
- data/CONTRIBUTING.md +186 -0
- data/README.md +34 -22
- data/docker/mysql/init.sql +33 -0
- data/docker/postgres/init.sql +40 -0
- data/docker-compose.yml +138 -0
- data/lib/whodunit/chronicles/adapter_loader.rb +69 -0
- data/lib/whodunit/chronicles/adapters/mysql.rb +261 -0
- data/lib/whodunit/chronicles/adapters/postgresql.rb +1 -1
- data/lib/whodunit/chronicles/change_event.rb +2 -2
- data/lib/whodunit/chronicles/composite_processor.rb +86 -0
- data/lib/whodunit/chronicles/configuration.rb +23 -12
- data/lib/whodunit/chronicles/connection.rb +88 -0
- data/lib/whodunit/chronicles/errors.rb +43 -0
- data/lib/whodunit/chronicles/persistence.rb +129 -0
- data/lib/whodunit/chronicles/processor.rb +127 -0
- data/lib/whodunit/chronicles/service.rb +26 -24
- data/lib/whodunit/chronicles/table.rb +120 -0
- data/lib/whodunit/chronicles/version.rb +1 -1
- data/lib/whodunit/chronicles.rb +13 -8
- data/whodunit-chronicles.gemspec +28 -10
- metadata +106 -10
- data/lib/whodunit/chronicles/audit_processor.rb +0 -270
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fa4438fb3140657b7b9d92db2988c0ca511c9fc0138ba42cfad061ab91dc0ac4
|
|
4
|
+
data.tar.gz: e3c5339e3ecce1b882be854bc906f9e1644bc812eb16302266d7b8ef68ba0f12
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '03590c69049ab8670411b6beb0e5bbb8d3fcf0989b163541fa606be5fc1c7b41f20c4e6cb5160632ed411b056664f16aa153499c947cd24c612912ff58504354'
|
|
7
|
+
data.tar.gz: 26d9cbc11e8efbf0aa716034b9ec46604d5bb78e066584f3e9674ef00732daf60fc7ba6a5506c3f9524b0ad216fcfb52e4418b2452451787b6c4498e84bab08a
|
data/.codeclimate.yml
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
version: "2"
|
|
2
|
+
checks:
|
|
3
|
+
argument-count:
|
|
4
|
+
config:
|
|
5
|
+
threshold: 4
|
|
6
|
+
complex-logic:
|
|
7
|
+
config:
|
|
8
|
+
threshold: 4
|
|
9
|
+
file-lines:
|
|
10
|
+
config:
|
|
11
|
+
threshold: 250
|
|
12
|
+
method-complexity:
|
|
13
|
+
config:
|
|
14
|
+
threshold: 5
|
|
15
|
+
method-count:
|
|
16
|
+
config:
|
|
17
|
+
threshold: 20
|
|
18
|
+
method-lines:
|
|
19
|
+
config:
|
|
20
|
+
threshold: 25
|
|
21
|
+
nested-control-flow:
|
|
22
|
+
config:
|
|
23
|
+
threshold: 4
|
|
24
|
+
return-statements:
|
|
25
|
+
config:
|
|
26
|
+
threshold: 4
|
|
27
|
+
similar-code:
|
|
28
|
+
config:
|
|
29
|
+
threshold: # language-specific defaults. an integer indicates the minimum number of lines within a block of similar code.
|
|
30
|
+
identical-code:
|
|
31
|
+
config:
|
|
32
|
+
threshold: # language-specific defaults. an integer indicates the minimum number of lines within a block of identical code.
|
|
33
|
+
plugins:
|
|
34
|
+
rubocop:
|
|
35
|
+
enabled: true
|
|
36
|
+
config:
|
|
37
|
+
file: .rubocop.yml
|
|
38
|
+
exclude_patterns:
|
|
39
|
+
- "config/"
|
|
40
|
+
- "db/"
|
|
41
|
+
- "dist/"
|
|
42
|
+
- "features/"
|
|
43
|
+
- "**/node_modules/"
|
|
44
|
+
- "script/"
|
|
45
|
+
- "**/spec/"
|
|
46
|
+
- "**/test/"
|
|
47
|
+
- "**/tests/"
|
|
48
|
+
- "**/vendor/"
|
|
49
|
+
- "**/*_test.rb"
|
|
50
|
+
- "**/*_spec.rb"
|
data/.rubocop.yml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Ruby version
|
|
2
2
|
AllCops:
|
|
3
|
-
TargetRubyVersion: 3.
|
|
3
|
+
TargetRubyVersion: 3.2
|
|
4
4
|
NewCops: enable
|
|
5
5
|
Exclude:
|
|
6
6
|
- 'vendor/**/*'
|
|
@@ -54,8 +54,8 @@ Style/FrozenStringLiteralComment:
|
|
|
54
54
|
Enabled: true
|
|
55
55
|
EnforcedStyle: always
|
|
56
56
|
|
|
57
|
-
Style/StringLiterals:
|
|
58
|
-
|
|
57
|
+
# Style/StringLiterals:
|
|
58
|
+
# EnforcedStyle: double_quotes
|
|
59
59
|
|
|
60
60
|
Style/HashSyntax:
|
|
61
61
|
EnforcedStyle: ruby19
|
data/.yardopts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
--markup markdown
|
|
2
2
|
--markup-provider kramdown
|
|
3
|
-
--output-dir docs
|
|
4
|
-
--exclude spec/
|
|
5
|
-
--exclude vendor/
|
|
6
3
|
--main README.md
|
|
7
|
-
--
|
|
8
|
-
--
|
|
4
|
+
--output-dir docs
|
|
5
|
+
--protected
|
|
6
|
+
--private
|
|
7
|
+
--title "Whodunit Chronicles API Documentation"
|
|
8
|
+
--readme README.md
|
|
9
|
+
--files CHANGELOG.md,LICENSE
|
|
9
10
|
lib/**/*.rb
|
|
10
11
|
-
|
|
11
12
|
README.md
|
|
12
13
|
CHANGELOG.md
|
|
14
|
+
LICENSE
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,130 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
|
|
9
|
+
## [0.3.0] - 2026-05-17
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **`CompositeProcessor`** — fans out a single change event to multiple processors
|
|
14
|
+
in sequence, enabling pipeline architectures (e.g. simultaneously storing audit
|
|
15
|
+
records, streaming to Grafana, and triggering alerts) without coupling processors
|
|
16
|
+
to each other. Fail-open by default: if one processor raises, the error is logged
|
|
17
|
+
and the remaining processors still execute. Set `fail_fast: true` to halt the
|
|
18
|
+
chain on the first error.
|
|
19
|
+
- **Typed error hierarchy** — base `Whodunit::Chronicles::Error` class with typed
|
|
20
|
+
subclasses: `ConfigurationError`, `AdapterLoadError`, `ConnectionError`,
|
|
21
|
+
`ProcessingError`, `PersistenceError`. Callers can now rescue the base class to
|
|
22
|
+
catch all gem errors, or rescue specific subclasses for targeted handling.
|
|
23
|
+
- **Lazy adapter loading** — `pg` and `trilogy` are now required only when the
|
|
24
|
+
matching adapter is actually used. A friendly `AdapterLoadError` with install
|
|
25
|
+
instructions is raised if the gem is missing, instead of a cryptic `LoadError`.
|
|
26
|
+
- **`bin/console`** — loads the gem with a sane test config and drops into Pry
|
|
27
|
+
for interactive development.
|
|
28
|
+
- **Docker Compose test environment** — spins up PostgreSQL 16 (`wal_level=logical`),
|
|
29
|
+
MySQL 8, and MariaDB 11 with matching audit databases, init SQL scripts, test
|
|
30
|
+
tables, publication setup, and replication role grants.
|
|
31
|
+
- Development environment setup guide added to docs.
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- Gemspec `homepage_uri` metadata corrected.
|
|
36
|
+
- `pg`, `trilogy`, and `bigdecimal` moved from runtime to `add_development_dependency`
|
|
37
|
+
— consuming apps are no longer forced to install database adapters they don't use.
|
|
38
|
+
- `rubocop-rake` and `rubocop-thread_safety` added to development dependencies.
|
|
39
|
+
|
|
40
|
+
### Chore
|
|
41
|
+
|
|
42
|
+
- `Gemfile.lock` removed from repo.
|
|
43
|
+
- `.gitignore` updated with recommended exclusions.
|
|
44
|
+
|
|
45
|
+
### Docs
|
|
46
|
+
|
|
47
|
+
- Broken links in README fixed.
|
|
48
|
+
- Development environment setup guide added.
|
|
49
|
+
|
|
50
|
+
### ⚠️ Breaking Changes
|
|
51
|
+
|
|
52
|
+
- `pg`, `trilogy`, and `bigdecimal` are no longer runtime dependencies. If your
|
|
53
|
+
application relies on them being pulled in transitively via `whodunit-chronicles`,
|
|
54
|
+
add them explicitly to your own Gemfile.
|
|
55
|
+
- remove ruby@3.1 from the supported versions of Ruby
|
|
56
|
+
|
|
57
|
+
## [0.2.0] - 2025-01-28
|
|
58
|
+
|
|
59
|
+
### Added
|
|
60
|
+
|
|
61
|
+
- **MySQL/MariaDB Support**: Complete multi-database adapter architecture
|
|
62
|
+
- MySQL adapter using trilogy gem for high-performance connections
|
|
63
|
+
- Binary log streaming support for MySQL change capture
|
|
64
|
+
- Cross-database compatibility testing
|
|
65
|
+
- **Enhanced Testing Suite**: Comprehensive test coverage improvements
|
|
66
|
+
- New test files: `table_test.rb`, `connection_test.rb`, `persistence_test.rb`
|
|
67
|
+
- Enhanced PostgreSQL adapter tests with connection and replication scenarios
|
|
68
|
+
- Increased line coverage from 91.28% to 97.29% (+6.01 percentage points)
|
|
69
|
+
- 227 tests with 552 assertions providing robust validation
|
|
70
|
+
- **Ruby 3.4+ Compatibility**: Forward compatibility improvements
|
|
71
|
+
- Added `bigdecimal` dependency for Ruby 3.4+ support
|
|
72
|
+
- Explicit dependency management for removed stdlib components
|
|
73
|
+
- **CI/CD Enhancements**: Improved automation and quality gates
|
|
74
|
+
- Matrix testing across PostgreSQL and MySQL databases
|
|
75
|
+
- Enhanced MySQL integration testing with proper connection handling
|
|
76
|
+
- Security scanning integration and automated dependency updates
|
|
77
|
+
|
|
78
|
+
### Changed
|
|
79
|
+
|
|
80
|
+
- **Architecture Refactoring**: Modular component extraction
|
|
81
|
+
- Extracted AuditProcessor into separate, focused components
|
|
82
|
+
- Improved service layer with multi-adapter support patterns
|
|
83
|
+
- Enhanced configuration system supporting both PostgreSQL and MySQL
|
|
84
|
+
- **Database Adapter Pattern**: Extensible multi-database support
|
|
85
|
+
- Abstract adapter base class for consistent interface
|
|
86
|
+
- Database-specific implementations with optimized performance
|
|
87
|
+
- Unified change event system across different database types
|
|
88
|
+
- **Test Infrastructure**: Comprehensive testing improvements
|
|
89
|
+
- Enhanced mock-based testing for complex database operations
|
|
90
|
+
- Improved test organization with better separation of concerns
|
|
91
|
+
- Integration test scenarios for real-world usage patterns
|
|
92
|
+
|
|
93
|
+
### Fixed
|
|
94
|
+
|
|
95
|
+
- **MySQL CI Integration**: Resolved connection and setup issues
|
|
96
|
+
- Fixed MySQL container configuration and health checks
|
|
97
|
+
- Improved database readiness detection and timeout handling
|
|
98
|
+
- Enhanced error reporting and debugging for CI environments
|
|
99
|
+
- **Dependency Management**: Ruby version compatibility
|
|
100
|
+
- Added explicit `bigdecimal ~> 3.1` dependency for Ruby 3.4+
|
|
101
|
+
- Resolved trilogy gem loading issues in newer Ruby versions
|
|
102
|
+
- Improved gem specification with proper version constraints
|
|
103
|
+
|
|
104
|
+
### Technical Improvements
|
|
105
|
+
|
|
106
|
+
- **Code Coverage**: Significant testing improvements
|
|
107
|
+
- Line coverage: 97.29% (647/665 lines covered)
|
|
108
|
+
- Branch coverage: 83.6% (158/189 branches covered)
|
|
109
|
+
- Comprehensive unit tests for all core modules
|
|
110
|
+
- **Performance Optimizations**: Multi-adapter efficiency
|
|
111
|
+
- Database-specific SQL generation and parameter binding
|
|
112
|
+
- Optimized connection management across different adapters
|
|
113
|
+
- Efficient batch processing for both PostgreSQL and MySQL
|
|
114
|
+
- **Error Handling**: Enhanced resilience and debugging
|
|
115
|
+
- Improved error messages and stack trace reporting
|
|
116
|
+
- Better handling of database-specific error conditions
|
|
117
|
+
- Enhanced logging for troubleshooting and monitoring
|
|
118
|
+
|
|
119
|
+
### Development Experience
|
|
120
|
+
|
|
121
|
+
- **Documentation**: Enhanced developer resources
|
|
122
|
+
- Updated README with MySQL/MariaDB configuration examples
|
|
123
|
+
- Improved inline documentation for multi-adapter usage
|
|
124
|
+
- Better error messages and troubleshooting guides
|
|
125
|
+
- **Testing Framework**: Improved development workflow
|
|
126
|
+
- Faster test execution with better mock strategies
|
|
127
|
+
- More reliable CI/CD pipeline with matrix testing
|
|
128
|
+
- Enhanced debugging capabilities for test failures
|
|
129
|
+
|
|
130
|
+
## [0.1.0] - 2025-01-21
|
|
131
|
+
|
|
8
132
|
### Added
|
|
9
133
|
|
|
10
134
|
- Comprehensive GitHub Actions CI/CD pipeline with multi-Ruby testing
|
|
@@ -30,7 +154,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
30
154
|
- **Core Architecture**: Complete zero-latency audit streaming implementation
|
|
31
155
|
- **PostgreSQL Adapter**: Logical replication streaming with WAL decoding
|
|
32
156
|
- **ChangeEvent System**: Unified change representation across database adapters
|
|
33
|
-
- **
|
|
157
|
+
- **Processor**: Intelligent transformation of changes into audit records
|
|
34
158
|
- **Configuration Management**: Comprehensive settings with validation using dry-configurable
|
|
35
159
|
- **Service Orchestration**: Thread-safe service with error handling and retry logic
|
|
36
160
|
- **Abstract Adapter Pattern**: Extensible design supporting multiple database systems
|
data/CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# Contributing to Whodunit Chronicles
|
|
2
|
+
|
|
3
|
+
Thank you for your interest in contributing! This document walks you through getting a working development environment set up, running the test suite, and submitting a pull request.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Table of Contents
|
|
8
|
+
|
|
9
|
+
1. [Prerequisites](#prerequisites)
|
|
10
|
+
2. [Setting Up Your Environment](#setting-up-your-environment)
|
|
11
|
+
3. [Database Setup](#database-setup)
|
|
12
|
+
4. [Running the Tests](#running-the-tests)
|
|
13
|
+
5. [Code Style](#code-style)
|
|
14
|
+
6. [Opening a Pull Request](#opening-a-pull-request)
|
|
15
|
+
7. [Contributing Custom Processors](#contributing-custom-processors)
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Prerequisites
|
|
20
|
+
|
|
21
|
+
| Tool | Minimum Version | Notes |
|
|
22
|
+
|------|----------------|-------|
|
|
23
|
+
| Ruby | 3.1.0 | `rbenv` or `asdf` recommended |
|
|
24
|
+
| Docker | 24+ | For running test databases |
|
|
25
|
+
| Docker Compose | v2 plugin | Bundled with Docker Desktop |
|
|
26
|
+
| Git | Any recent | |
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Setting Up Your Environment
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
git clone https://github.com/kanutocd/whodunit-chronicles.git
|
|
34
|
+
cd whodunit-chronicles
|
|
35
|
+
|
|
36
|
+
# Install dependencies
|
|
37
|
+
bundle install
|
|
38
|
+
|
|
39
|
+
# Verify the console works
|
|
40
|
+
bundle exec bin/console
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Database Setup
|
|
46
|
+
|
|
47
|
+
Chronicles streams directly from PostgreSQL logical replication and MySQL/MariaDB binary logs. The test suite needs real database processes — no SQLite or in-memory substitute.
|
|
48
|
+
|
|
49
|
+
**Start all test databases with Docker Compose:**
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
docker compose up -d
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
This starts:
|
|
56
|
+
|
|
57
|
+
| Service | Port | Purpose |
|
|
58
|
+
|---------|------|---------|
|
|
59
|
+
| PostgreSQL 16 (wal_level=logical) | 5432 | Source DB for pg tests |
|
|
60
|
+
| PostgreSQL 16 (audit store) | 5433 | Audit DB for pg tests |
|
|
61
|
+
| MySQL 8.0 (binlog ROW format) | 3306 | Source DB for MySQL tests |
|
|
62
|
+
| MySQL 8.0 (audit store) | 3307 | Audit DB for MySQL tests |
|
|
63
|
+
| MariaDB 11 (binlog ROW format) | 3308 | Source DB for MariaDB tests |
|
|
64
|
+
|
|
65
|
+
Wait for all services to be healthy before running tests:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
docker compose ps # all should show "(healthy)"
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### PostgreSQL — enabling logical replication
|
|
72
|
+
|
|
73
|
+
The Docker image is pre-configured (`wal_level=logical`). If you're using a system Postgres instead, add to `postgresql.conf`:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
wal_level = logical
|
|
77
|
+
max_replication_slots = 10
|
|
78
|
+
max_wal_senders = 10
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Then restart Postgres and grant the replication role to your user:
|
|
82
|
+
|
|
83
|
+
```sql
|
|
84
|
+
ALTER ROLE your_user WITH REPLICATION;
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### MySQL — enabling binary logging
|
|
88
|
+
|
|
89
|
+
The Docker image launches with `--binlog-format=ROW --binlog-row-image=FULL`. If using a system MySQL, add to `my.cnf`:
|
|
90
|
+
|
|
91
|
+
```ini
|
|
92
|
+
[mysqld]
|
|
93
|
+
server-id = 1
|
|
94
|
+
log-bin = mysql-bin
|
|
95
|
+
binlog-format = ROW
|
|
96
|
+
binlog-row-image = FULL
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Running the Tests
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
# Full suite
|
|
105
|
+
bundle exec rake test
|
|
106
|
+
|
|
107
|
+
# Single file
|
|
108
|
+
bundle exec ruby test/whodunit/chronicles/composite_processor_test.rb
|
|
109
|
+
|
|
110
|
+
# With coverage report
|
|
111
|
+
bundle exec rake test
|
|
112
|
+
open coverage/index.html
|
|
113
|
+
|
|
114
|
+
# Code style
|
|
115
|
+
bundle exec rubocop
|
|
116
|
+
|
|
117
|
+
# Security scan
|
|
118
|
+
bundle exec bundler-audit check --update
|
|
119
|
+
bundle exec brakeman
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Environment variables for test databases
|
|
123
|
+
|
|
124
|
+
The test suite reads these environment variables (defaults match the Docker Compose setup):
|
|
125
|
+
|
|
126
|
+
| Variable | Default |
|
|
127
|
+
|----------|---------|
|
|
128
|
+
| `CHRONICLES_PG_URL` | `postgresql://chronicles:chronicles@localhost/chronicles_test` |
|
|
129
|
+
| `CHRONICLES_PG_AUDIT_URL` | `postgresql://chronicles:chronicles@localhost:5433/chronicles_audit_test` |
|
|
130
|
+
| `CHRONICLES_MYSQL_URL` | `mysql://chronicles:chronicles@localhost:3306/chronicles_test` |
|
|
131
|
+
| `CHRONICLES_MYSQL_AUDIT_URL` | `mysql://chronicles:chronicles@localhost:3307/chronicles_audit_test` |
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Code Style
|
|
136
|
+
|
|
137
|
+
- All files must have `# frozen_string_literal: true`
|
|
138
|
+
- Public methods require YARD documentation
|
|
139
|
+
- Follow the existing RuboCop configuration (`.rubocop.yml`)
|
|
140
|
+
- Tests live in `test/` and use Minitest
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## Opening a Pull Request
|
|
145
|
+
|
|
146
|
+
1. Fork the repository and create a feature branch:
|
|
147
|
+
```bash
|
|
148
|
+
git checkout -b feature/my-improvement
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
2. Make your changes with tests. Coverage should not drop below 90%.
|
|
152
|
+
|
|
153
|
+
3. Run the full quality check before pushing:
|
|
154
|
+
```bash
|
|
155
|
+
bundle exec rake test
|
|
156
|
+
bundle exec rubocop
|
|
157
|
+
bundle exec bundler-audit check --update
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
4. Push and open a PR with:
|
|
161
|
+
- A clear description of what changed and why
|
|
162
|
+
- Any migration steps if you changed configuration
|
|
163
|
+
- A note on which databases you tested against
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## Contributing Custom Processors
|
|
168
|
+
|
|
169
|
+
The best contributions are custom processors for real-world domains. Here are domains we'd love to see:
|
|
170
|
+
|
|
171
|
+
- **E-commerce** — order tracking, inventory changes, cart events
|
|
172
|
+
- **Financial services** — transaction monitoring, compliance audit trails
|
|
173
|
+
- **Healthcare** — patient record changes, HIPAA-relevant audit events
|
|
174
|
+
- **SaaS** — feature flag changes, subscription events, seat management
|
|
175
|
+
- **Education** — student progress, grade changes, enrollment events
|
|
176
|
+
|
|
177
|
+
A good processor contribution includes:
|
|
178
|
+
|
|
179
|
+
1. The processor class in `lib/whodunit/chronicles/processors/`
|
|
180
|
+
2. Tests in `test/whodunit/chronicles/processors/`
|
|
181
|
+
3. A usage example in `examples/`
|
|
182
|
+
4. An entry in `CHANGELOG.md`
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
Questions? Open an issue or start a discussion on GitHub.
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# 📜 Whodunit Chronicles
|
|
2
2
|
|
|
3
3
|
[](https://badge.fury.io/rb/whodunit-chronicles)
|
|
4
4
|
[](https://github.com/kanutocd/whodunit-chronicles/actions)
|
|
@@ -14,9 +14,9 @@ While [Whodunit](https://github.com/kanutocd/whodunit) tracks _who_ made changes
|
|
|
14
14
|
|
|
15
15
|
## ✨ Features
|
|
16
16
|
|
|
17
|
-
- **🚄 Zero-Latency Streaming**: PostgreSQL logical replication
|
|
17
|
+
- **🚄 Zero-Latency Streaming**: PostgreSQL logical replication + MySQL/MariaDB binary log streaming
|
|
18
18
|
- **🔄 Zero Application Overhead**: No Rails callbacks or Active Record hooks required
|
|
19
|
-
- **🏗️ Database Agnostic**: Abstract adapter pattern supports PostgreSQL
|
|
19
|
+
- **🏗️ Database Agnostic**: Abstract adapter pattern supports PostgreSQL and MySQL/MariaDB
|
|
20
20
|
- **⚡ Thread-Safe**: Concurrent processing with configurable thread pools
|
|
21
21
|
- **🛡️ Resilient**: Built-in error handling, retry logic, and monitoring
|
|
22
22
|
- **📊 Complete Audit Trail**: Captures INSERT, UPDATE, DELETE with full before/after data
|
|
@@ -36,7 +36,7 @@ Perfect for applications that need comprehensive change tracking alongside Whodu
|
|
|
36
36
|
|
|
37
37
|
```ruby
|
|
38
38
|
# Basic setup for user activity tracking
|
|
39
|
-
class
|
|
39
|
+
class BasicProcessor < Whodunit::Chronicles::Processor
|
|
40
40
|
def build_chronicles_record(change_event)
|
|
41
41
|
super.tap do |record|
|
|
42
42
|
# Add basic business context
|
|
@@ -66,7 +66,7 @@ Sophisticated business intelligence for talent acquisition platforms:
|
|
|
66
66
|
|
|
67
67
|
```ruby
|
|
68
68
|
# Advanced processor for recruitment metrics
|
|
69
|
-
class RecruitmentAnalyticsProcessor < Whodunit::Chronicles::
|
|
69
|
+
class RecruitmentAnalyticsProcessor < Whodunit::Chronicles::Processor
|
|
70
70
|
def build_chronicles_record(change_event)
|
|
71
71
|
super.tap do |record|
|
|
72
72
|
# Add recruitment-specific business metrics
|
|
@@ -151,19 +151,19 @@ The recruitment analytics processor creates comprehensive Grafana dashboards for
|
|
|
151
151
|
<a href="examples/images/campaign-performance-analytics.png" title="Click to view full size image">
|
|
152
152
|
<img src="examples/images/campaign-performance-analytics.png" width="300" />
|
|
153
153
|
</a>
|
|
154
|
-
|
|
154
|
+
_Track campaign ROI, cost-per-hire by channel, and conversion rates across marketing sources_
|
|
155
155
|
|
|
156
|
-
**Candidate Journey Analytics**
|
|
156
|
+
**Candidate Journey Analytics**
|
|
157
157
|
<a href="examples/images/candidate-journey-analytics.png" title="Click to view full size image">
|
|
158
158
|
<img src="examples/images/candidate-journey-analytics.png" width="300" />
|
|
159
159
|
</a>
|
|
160
|
-
|
|
160
|
+
_Monitor candidate engagement, funnel conversion rates, and application completion patterns_
|
|
161
161
|
|
|
162
162
|
**Recruitment Funnel Analytics**
|
|
163
163
|
<a href="examples/images/recruitment-funnel-analytics.png" title="Click to view full size image">
|
|
164
164
|
<img src="examples/images/recruitment-funnel-analytics.png" width="300" />
|
|
165
165
|
</a>
|
|
166
|
-
|
|
166
|
+
_Analyze hiring pipeline progression, department performance, and time-series trends_
|
|
167
167
|
|
|
168
168
|
</div>
|
|
169
169
|
|
|
@@ -188,7 +188,9 @@ gem install whodunit-chronicles
|
|
|
188
188
|
```ruby
|
|
189
189
|
require 'whodunit/chronicles'
|
|
190
190
|
|
|
191
|
-
#
|
|
191
|
+
# Database Configuration
|
|
192
|
+
|
|
193
|
+
## PostgreSQL Configuration
|
|
192
194
|
Whodunit::Chronicles.configure do |config|
|
|
193
195
|
config.adapter = :postgresql
|
|
194
196
|
config.database_url = 'postgresql://localhost/myapp_production'
|
|
@@ -197,6 +199,14 @@ Whodunit::Chronicles.configure do |config|
|
|
|
197
199
|
config.replication_slot_name = 'myapp_chronicles_slot'
|
|
198
200
|
end
|
|
199
201
|
|
|
202
|
+
## MySQL/MariaDB Configuration
|
|
203
|
+
Whodunit::Chronicles.configure do |config|
|
|
204
|
+
config.adapter = :mysql
|
|
205
|
+
config.database_url = 'mysql://user:password@localhost/myapp_production'
|
|
206
|
+
config.audit_database_url = 'mysql://user:password@localhost/myapp_audit'
|
|
207
|
+
config.mysql_server_id = 1001 # Unique server ID for replication
|
|
208
|
+
end
|
|
209
|
+
|
|
200
210
|
# Create and start the service
|
|
201
211
|
service = Whodunit::Chronicles.service
|
|
202
212
|
service.setup! # Create publication/replication setup
|
|
@@ -212,7 +222,7 @@ service.teardown! # Clean up database objects
|
|
|
212
222
|
|
|
213
223
|
## 🏗️ Architecture
|
|
214
224
|
|
|
215
|
-
Chronicles uses **PostgreSQL logical replication**
|
|
225
|
+
Chronicles uses **PostgreSQL logical replication** and **MySQL/MariaDB binary log streaming** to capture database changes without impacting your application:
|
|
216
226
|
|
|
217
227
|
```
|
|
218
228
|
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
|
|
@@ -233,9 +243,9 @@ Chronicles uses **PostgreSQL logical replication** (TODO: **MySQL/MariaDB binary
|
|
|
233
243
|
|
|
234
244
|
### Core Components
|
|
235
245
|
|
|
236
|
-
- **StreamAdapter**: Database-specific change streaming (PostgreSQL, MySQL/MariaDB)
|
|
246
|
+
- **StreamAdapter**: Database-specific change streaming (PostgreSQL logical replication, MySQL/MariaDB binary log streaming)
|
|
237
247
|
- **ChangeEvent**: Unified change representation across adapters
|
|
238
|
-
- **
|
|
248
|
+
- **Processor**: Transforms changes into searchable audit records
|
|
239
249
|
- **Service**: Orchestrates streaming with error handling and retry logic
|
|
240
250
|
|
|
241
251
|
## ⚙️ Configuration
|
|
@@ -311,7 +321,7 @@ Transform database changes into actionable business intelligence with features l
|
|
|
311
321
|
#### Analytics-Focused Processor
|
|
312
322
|
|
|
313
323
|
```ruby
|
|
314
|
-
class AnalyticsProcessor < Whodunit::Chronicles::
|
|
324
|
+
class AnalyticsProcessor < Whodunit::Chronicles::Processor
|
|
315
325
|
def build_chronicles_record(change_event)
|
|
316
326
|
super.tap do |record|
|
|
317
327
|
# Add business metrics
|
|
@@ -365,7 +375,7 @@ end
|
|
|
365
375
|
#### Grafana Dashboard Ready
|
|
366
376
|
|
|
367
377
|
```ruby
|
|
368
|
-
class GrafanaProcessor < Whodunit::Chronicles::
|
|
378
|
+
class GrafanaProcessor < Whodunit::Chronicles::Processor
|
|
369
379
|
def build_chronicles_record(change_event)
|
|
370
380
|
{
|
|
371
381
|
# Core metrics for Grafana time series
|
|
@@ -399,7 +409,7 @@ end
|
|
|
399
409
|
#### Real-Time Alerts Processor
|
|
400
410
|
|
|
401
411
|
```ruby
|
|
402
|
-
class AlertingProcessor < Whodunit::Chronicles::
|
|
412
|
+
class AlertingProcessor < Whodunit::Chronicles::Processor
|
|
403
413
|
def process(change_event)
|
|
404
414
|
record = build_chronicles_record(change_event)
|
|
405
415
|
|
|
@@ -436,7 +446,7 @@ end
|
|
|
436
446
|
# Chain multiple processors for different purposes
|
|
437
447
|
service = Whodunit::Chronicles::Service.new(
|
|
438
448
|
adapter: Adapters::PostgreSQL.new,
|
|
439
|
-
processor: CompositeProcessor.new([
|
|
449
|
+
processor: Whodunit::Chronicles::CompositeProcessor.new([
|
|
440
450
|
AnalyticsProcessor.new, # For business intelligence
|
|
441
451
|
AlertingProcessor.new, # For real-time monitoring
|
|
442
452
|
ComplianceProcessor.new, # For regulatory requirements
|
|
@@ -615,13 +625,14 @@ We especially welcome custom processors for different business domains. Consider
|
|
|
615
625
|
|
|
616
626
|
- **Ruby**: 3.1.0 or higher
|
|
617
627
|
- **PostgreSQL**: 10.0 or higher (with logical replication enabled)
|
|
628
|
+
- **MySQL/MariaDB**: 5.6+ (with binary logging enabled)
|
|
618
629
|
|
|
619
630
|
## 🗺️ Roadmap
|
|
620
631
|
|
|
621
632
|
- [ ] **Prometheus Metrics**: Production monitoring integration (with complete codebase included in examples/)
|
|
622
633
|
- [ ] **Advanced Example Apps**: Real-world use cases with complete monitoring stack (with complete codebase included in examples/)
|
|
623
634
|
- [ ] **Custom Analytics Processors**: Business intelligence and real-time monitoring (with complete codebase included in examples/)
|
|
624
|
-
- [
|
|
635
|
+
- [x] **MySQL/MariaDB Support**: MySQL/MariaDB databases binlog streaming adapter
|
|
625
636
|
- [ ] **Redis Streams**: Alternative lightweight streaming backend
|
|
626
637
|
- [ ] **Compression**: Optional audit record compression
|
|
627
638
|
- [ ] **Retention Policies**: Automated audit record cleanup
|
|
@@ -630,10 +641,11 @@ We especially welcome custom processors for different business domains. Consider
|
|
|
630
641
|
## 📚 Documentation
|
|
631
642
|
|
|
632
643
|
- **[API Documentation](https://kanutocd.github.io/whodunit-chronicles/)**
|
|
633
|
-
- **
|
|
634
|
-
- **
|
|
635
|
-
- **
|
|
636
|
-
-
|
|
644
|
+
- [ ] TODO: **Configuration Guide** _(docs/configuration-todo.md)_
|
|
645
|
+
- [ ] TODO: **Architecture Deep Dive** _(docs/architecture-todo.md)_
|
|
646
|
+
- [ ] TODO: **PostgreSQL Setup** _(docs/postgresql-setup-todo.md)_
|
|
647
|
+
- [ ] TODO: **MySQL/MariaDB Setup** _(docs/mysql-setup.md)_
|
|
648
|
+
- [ ] TODO: **Production Deployment** _(docs/production-todo.md)_
|
|
637
649
|
|
|
638
650
|
## 📄 License
|
|
639
651
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
-- docker/mysql/init.sql
|
|
2
|
+
-- Run once when the MySQL/MariaDB container is first created.
|
|
3
|
+
|
|
4
|
+
USE chronicles_test;
|
|
5
|
+
|
|
6
|
+
CREATE TABLE IF NOT EXISTS users (
|
|
7
|
+
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
8
|
+
name VARCHAR(255) NOT NULL,
|
|
9
|
+
email VARCHAR(255) NOT NULL UNIQUE,
|
|
10
|
+
creator_id INT DEFAULT NULL,
|
|
11
|
+
updater_id INT DEFAULT NULL,
|
|
12
|
+
deleter_id INT DEFAULT NULL,
|
|
13
|
+
created_at DATETIME(6) NOT NULL DEFAULT NOW(6),
|
|
14
|
+
updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6)
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
CREATE TABLE IF NOT EXISTS posts (
|
|
18
|
+
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
19
|
+
user_id INT NOT NULL,
|
|
20
|
+
title VARCHAR(500) NOT NULL,
|
|
21
|
+
body TEXT,
|
|
22
|
+
status VARCHAR(50) DEFAULT 'draft',
|
|
23
|
+
creator_id INT DEFAULT NULL,
|
|
24
|
+
updater_id INT DEFAULT NULL,
|
|
25
|
+
deleter_id INT DEFAULT NULL,
|
|
26
|
+
created_at DATETIME(6) NOT NULL DEFAULT NOW(6),
|
|
27
|
+
updated_at DATETIME(6) NOT NULL DEFAULT NOW(6) ON UPDATE NOW(6),
|
|
28
|
+
FOREIGN KEY (user_id) REFERENCES users (id)
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
-- Grant replication privileges to the chronicles user
|
|
32
|
+
GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'chronicles'@'%';
|
|
33
|
+
FLUSH PRIVILEGES;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
-- docker/postgres/init.sql
|
|
2
|
+
-- Run once when the container is first created.
|
|
3
|
+
-- Sets up the replication role and test tables used by the test suite.
|
|
4
|
+
|
|
5
|
+
-- Allow the chronicles user to create replication slots and use logical replication
|
|
6
|
+
ALTER ROLE chronicles WITH REPLICATION;
|
|
7
|
+
|
|
8
|
+
-- Grant superuser for test convenience (never do this in production)
|
|
9
|
+
-- Remove this line and grant only necessary permissions for a tighter setup.
|
|
10
|
+
ALTER ROLE chronicles WITH SUPERUSER;
|
|
11
|
+
|
|
12
|
+
-- Create the test tables Chronicles will stream from
|
|
13
|
+
\c chronicles_test
|
|
14
|
+
|
|
15
|
+
CREATE TABLE IF NOT EXISTS users (
|
|
16
|
+
id SERIAL PRIMARY KEY,
|
|
17
|
+
name VARCHAR(255) NOT NULL,
|
|
18
|
+
email VARCHAR(255) NOT NULL UNIQUE,
|
|
19
|
+
creator_id INTEGER,
|
|
20
|
+
updater_id INTEGER,
|
|
21
|
+
deleter_id INTEGER,
|
|
22
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
23
|
+
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
CREATE TABLE IF NOT EXISTS posts (
|
|
27
|
+
id SERIAL PRIMARY KEY,
|
|
28
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
29
|
+
title VARCHAR(500) NOT NULL,
|
|
30
|
+
body TEXT,
|
|
31
|
+
status VARCHAR(50) DEFAULT 'draft',
|
|
32
|
+
creator_id INTEGER,
|
|
33
|
+
updater_id INTEGER,
|
|
34
|
+
deleter_id INTEGER,
|
|
35
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
36
|
+
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
-- Publication used by the Chronicles PostgreSQL adapter
|
|
40
|
+
CREATE PUBLICATION chronicles_test_pub FOR TABLE users, posts;
|