schema_ferry 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/LICENSE.txt +21 -0
- data/README.md +202 -0
- data/exe/schema_ferry +7 -0
- data/lib/schema_ferry/cli.rb +98 -0
- data/lib/schema_ferry/converter/column_converter.rb +74 -0
- data/lib/schema_ferry/converter/enum_check_builder.rb +48 -0
- data/lib/schema_ferry/converter/identifier_shortener.rb +37 -0
- data/lib/schema_ferry/converter/schema_converter.rb +144 -0
- data/lib/schema_ferry/converter/type_mapper.rb +67 -0
- data/lib/schema_ferry/dsl/config.rb +77 -0
- data/lib/schema_ferry/dsl/table_rule.rb +51 -0
- data/lib/schema_ferry/errors.rb +11 -0
- data/lib/schema_ferry/pipeline.rb +23 -0
- data/lib/schema_ferry/schema_model.rb +58 -0
- data/lib/schema_ferry/source/connection_registry.rb +29 -0
- data/lib/schema_ferry/source/mysql_reader.rb +89 -0
- data/lib/schema_ferry/target/ridgepole_runner.rb +44 -0
- data/lib/schema_ferry/target/schemafile_renderer.rb +138 -0
- data/lib/schema_ferry/version.rb +5 -0
- data/lib/schema_ferry/warnings.rb +13 -0
- data/lib/schema_ferry.rb +26 -0
- metadata +120 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: aacec6bc39b272cc96454401749eea5b880bb7c3a3d08e4666927f00f92d0379
|
|
4
|
+
data.tar.gz: eb407968fdbb8b848a1c4f942fb824cef3a80a063a1effb81bf02389342a69ce
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 2e4663eb185b31ab7724e574c01c9b23fb49b8844a3e20bcebcba2ebc467d1abe8b8e7e7272fa73b0cf122f4a74cc6433811b4598a328b8847edad41b73963ae
|
|
7
|
+
data.tar.gz: db642c4079f769cc430a79bbe7174a4dd1b13b89000bfcdc0ae9b4ce3e47c29e57db2c756820c543ab06531b3ce6a430f90861d8ed70a46c91bc7bc2f1d82b82
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 kyuuri1791
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# schema_ferry
|
|
2
|
+
|
|
3
|
+
You're migrating a production MySQL database to PostgreSQL. Moving the data takes days or weeks — and meanwhile, developers keep shipping schema changes to MySQL. schema_ferry is a Ruby gem that keeps the PostgreSQL schema continuously in sync until cutover, driven by a declarative DSL.
|
|
4
|
+
|
|
5
|
+
- **Incremental by design** — if the source schema changes mid-migration, just run it again; no manual diffing needed
|
|
6
|
+
- **Sensible defaults, fully customizable** — built-in type mappings handle most cases; override anything with a few DSL rules
|
|
7
|
+
- **Safe to iterate** — `dry_run` shows the exact changes that would be applied, before touching anything
|
|
8
|
+
|
|
9
|
+
schema_ferry is designed to run repeatedly (e.g. via cron). Data migration is out of scope — pair it with [pgloader](https://pgloader.io/) (one-shot bulk copy) or CDC replication (AWS DMS, [Debezium](https://debezium.io/), …), which load rows into the tables schema_ferry has created.
|
|
10
|
+
|
|
11
|
+
## Requirements
|
|
12
|
+
|
|
13
|
+
- Ruby >= 3.1
|
|
14
|
+
- ActiveRecord >= 7.1
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
Add to your Gemfile:
|
|
19
|
+
|
|
20
|
+
```ruby
|
|
21
|
+
gem "schema_ferry"
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
bundle install
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
### Basic
|
|
31
|
+
|
|
32
|
+
```ruby
|
|
33
|
+
require "schema_ferry"
|
|
34
|
+
|
|
35
|
+
pipeline = SchemaFerry.define do
|
|
36
|
+
source "mysql2://user:password@host:3306/source_db"
|
|
37
|
+
target "postgresql://user:password@host:5432/target_db"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
pipeline.dry_run # returns the changes that would be applied, without applying them
|
|
41
|
+
pipeline.apply! # applies the schema to PostgreSQL
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
There is also `pipeline.schemafile`, which returns the generated schema as a string without connecting to the target.
|
|
45
|
+
|
|
46
|
+
`apply!` makes the target match the generated schema — including **dropping** columns and indexes that are not part of it. Before running against a target that holds data, read [Destructive changes](#destructive-changes) below.
|
|
47
|
+
|
|
48
|
+
### CLI
|
|
49
|
+
|
|
50
|
+
For cron jobs — or whenever you'd rather not write a runner script — there is a small CLI. Put the same DSL (without the `SchemaFerry.define` wrapper) in a `Ferryfile`:
|
|
51
|
+
|
|
52
|
+
```ruby
|
|
53
|
+
source "mysql2://user:password@host:3306/source_db"
|
|
54
|
+
target "postgresql://user:password@host:5432/target_db"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Then:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
schema_ferry dry-run # show what would change (reads ./Ferryfile)
|
|
61
|
+
schema_ferry apply # apply to PostgreSQL
|
|
62
|
+
schema_ferry apply -c path/to/Ferryfile # explicit definition file path
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Each command prints the changes it applied (or would apply) followed by a one-line summary (`118 tables synced, 3 changes applied`). The exit status is 0 on success and 1 on any error, so cron mail and monitoring can rely on it.
|
|
66
|
+
|
|
67
|
+
### Custom conversion rules
|
|
68
|
+
|
|
69
|
+
```ruby
|
|
70
|
+
pipeline = SchemaFerry.define do
|
|
71
|
+
source "mysql2://user:password@host:3306/source_db"
|
|
72
|
+
target "postgresql://user:password@host:5432/target_db"
|
|
73
|
+
|
|
74
|
+
map_type :datetime, to: :timestamptz # override a default mapping (datetime → timestamp) globally
|
|
75
|
+
map_type :json, to: :json # e.g. opt out of the default json → jsonb conversion
|
|
76
|
+
|
|
77
|
+
table :users do
|
|
78
|
+
map_column :is_admin, type: :boolean # override a specific column's type
|
|
79
|
+
ignore_column :legacy_field # exclude a column
|
|
80
|
+
ignore_index :idx_old_legacy # exclude an index
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
ignore_table :old_sessions # exclude an entire table
|
|
84
|
+
end
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The same rules work in a CLI definition file.
|
|
88
|
+
|
|
89
|
+
### Destructive changes
|
|
90
|
+
|
|
91
|
+
`apply!` delegates to [ridgepole](https://github.com/ridgepole/ridgepole), which makes the target match the generated schema. For tables it manages, **columns and indexes that are missing from the generated schema are dropped from the target** (e.g. a column excluded via `ignore_column`, or an index created by hand on the target — declare those with `add_index` instead). Tables absent from the generated schema are themselves left untouched.
|
|
92
|
+
|
|
93
|
+
Review `dry_run` output before your first `apply!` and whenever you change the conversion rules — those are the moments that introduce drops. Unattended runs in between only mirror changes made to the MySQL schema; if even those need review, schedule `dry-run` instead and apply by hand.
|
|
94
|
+
|
|
95
|
+
## DSL reference
|
|
96
|
+
|
|
97
|
+
### Top-level
|
|
98
|
+
|
|
99
|
+
| Method | Description |
|
|
100
|
+
|---|---|
|
|
101
|
+
| `source "mysql2://..."` | Source MySQL connection string |
|
|
102
|
+
| `target "postgresql://..."` | Target PostgreSQL connection string |
|
|
103
|
+
| `map_type :from, to: :to` | Override a type globally (e.g. `map_type :datetime, to: :timestamptz`) |
|
|
104
|
+
| `enum_as :check` | Convert enum columns to varchar **plus a CHECK constraint** (default `:string` = plain varchar) |
|
|
105
|
+
| `ignore_table :name` | Exclude a table from conversion |
|
|
106
|
+
| `table :name do ... end` | Define per-table rules |
|
|
107
|
+
|
|
108
|
+
### Inside a `table` block
|
|
109
|
+
|
|
110
|
+
| Method | Description |
|
|
111
|
+
|---|---|
|
|
112
|
+
| `map_column :col, type: :type` | Override a column's type |
|
|
113
|
+
| `map_column :col, type: :type, default: value` | …and give it an explicit default |
|
|
114
|
+
| `ignore_column :col` | Exclude a column |
|
|
115
|
+
| `ignore_index :index_name` | Exclude an index |
|
|
116
|
+
| `add_index :col, ...options` | Declare a PostgreSQL-only index (options: `name`, `unique`, `using`, `opclass`, `where`, `order`) |
|
|
117
|
+
|
|
118
|
+
Ignoring a column also drops indexes and foreign keys that reference it. Renaming tables or columns is out of scope — clean up names after the cutover with a regular migration.
|
|
119
|
+
|
|
120
|
+
**tinyint(1) caveat:** ActiveRecord reads `tinyint(1)` as boolean, including its default (`DEFAULT 2` is read as `true`). If a `tinyint(1)` column actually holds 0/1/2-style values, override both the type and the default: `map_column :flags, type: :integer, default: 2`. Without an explicit default, schema_ferry drops the unreliable boolean default and warns.
|
|
121
|
+
|
|
122
|
+
## Default type mapping
|
|
123
|
+
|
|
124
|
+
| MySQL | PostgreSQL | Notes |
|
|
125
|
+
|---|---|---|
|
|
126
|
+
| `VARCHAR(n)` / `CHAR(n)` | `varchar(n)` | length preserved |
|
|
127
|
+
| `TEXT` / `MEDIUMTEXT` / `LONGTEXT` | `text` | size classes dropped — PostgreSQL `text` is unbounded |
|
|
128
|
+
| `TINYINT(1)` | `boolean` | see the caveat above if a column holds more than 0/1 |
|
|
129
|
+
| `TINYINT`…`BIGINT` (signed) | `smallint` / `integer` / `bigint` | widths normalized to PostgreSQL's three integer sizes |
|
|
130
|
+
| `TINYINT`…`INT` `UNSIGNED` | one size larger | e.g. `INT UNSIGNED` → `bigint` |
|
|
131
|
+
| `BIGINT UNSIGNED` | `numeric(20)` | PostgreSQL has no unsigned 8-byte integer; emitted with a warning |
|
|
132
|
+
| `FLOAT` / `DOUBLE` | `double precision` | |
|
|
133
|
+
| `DECIMAL(p,s)` | `numeric(p,s)` | |
|
|
134
|
+
| `DATETIME` / `TIMESTAMP` | `timestamp` | use `map_type :datetime, to: :timestamptz` for `timestamptz` |
|
|
135
|
+
| `DATE` / `TIME` | `date` / `time` | |
|
|
136
|
+
| `BINARY` / `BLOB` family | `bytea` | |
|
|
137
|
+
| `JSON` | `jsonb` | opt out with `map_type :json, to: :json` |
|
|
138
|
+
| `ENUM(...)` | `varchar` | add `enum_as :check` to enforce the values with a CHECK constraint |
|
|
139
|
+
|
|
140
|
+
`map_type` / `map_column` take Rails-style abstract type symbols (`:string`, `:integer`, `:jsonb`, …), not raw SQL type names.
|
|
141
|
+
|
|
142
|
+
### Automatic adjustments (with warnings)
|
|
143
|
+
|
|
144
|
+
Some MySQL constructs have no PostgreSQL equivalent. schema_ferry handles them and prints a `[schema_ferry]` warning to stderr:
|
|
145
|
+
|
|
146
|
+
- **FULLTEXT / SPATIAL indexes** are skipped. Declare a replacement with `add_index` (e.g. `add_index :body, using: :gin, opclass: :gin_trgm_ops` — requires `CREATE EXTENSION pg_trgm` on the target, done once by hand) and silence the warning with `ignore_index`. Don't create replacement indexes by hand: anything not in the generated schema is dropped on the next run.
|
|
147
|
+
- **Index prefix lengths** (`KEY (col(10))`) are dropped silently — PostgreSQL indexes the full column.
|
|
148
|
+
- **Identifiers over 63 bytes** (MySQL allows 64): index and foreign key names are shortened deterministically (`first 54 bytes + _ + 8-char digest`), so repeated runs stay stable. Overlong table names are only warned about — rename those yourself.
|
|
149
|
+
- **Zero-date defaults** (`'0000-00-00 00:00:00'`) are invalid in PostgreSQL and are dropped.
|
|
150
|
+
|
|
151
|
+
## How it works
|
|
152
|
+
|
|
153
|
+
Each run executes a three-stage pipeline:
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
MySQL schema
|
|
157
|
+
│
|
|
158
|
+
│ 1. Read (ActiveRecord)
|
|
159
|
+
▼
|
|
160
|
+
table definitions
|
|
161
|
+
│
|
|
162
|
+
│ 2. Convert (default mappings + your DSL rules)
|
|
163
|
+
▼
|
|
164
|
+
Schemafile
|
|
165
|
+
│
|
|
166
|
+
│ 3. Apply (ridgepole, diff only)
|
|
167
|
+
▼
|
|
168
|
+
PostgreSQL schema
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
1. **Read** — connects to MySQL and reads table definitions (columns, indexes, foreign keys) via ActiveRecord, using a connection pool isolated from any host Rails app
|
|
172
|
+
2. **Convert** — applies the default type mappings and your custom rules to build a PostgreSQL-ready schema
|
|
173
|
+
3. **Apply** — renders the schema as a [ridgepole](https://github.com/ridgepole/ridgepole) Schemafile and runs `ridgepole --apply` (or `--dry-run`) against the target database. ridgepole compares the declared schema with the target's current state and applies only the difference — that diffing is what makes runs incremental and idempotent, so schema_ferry never has to track what it applied before
|
|
174
|
+
|
|
175
|
+
## Development
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
bundle install
|
|
179
|
+
bundle exec rubocop
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Unit tests
|
|
183
|
+
|
|
184
|
+
Cover the DSL, conversion rules, and schema rendering. No database needed:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
bundle exec rspec spec/lib/
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Integration tests
|
|
191
|
+
|
|
192
|
+
Run the full pipeline against real MySQL and PostgreSQL containers:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
docker compose up -d --wait
|
|
196
|
+
INTEGRATION=true bundle exec rspec spec/integration/
|
|
197
|
+
docker compose down
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
[MIT License](https://opensource.org/licenses/MIT)
|
data/exe/schema_ferry
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module SchemaFerry
|
|
6
|
+
# Minimal command-line interface: `schema_ferry <apply|dry-run> [-c FILE]`.
|
|
7
|
+
# #run returns a process exit code instead of calling Kernel#exit so the
|
|
8
|
+
# class stays testable.
|
|
9
|
+
class CLI
|
|
10
|
+
DEFAULT_CONFIG_PATH = "Ferryfile"
|
|
11
|
+
COMMANDS = %w[apply dry-run].freeze
|
|
12
|
+
|
|
13
|
+
def initialize(stdout: $stdout, stderr: $stderr)
|
|
14
|
+
@stdout = stdout
|
|
15
|
+
@stderr = stderr
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def run(argv)
|
|
19
|
+
args = parser.parse(argv)
|
|
20
|
+
return print_and_succeed(parser.to_s) if @mode == :help
|
|
21
|
+
return print_and_succeed(VERSION) if @mode == :version
|
|
22
|
+
|
|
23
|
+
command = args.first
|
|
24
|
+
return usage_error("missing command") if command.nil?
|
|
25
|
+
return usage_error("unknown command: #{command}") unless COMMANDS.include?(command)
|
|
26
|
+
|
|
27
|
+
run_command(command)
|
|
28
|
+
rescue Error, OptionParser::ParseError => e
|
|
29
|
+
@stderr.puts "schema_ferry: #{e.message}"
|
|
30
|
+
1
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def run_command(command)
|
|
36
|
+
config = load_config
|
|
37
|
+
schemafile = Pipeline.new(config).schemafile
|
|
38
|
+
dry_run = command == "dry-run"
|
|
39
|
+
output = Target::RidgepoleRunner.new(config.target_url).run(schemafile, dry_run: dry_run)
|
|
40
|
+
|
|
41
|
+
@stdout.puts output
|
|
42
|
+
@stdout.puts summary(schemafile, output, dry_run: dry_run)
|
|
43
|
+
0
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def summary(schemafile, output, dry_run:)
|
|
47
|
+
tables = schemafile.scan(/^create_table /).count
|
|
48
|
+
changes = count_changes(output)
|
|
49
|
+
detail =
|
|
50
|
+
if changes.zero?
|
|
51
|
+
"no changes"
|
|
52
|
+
else
|
|
53
|
+
"#{changes} #{pluralize(changes, "change")} #{dry_run ? "pending" : "applied"}"
|
|
54
|
+
end
|
|
55
|
+
"#{tables} #{pluralize(tables, "table")} #{dry_run ? "checked" : "synced"}, #{detail}"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# `--apply` echoes each executed operation as a "-- op(...)" line;
|
|
59
|
+
# `--dry-run` prints the pending operations as top-level DSL calls.
|
|
60
|
+
def count_changes(output)
|
|
61
|
+
return 0 if output.include?("No change")
|
|
62
|
+
|
|
63
|
+
applied = output.scan(/^-- /).count
|
|
64
|
+
applied.positive? ? applied : output.scan(/^\w+\(/).count
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def pluralize(count, word)
|
|
68
|
+
count == 1 ? word : "#{word}s"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def load_config
|
|
72
|
+
path = @config_path || DEFAULT_CONFIG_PATH
|
|
73
|
+
raise ConfigError, "definition file not found: #{path}" unless File.exist?(path)
|
|
74
|
+
|
|
75
|
+
DSL::Config.load_file(path)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def parser
|
|
79
|
+
@parser ||= OptionParser.new do |o|
|
|
80
|
+
o.banner = "Usage: schema_ferry <apply|dry-run> [options]"
|
|
81
|
+
o.on("-c", "--config FILE", "Definition file (default: #{DEFAULT_CONFIG_PATH})") { |v| @config_path = v }
|
|
82
|
+
o.on("-h", "--help", "Show this help") { @mode = :help }
|
|
83
|
+
o.on("--version", "Show version") { @mode = :version }
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def print_and_succeed(text)
|
|
88
|
+
@stdout.puts text
|
|
89
|
+
0
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def usage_error(message)
|
|
93
|
+
@stderr.puts "schema_ferry: #{message}"
|
|
94
|
+
@stderr.puts parser
|
|
95
|
+
1
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaFerry
|
|
4
|
+
module Converter
|
|
5
|
+
class ColumnConverter
|
|
6
|
+
include Warnings
|
|
7
|
+
|
|
8
|
+
# PG has no unsigned integers: bump to the next size that holds the full
|
|
9
|
+
# unsigned range. Keys/values are AR byte limits.
|
|
10
|
+
UNSIGNED_LIMIT_BUMP = { 1 => 2, 2 => 4, 3 => 4, 4 => 8 }.freeze
|
|
11
|
+
|
|
12
|
+
def initialize(type_mapper)
|
|
13
|
+
@type_mapper = type_mapper
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call(raw, table_name, rule)
|
|
17
|
+
raw = bump_unsigned_integer(raw)
|
|
18
|
+
raw = drop_zero_date_default(raw, table_name)
|
|
19
|
+
override = rule&.column_type_overrides&.[](raw[:name])
|
|
20
|
+
col_opts = raw.slice(:limit, :precision, :scale, :null, :default, :default_function, :comment)
|
|
21
|
+
|
|
22
|
+
if override
|
|
23
|
+
col_opts[:default] = override_default(raw, rule, override)
|
|
24
|
+
ColumnSchema.new(name: raw[:name], type: override, **col_opts)
|
|
25
|
+
else
|
|
26
|
+
pg_type, pg_opts = @type_mapper.call(raw[:type], col_opts)
|
|
27
|
+
ColumnSchema.new(name: raw[:name], type: pg_type, **pg_opts)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def bump_unsigned_integer(raw)
|
|
34
|
+
return raw unless raw[:type] == :integer && raw[:sql_type].to_s.include?("unsigned")
|
|
35
|
+
|
|
36
|
+
if raw[:limit] == 8
|
|
37
|
+
emit_warning "column #{raw[:name].inspect}: BIGINT UNSIGNED has no PostgreSQL " \
|
|
38
|
+
"integer equivalent; mapped to decimal(20, 0)."
|
|
39
|
+
raw.merge(type: :decimal, limit: nil, precision: 20, scale: 0)
|
|
40
|
+
else
|
|
41
|
+
raw.merge(limit: UNSIGNED_LIMIT_BUMP.fetch(raw[:limit], raw[:limit]))
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# MySQL zero dates ('0000-00-00' …) are invalid on PostgreSQL. AR already
|
|
46
|
+
# nils out zero DATE defaults, but zero DATETIME defaults come through as
|
|
47
|
+
# strings.
|
|
48
|
+
def drop_zero_date_default(raw, table_name)
|
|
49
|
+
return raw unless raw[:default].is_a?(String) && raw[:default].start_with?("0000-00-00")
|
|
50
|
+
|
|
51
|
+
emit_warning "#{table_name}.#{raw[:name]}: default #{raw[:default].inspect} is " \
|
|
52
|
+
"invalid on PostgreSQL; the default was dropped."
|
|
53
|
+
raw.merge(default: nil)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# AR reads tinyint(1) defaults as booleans (DEFAULT 2 becomes true), so a
|
|
57
|
+
# default is unreliable once the type is overridden away from :boolean.
|
|
58
|
+
def override_default(raw, rule, override)
|
|
59
|
+
defaults = rule.column_default_overrides
|
|
60
|
+
return defaults[raw[:name]] if defaults.key?(raw[:name])
|
|
61
|
+
|
|
62
|
+
default = raw[:default]
|
|
63
|
+
if [true, false].include?(default) && override != :boolean
|
|
64
|
+
emit_warning "#{rule.table_name}.#{raw[:name]}: dropping default #{default.inspect} — " \
|
|
65
|
+
"MySQL reported a tinyint(1) default as boolean, which is unreliable under " \
|
|
66
|
+
"a type override. Restore it explicitly: " \
|
|
67
|
+
"map_column :#{raw[:name]}, type: :#{override}, default: <value>"
|
|
68
|
+
return nil
|
|
69
|
+
end
|
|
70
|
+
default
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaFerry
|
|
4
|
+
module Converter
|
|
5
|
+
# Builds CHECK constraints enforcing MySQL enum values on varchar columns
|
|
6
|
+
# (enum_as :check).
|
|
7
|
+
class EnumCheckBuilder
|
|
8
|
+
def call(raw_table, rule, ignored)
|
|
9
|
+
raw_table[:columns].filter_map do |col|
|
|
10
|
+
next if ignored.include?(col[:name])
|
|
11
|
+
# A type override takes the column away from varchar; the caller owns
|
|
12
|
+
# any constraint then.
|
|
13
|
+
next if rule&.column_type_overrides&.key?(col[:name])
|
|
14
|
+
|
|
15
|
+
values = enum_values(col[:sql_type])
|
|
16
|
+
next if values.nil?
|
|
17
|
+
|
|
18
|
+
build_constraint(raw_table[:name], col[:name], values)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def build_constraint(table_name, col_name, values)
|
|
25
|
+
CheckConstraintSchema.new(
|
|
26
|
+
expression: expression(col_name, values),
|
|
27
|
+
name: IdentifierShortener.shorten("chk_#{table_name}_#{col_name}",
|
|
28
|
+
kind: "check constraint", table: table_name)
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# "enum('a','b')" → ["a", "b"] (values keep MySQL's quote escaping).
|
|
33
|
+
def enum_values(sql_type)
|
|
34
|
+
return nil unless sql_type.to_s.start_with?("enum(")
|
|
35
|
+
|
|
36
|
+
sql_type.scan(/'((?:[^']|'')*)'/).flatten
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# PG stores CHECK expressions normalized; this is the fixed point of that
|
|
40
|
+
# normalization for a varchar column (verified on PG 16). Anything else —
|
|
41
|
+
# e.g. "kind IN ('a', 'b')" — makes ridgepole re-create it on every run.
|
|
42
|
+
def expression(column, values)
|
|
43
|
+
list = values.map { |v| "'#{v}'::character varying::text" }.join(", ")
|
|
44
|
+
"#{column}::text = ANY (ARRAY[#{list}])"
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module SchemaFerry
|
|
6
|
+
module Converter
|
|
7
|
+
# PostgreSQL truncates identifiers to 63 bytes (MySQL allows 64). A silently
|
|
8
|
+
# truncated index name makes ridgepole see a diff on every run, so names that
|
|
9
|
+
# would overflow are shortened deterministically instead.
|
|
10
|
+
module IdentifierShortener
|
|
11
|
+
extend Warnings
|
|
12
|
+
|
|
13
|
+
MAX_BYTES = 63
|
|
14
|
+
HASH_LENGTH = 8
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
def shorten(name, kind:, table:)
|
|
19
|
+
return name if name.nil? || name.bytesize <= MAX_BYTES
|
|
20
|
+
|
|
21
|
+
prefix = name.byteslice(0, MAX_BYTES - HASH_LENGTH - 1)
|
|
22
|
+
short = "#{prefix}_#{Digest::MD5.hexdigest(name)[0, HASH_LENGTH]}"
|
|
23
|
+
emit_warning "#{table}: #{kind} name #{name.inspect} exceeds PostgreSQL's " \
|
|
24
|
+
"#{MAX_BYTES}-byte identifier limit; renamed to #{short.inspect}."
|
|
25
|
+
short
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def warn_long_table_name(name)
|
|
29
|
+
return if name.bytesize <= MAX_BYTES
|
|
30
|
+
|
|
31
|
+
emit_warning "table name #{name.inspect} exceeds PostgreSQL's #{MAX_BYTES}-byte " \
|
|
32
|
+
"identifier limit and will be truncated by PostgreSQL. Rename the " \
|
|
33
|
+
"table before applying to avoid ridgepole re-creating it on every run."
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaFerry
|
|
4
|
+
module Converter
|
|
5
|
+
class SchemaConverter
|
|
6
|
+
include Warnings
|
|
7
|
+
|
|
8
|
+
# PostgreSQL has no FULLTEXT/SPATIAL equivalent that ridgepole can express.
|
|
9
|
+
UNSUPPORTED_INDEX_TYPES = %i[fulltext spatial].freeze
|
|
10
|
+
|
|
11
|
+
def initialize(config)
|
|
12
|
+
@column_converter = ColumnConverter.new(TypeMapper.new(config.global_type_overrides))
|
|
13
|
+
@table_rules = config.table_rules
|
|
14
|
+
@ignored_tables = config.ignored_tables
|
|
15
|
+
@enum_check = (EnumCheckBuilder.new if config.enum_mode == :check)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def convert(raw_tables)
|
|
19
|
+
raw_tables
|
|
20
|
+
.reject { |t| @ignored_tables.include?(t[:name]) }
|
|
21
|
+
.map { |t| convert_table(t) }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def convert_table(raw)
|
|
27
|
+
rule = @table_rules[raw[:name]]
|
|
28
|
+
ignored = rule&.ignored_columns || []
|
|
29
|
+
IdentifierShortener.warn_long_table_name(raw[:name])
|
|
30
|
+
|
|
31
|
+
TableSchema.new(
|
|
32
|
+
name: raw[:name],
|
|
33
|
+
primary_key: raw[:primary_key],
|
|
34
|
+
pk_type: convert_pk_type(raw),
|
|
35
|
+
pk_limit: raw[:pk_limit],
|
|
36
|
+
comment: raw[:comment],
|
|
37
|
+
columns: convert_columns(raw[:columns], raw[:name], rule, ignored),
|
|
38
|
+
indexes: convert_indexes(raw[:indexes], raw[:name], rule, ignored),
|
|
39
|
+
foreign_keys: convert_foreign_keys(raw[:foreign_keys], ignored),
|
|
40
|
+
check_constraints: build_check_constraints(raw, rule, ignored)
|
|
41
|
+
)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# MySQL BIGINT comes through AR as :integer with limit 8, so the sql_type
|
|
45
|
+
# is the only reliable source for the id: option.
|
|
46
|
+
def convert_pk_type(raw)
|
|
47
|
+
return raw[:pk_type] unless raw[:pk_type] == :integer
|
|
48
|
+
|
|
49
|
+
sql_type = raw[:pk_sql_type].to_s
|
|
50
|
+
if sql_type.include?("unsigned")
|
|
51
|
+
if sql_type.start_with?("bigint")
|
|
52
|
+
emit_warning "table #{raw[:name]}: BIGINT UNSIGNED primary key has no PostgreSQL " \
|
|
53
|
+
"equivalent; using signed bigint (values above 2^63-1 will not fit)."
|
|
54
|
+
end
|
|
55
|
+
:bigint
|
|
56
|
+
elsif sql_type.start_with?("bigint")
|
|
57
|
+
:bigint
|
|
58
|
+
else
|
|
59
|
+
:integer
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def convert_columns(raw_columns, table_name, rule, ignored)
|
|
64
|
+
raw_columns
|
|
65
|
+
.reject { |c| ignored.include?(c[:name]) }
|
|
66
|
+
.map { |c| @column_converter.call(c, table_name, rule) }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def convert_indexes(raw_indexes, table_name, rule, ignored)
|
|
70
|
+
converted = raw_indexes
|
|
71
|
+
.reject { |idx| rule&.ignored_indexes&.include?(idx[:name]) }
|
|
72
|
+
.reject { |idx| skip_unsupported_index?(idx, table_name) }
|
|
73
|
+
.reject { |idx| idx[:columns].intersect?(ignored) }
|
|
74
|
+
.map { |idx| build_index_schema(idx, table_name) }
|
|
75
|
+
converted + extra_indexes(rule, table_name)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def extra_indexes(rule, table_name)
|
|
79
|
+
(rule&.extra_indexes || []).map do |extra|
|
|
80
|
+
opts = extra[:options]
|
|
81
|
+
name = opts[:name] || "index_#{table_name}_on_#{extra[:columns].join("_")}"
|
|
82
|
+
IndexSchema.new(
|
|
83
|
+
name: IdentifierShortener.shorten(name.to_s, kind: "index", table: table_name),
|
|
84
|
+
columns: extra[:columns],
|
|
85
|
+
unique: opts[:unique],
|
|
86
|
+
using: opts[:using],
|
|
87
|
+
opclass: opts[:opclass],
|
|
88
|
+
where: opts[:where],
|
|
89
|
+
orders: opts[:order],
|
|
90
|
+
lengths: nil
|
|
91
|
+
)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def convert_foreign_keys(raw_fkeys, ignored)
|
|
96
|
+
raw_fkeys
|
|
97
|
+
.reject { |fk| @ignored_tables.include?(fk[:to_table]) }
|
|
98
|
+
.reject { |fk| ignored.include?(fk[:column]) }
|
|
99
|
+
.map { |fk| build_fk_schema(fk) }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def skip_unsupported_index?(idx, table_name)
|
|
103
|
+
return false unless UNSUPPORTED_INDEX_TYPES.include?(idx[:type])
|
|
104
|
+
|
|
105
|
+
emit_warning "#{table_name}: skipping #{idx[:type].to_s.upcase} index #{idx[:name].inspect} " \
|
|
106
|
+
"(no PostgreSQL equivalent). Declare a replacement with add_index " \
|
|
107
|
+
"(e.g. add_index :col, using: :gin, opclass: :gin_trgm_ops) and " \
|
|
108
|
+
"silence this warning with ignore_index :#{idx[:name]}."
|
|
109
|
+
true
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def build_index_schema(raw, table_name)
|
|
113
|
+
IndexSchema.new(
|
|
114
|
+
name: IdentifierShortener.shorten(raw[:name], kind: "index", table: table_name),
|
|
115
|
+
columns: raw[:columns],
|
|
116
|
+
unique: raw[:unique],
|
|
117
|
+
using: raw[:using],
|
|
118
|
+
opclass: nil, # MySQL has no operator classes
|
|
119
|
+
where: raw[:where],
|
|
120
|
+
lengths: raw[:lengths],
|
|
121
|
+
orders: raw[:orders]
|
|
122
|
+
)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def build_fk_schema(raw)
|
|
126
|
+
ForeignKeySchema.new(
|
|
127
|
+
from_table: raw[:from_table],
|
|
128
|
+
to_table: raw[:to_table],
|
|
129
|
+
column: raw[:column],
|
|
130
|
+
primary_key: raw[:primary_key],
|
|
131
|
+
on_update: raw[:on_update],
|
|
132
|
+
on_delete: raw[:on_delete],
|
|
133
|
+
name: IdentifierShortener.shorten(raw[:name], kind: "foreign key", table: raw[:from_table])
|
|
134
|
+
)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def build_check_constraints(raw, rule, ignored)
|
|
138
|
+
return [] unless @enum_check
|
|
139
|
+
|
|
140
|
+
@enum_check.call(raw, rule, ignored)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|