@memberjunction/sqlglot-ts 5.40.2 → 5.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/MJPostgresTranspiler.d.ts +66 -0
- package/dist/MJPostgresTranspiler.d.ts.map +1 -0
- package/dist/MJPostgresTranspiler.js +126 -0
- package/dist/MJPostgresTranspiler.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/python/mj_postgres.py +1571 -0
- package/src/python/test_mj_postgres.py +573 -0
|
@@ -0,0 +1,1571 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MemberJunction PostgreSQL dialect for sqlglot.
|
|
3
|
+
|
|
4
|
+
Encodes MJ's SQL-Server→PostgreSQL conventions in the sqlglot AST (custom
|
|
5
|
+
Generator + AST transform passes) rather than in output-text regex. This is the
|
|
6
|
+
"Category B" (regular DDL/DML) transpiler for the split-and-regenerate pipeline;
|
|
7
|
+
procedural SQL, CodeGen objects, and mj-sync metadata are handled out of band
|
|
8
|
+
(regenerated / re-seeded / hand-authored), so this dialect only has to be strong
|
|
9
|
+
on the regular-DDL surface — which is sqlglot's sweet spot.
|
|
10
|
+
|
|
11
|
+
What is encoded in the AST:
|
|
12
|
+
* Type mapping: BIT→BOOLEAN, UNIQUEIDENTIFIER→UUID, NVARCHAR/VARCHAR(MAX)→TEXT,
|
|
13
|
+
DATETIME/DATETIME2/SMALLDATETIME→TIMESTAMPTZ, MONEY→DECIMAL(19,4).
|
|
14
|
+
* Function rewrites: NEWID/NEWSEQUENTIALID→gen_random_uuid(), GETDATE/GETUTCDATE→now().
|
|
15
|
+
* Boolean column defaults: BIT DEFAULT 1/0 → BOOLEAN DEFAULT TRUE/FALSE.
|
|
16
|
+
* Identifier quoting (PascalCase identifiers preserved via double-quotes).
|
|
17
|
+
|
|
18
|
+
The one non-SQL token, the Flyway macro ``${flyway:defaultSchema}``, is protected
|
|
19
|
+
to a sentinel identifier before parsing and restored verbatim by the Generator
|
|
20
|
+
(``identifier_sql``) — an AST-level restore, not an output-text rewrite. It is a
|
|
21
|
+
templating macro, not SQL, so it cannot live inside the SQL grammar itself.
|
|
22
|
+
|
|
23
|
+
Reference for the SS→PG type overrides: SQLConverter ``TypeResolver.MJ_OVERRIDES``.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json as _json
|
|
28
|
+
import os as _os
|
|
29
|
+
import sqlglot
|
|
30
|
+
from sqlglot import exp
|
|
31
|
+
from sqlglot.dialects.postgres import Postgres
|
|
32
|
+
from sqlglot.dialects.tsql import TSQL
|
|
33
|
+
|
|
34
|
+
# The Flyway schema macro is not SQL; protect it to a sentinel identifier for the
|
|
35
|
+
# parse, then the Generator restores it verbatim.
|
|
36
|
+
FLYWAY_MACRO = "${flyway:defaultSchema}"
|
|
37
|
+
FLYWAY_SENTINEL = "__mj_flyway_default_schema__"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class MJPostgres(Postgres):
|
|
41
|
+
"""PostgreSQL generation with MJ's type/function/boolean conventions baked in."""
|
|
42
|
+
|
|
43
|
+
class Generator(Postgres.Generator):
|
|
44
|
+
TRANSFORMS = {
|
|
45
|
+
**Postgres.Generator.TRANSFORMS,
|
|
46
|
+
# SS JSON_VALUE(col,'$.path') → JSONExtractScalar. PG's default
|
|
47
|
+
# JSON_EXTRACT_PATH_TEXT needs a json arg, but MJ stores JSON in text
|
|
48
|
+
# columns; cast to jsonb and use ->> (single key) / #>> (nested).
|
|
49
|
+
exp.JSONExtractScalar: lambda self, e: _json_extract_scalar_sql(self, e),
|
|
50
|
+
}
|
|
51
|
+
TYPE_MAPPING = {
|
|
52
|
+
**Postgres.Generator.TYPE_MAPPING,
|
|
53
|
+
exp.DataType.Type.BIT: "BOOLEAN",
|
|
54
|
+
# SS date/time types are tz-aware in MJ (oracle stores them as
|
|
55
|
+
# `timestamp with time zone`); sqlglot otherwise defaults to plain TIMESTAMP.
|
|
56
|
+
exp.DataType.Type.DATETIME: "TIMESTAMPTZ",
|
|
57
|
+
exp.DataType.Type.DATETIME2: "TIMESTAMPTZ",
|
|
58
|
+
exp.DataType.Type.SMALLDATETIME: "TIMESTAMPTZ",
|
|
59
|
+
# SS currency types: PG's MONEY is locale-dependent and SMALLMONEY isn't a
|
|
60
|
+
# PG type at all; match codegen's regenerated schema (DECIMAL with SS scale).
|
|
61
|
+
exp.DataType.Type.MONEY: "DECIMAL(19, 4)",
|
|
62
|
+
exp.DataType.Type.SMALLMONEY: "DECIMAL(10, 4)",
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
def identifier_sql(self, expression: exp.Identifier) -> str:
|
|
66
|
+
# Restore the Flyway macro verbatim (unquoted) — AST-level, not regex.
|
|
67
|
+
if expression.name == FLYWAY_SENTINEL:
|
|
68
|
+
return FLYWAY_MACRO
|
|
69
|
+
return super().identifier_sql(expression)
|
|
70
|
+
|
|
71
|
+
def literal_sql(self, expression: exp.Literal) -> str:
|
|
72
|
+
# The macro is protected to the sentinel by a blanket text replace before the
|
|
73
|
+
# parse, so it also gets baked into the *content* of string literals — e.g. the
|
|
74
|
+
# Entity.SchemaName seed value `'${flyway:defaultSchema}'`, or a schema qualifier
|
|
75
|
+
# embedded in a stored RowLevelSecurityFilter predicate. `identifier_sql` only
|
|
76
|
+
# restores the macro in identifier position; string-literal content never passes
|
|
77
|
+
# through it, so restore the sentinel here too (substring, since it may be a
|
|
78
|
+
# qualifier inside a larger predicate string). AST-level, not an output rewrite.
|
|
79
|
+
if expression.is_string and expression.this and FLYWAY_SENTINEL in expression.this:
|
|
80
|
+
restored = expression.this.replace(FLYWAY_SENTINEL, FLYWAY_MACRO)
|
|
81
|
+
return f"{self.dialect.QUOTE_START}{self.escape_str(restored)}{self.dialect.QUOTE_END}"
|
|
82
|
+
return super().literal_sql(expression)
|
|
83
|
+
|
|
84
|
+
def datatype_sql(self, expression: exp.DataType) -> str:
|
|
85
|
+
has_max = any(
|
|
86
|
+
isinstance(e, exp.DataTypeParam) and isinstance(e.this, exp.Var) and e.name.upper() == "MAX"
|
|
87
|
+
for e in expression.expressions
|
|
88
|
+
)
|
|
89
|
+
# SS NVARCHAR(MAX)/VARCHAR(MAX) → PG TEXT (PG has no MAX length sentinel).
|
|
90
|
+
if has_max and expression.this in (exp.DataType.Type.VARCHAR, exp.DataType.Type.NVARCHAR, exp.DataType.Type.CHAR):
|
|
91
|
+
return "TEXT"
|
|
92
|
+
# SS VARBINARY/BINARY(MAX|n) → PG BYTEA (takes no length modifier).
|
|
93
|
+
if expression.this in (exp.DataType.Type.VARBINARY, exp.DataType.Type.BINARY):
|
|
94
|
+
return "BYTEA"
|
|
95
|
+
# SS FLOAT[(n)]: n<=24 is 4-byte (REAL), n>24 or bare is 8-byte (DOUBLE
|
|
96
|
+
# PRECISION). sqlglot maps bare FLOAT→REAL, narrowing to 4-byte; correct it.
|
|
97
|
+
if expression.this in (exp.DataType.Type.FLOAT, exp.DataType.Type.DOUBLE):
|
|
98
|
+
params = [e for e in expression.expressions if isinstance(e, exp.DataTypeParam)]
|
|
99
|
+
precision = None
|
|
100
|
+
if params and isinstance(params[0].this, exp.Literal) and not params[0].this.is_string:
|
|
101
|
+
precision = int(params[0].name)
|
|
102
|
+
return "REAL" if (precision is not None and precision <= 24) else "DOUBLE PRECISION"
|
|
103
|
+
return super().datatype_sql(expression)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# Functions whose SS form maps to a PG builtin (name-only rewrite).
|
|
107
|
+
_FUNC_REWRITES = {
|
|
108
|
+
"NEWID": "GEN_RANDOM_UUID",
|
|
109
|
+
"NEWSEQUENTIALID": "GEN_RANDOM_UUID",
|
|
110
|
+
"GETDATE": "NOW",
|
|
111
|
+
"GETUTCDATE": "NOW",
|
|
112
|
+
"SYSDATETIME": "NOW",
|
|
113
|
+
"SYSUTCDATETIME": "NOW",
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# SS principal/session functions → PG CURRENT_USER (emitted bare, no parens).
|
|
118
|
+
# (SUSER_NAME/SUSER_SNAME already parse to exp.CurrentUser in sqlglot; USER_NAME
|
|
119
|
+
# parses to an Anonymous func and needs the explicit rewrite. These show up in
|
|
120
|
+
# column DEFAULTs — left untranslated they fail the whole CREATE TABLE.)
|
|
121
|
+
_CURRENT_USER_FUNCS = {"USER_NAME", "SUSER_NAME", "SUSER_SNAME", "CURRENT_USER", "SESSION_USER"}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _rewrite_functions(node: exp.Expression) -> exp.Expression:
|
|
125
|
+
"""Rewrite SS-specific functions to PG equivalents in the AST."""
|
|
126
|
+
if isinstance(node, exp.Anonymous) and node.name:
|
|
127
|
+
upper = node.name.upper()
|
|
128
|
+
if upper in _CURRENT_USER_FUNCS:
|
|
129
|
+
return exp.CurrentUser()
|
|
130
|
+
if upper in _FUNC_REWRITES:
|
|
131
|
+
return exp.func(_FUNC_REWRITES[upper])
|
|
132
|
+
# Some SS funcs parse to typed nodes rather than Anonymous. SYSDATETIMEOFFSET()
|
|
133
|
+
# parses to CurrentTimestampLTZ; GETDATE() to CurrentTimestamp.
|
|
134
|
+
if isinstance(node, (exp.CurrentTimestamp, exp.CurrentTimestampLTZ)):
|
|
135
|
+
return exp.func("NOW")
|
|
136
|
+
return node
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _rewrite_string_concat(node: exp.Expression) -> exp.Expression:
|
|
140
|
+
"""T-SQL string concatenation with `+` → PG `||`. Only when an operand is clearly
|
|
141
|
+
a string (literal, N'...', or an already-rewritten `||` chain), so numeric `a + b`
|
|
142
|
+
is left alone. Post-order traversal handles chains (`'a' + b + 'c'`) innermost-first."""
|
|
143
|
+
if isinstance(node, exp.Add):
|
|
144
|
+
def is_strish(x: exp.Expression) -> bool:
|
|
145
|
+
return isinstance(x, (exp.National, exp.DPipe)) or (isinstance(x, exp.Literal) and x.is_string)
|
|
146
|
+
if is_strish(node.left) or is_strish(node.right):
|
|
147
|
+
return exp.DPipe(this=node.left, expression=node.right)
|
|
148
|
+
return node
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _json_extract_scalar_sql(self, expression: exp.JSONExtractScalar) -> str:
|
|
152
|
+
"""JSONExtractScalar (from SS JSON_VALUE) → `(col::jsonb) ->> 'key'` (single path)
|
|
153
|
+
or `(col::jsonb) #>> '{a,b}'` (nested). Casting to jsonb lets it work on the text
|
|
154
|
+
columns MJ stores JSON in (PG's JSON_EXTRACT_PATH_TEXT requires a real json arg)."""
|
|
155
|
+
this = self.sql(expression, "this")
|
|
156
|
+
path = expression.expression
|
|
157
|
+
# Skip the `$` root element (empty name) — keep only the key/segment names.
|
|
158
|
+
parts = [p.name for p in path.expressions if p.name] if isinstance(path, exp.JSONPath) else None
|
|
159
|
+
if parts and len(parts) > 1:
|
|
160
|
+
return f"(({this})::jsonb #>> '{{{','.join(parts)}}}')"
|
|
161
|
+
key = parts[0] if parts else (path.name if path else "")
|
|
162
|
+
return f"(({this})::jsonb ->> '{key}')"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _rewrite_insert_booleans(node: exp.Expression) -> exp.Expression:
|
|
166
|
+
"""In `INSERT INTO t (…cols…) VALUES (…)`, coerce 1/0 → TRUE/FALSE for any column
|
|
167
|
+
known to be BIT/BOOLEAN (file-level + MJ_EXTRA_BIT_COLS registry). SS seeds bit
|
|
168
|
+
columns with 1/0; PG won't implicitly cast integer → boolean, so the INSERT aborts."""
|
|
169
|
+
if not isinstance(node, exp.Insert) or not isinstance(node.this, exp.Schema):
|
|
170
|
+
return node
|
|
171
|
+
tbl = node.this.this
|
|
172
|
+
tname = tbl.name.lower() if isinstance(tbl, exp.Table) else None
|
|
173
|
+
if not tname:
|
|
174
|
+
return node
|
|
175
|
+
cols = [c.name.lower() for c in node.this.expressions]
|
|
176
|
+
bool_pos = [i for i, c in enumerate(cols) if (tname, c) in _BIT_COLS]
|
|
177
|
+
if not bool_pos:
|
|
178
|
+
return node
|
|
179
|
+
values = node.expression
|
|
180
|
+
if not isinstance(values, exp.Values):
|
|
181
|
+
return node
|
|
182
|
+
for tup in values.expressions:
|
|
183
|
+
if not isinstance(tup, exp.Tuple):
|
|
184
|
+
continue
|
|
185
|
+
for i in bool_pos:
|
|
186
|
+
if i < len(tup.expressions):
|
|
187
|
+
v = tup.expressions[i]
|
|
188
|
+
if isinstance(v, exp.Literal) and not v.is_string and v.name in ("0", "1"):
|
|
189
|
+
tup.set("expressions", [
|
|
190
|
+
(exp.true() if v.name == "1" else exp.false()) if j == i else e
|
|
191
|
+
for j, e in enumerate(tup.expressions)
|
|
192
|
+
])
|
|
193
|
+
return node
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _is_isjson(node: exp.Expression) -> bool:
|
|
197
|
+
return isinstance(node, exp.Anonymous) and (node.name or "").upper() == "ISJSON"
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _rewrite_isjson_eq(node: exp.Expression) -> exp.Expression:
|
|
201
|
+
"""SS `ISJSON(x) = 1` / `ISJSON(x) = 0` → PG `x IS JSON` / `x IS NOT JSON`.
|
|
202
|
+
PG has no ISJSON function; it has the SQL:2016 `IS JSON` predicate (PG16+)."""
|
|
203
|
+
if isinstance(node, exp.EQ) and _is_isjson(node.this) and isinstance(node.expression, exp.Literal):
|
|
204
|
+
arg = node.this.expressions[0]
|
|
205
|
+
is_json = exp.Is(this=arg.copy(), expression=exp.JSON())
|
|
206
|
+
return exp.Not(this=is_json) if node.expression.name == "0" else is_json
|
|
207
|
+
return node
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _rewrite_isjson_bare(node: exp.Expression) -> exp.Expression:
|
|
211
|
+
"""Bare `ISJSON(x)` predicate (not compared to 1/0) → `x IS JSON`."""
|
|
212
|
+
if _is_isjson(node):
|
|
213
|
+
return exp.Is(this=node.expressions[0].copy(), expression=exp.JSON())
|
|
214
|
+
return node
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _is_outer_join(join: exp.Join) -> bool:
|
|
218
|
+
"""True for LEFT/RIGHT/FULL (or explicit OUTER) joins."""
|
|
219
|
+
return (join.side or "").upper() in ("LEFT", "RIGHT", "FULL") or (join.kind or "").upper() == "OUTER"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# sqlglot stores the UPDATE…FROM clause under the `from_` arg key (older versions used
|
|
223
|
+
# `from`). Resolve the live key once so the self-aliased-UPDATE rewrite keeps working across
|
|
224
|
+
# sqlglot upgrades — a silent miss here leaves `UPDATE alias …` which PG rejects (`relation
|
|
225
|
+
# "alias" does not exist`).
|
|
226
|
+
_UPDATE_FROM_KEY = "from_" if "from_" in exp.Update.arg_types else "from"
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _update_alias_outer_join(stmt: exp.Update) -> bool:
|
|
230
|
+
"""True when the self-aliased `UPDATE alias … FROM Target AS alias JOIN …` shape uses
|
|
231
|
+
a LEFT/RIGHT/FULL join. `_rewrite_update_from_alias` moves join ON conditions into
|
|
232
|
+
WHERE, which is only sound for INNER joins — an outer-join anti-join (`LEFT JOIN o …
|
|
233
|
+
WHERE o.ID IS NULL`) would silently become inner-join semantics and update zero rows.
|
|
234
|
+
There is no safe mechanical PG rewrite, so these are reported as unhandled instead."""
|
|
235
|
+
frm = stmt.args.get(_UPDATE_FROM_KEY)
|
|
236
|
+
tgt = stmt.this
|
|
237
|
+
if not frm or not isinstance(frm.this, exp.Table) or not isinstance(tgt, exp.Table):
|
|
238
|
+
return False
|
|
239
|
+
base = frm.this
|
|
240
|
+
joins = base.args.get("joins") or []
|
|
241
|
+
if not base.alias or tgt.name != base.alias or not joins:
|
|
242
|
+
return False
|
|
243
|
+
return any(_is_outer_join(j) for j in joins)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _rewrite_update_from_alias(node: exp.Expression) -> exp.Expression:
|
|
247
|
+
"""SS `UPDATE alias SET alias.c = … FROM Target AS alias [JOIN Other o ON j] WHERE w`
|
|
248
|
+
→ PG `UPDATE Target AS alias SET c = … FROM Other o WHERE j AND w`.
|
|
249
|
+
PG forbids naming the target via an alias defined in FROM and forbids the alias
|
|
250
|
+
qualifier on SET targets; the target's join condition must move to WHERE.
|
|
251
|
+
INNER joins only — outer joins are routed to unhandled upstream (see
|
|
252
|
+
`_update_alias_outer_join`)."""
|
|
253
|
+
if not isinstance(node, exp.Update):
|
|
254
|
+
return node
|
|
255
|
+
frm = node.args.get(_UPDATE_FROM_KEY)
|
|
256
|
+
tgt = node.this
|
|
257
|
+
if not frm or not isinstance(frm.this, exp.Table) or not isinstance(tgt, exp.Table):
|
|
258
|
+
return node
|
|
259
|
+
base = frm.this
|
|
260
|
+
base_alias = base.alias
|
|
261
|
+
joins = base.args.get("joins") or []
|
|
262
|
+
if not base_alias or tgt.name != base_alias or not joins:
|
|
263
|
+
return node # only the self-aliased-target + join shape
|
|
264
|
+
if any(_is_outer_join(j) for j in joins):
|
|
265
|
+
return node # moving an outer ON to WHERE changes semantics — handled upstream
|
|
266
|
+
# target ← the real FROM base table (alias preserved), stripped of its joins
|
|
267
|
+
new_target = base.copy()
|
|
268
|
+
new_target.set("joins", None)
|
|
269
|
+
node.set("this", new_target)
|
|
270
|
+
# ALL join ON conditions → WHERE; the joined tables become the new FROM. The
|
|
271
|
+
# extra joins keep only their table source (CROSS JOIN) so their predicate isn't
|
|
272
|
+
# duplicated between FROM and WHERE — WHERE alone carries it (inner semantics).
|
|
273
|
+
conds = [j.args["on"].copy() for j in joins if j.args.get("on")]
|
|
274
|
+
first = joins[0].this.copy()
|
|
275
|
+
if len(joins) > 1:
|
|
276
|
+
extra = []
|
|
277
|
+
for j in joins[1:]:
|
|
278
|
+
jc = j.copy()
|
|
279
|
+
jc.set("on", None)
|
|
280
|
+
jc.set("side", None)
|
|
281
|
+
jc.set("kind", "CROSS")
|
|
282
|
+
extra.append(jc)
|
|
283
|
+
first.set("joins", extra)
|
|
284
|
+
node.set(_UPDATE_FROM_KEY, exp.From(this=first))
|
|
285
|
+
existing = node.args["where"].this if node.args.get("where") else None
|
|
286
|
+
clauses = conds + ([existing] if existing else [])
|
|
287
|
+
if clauses:
|
|
288
|
+
combined = clauses[0]
|
|
289
|
+
for c in clauses[1:]:
|
|
290
|
+
combined = exp.And(this=combined, expression=c)
|
|
291
|
+
node.set("where", exp.Where(this=combined))
|
|
292
|
+
# strip the target-alias qualifier from SET LHS (PG: SET col = …, not alias.col = …)
|
|
293
|
+
for assign in node.expressions:
|
|
294
|
+
lhs = assign.this if isinstance(assign, exp.EQ) else None
|
|
295
|
+
if isinstance(lhs, exp.Column) and lhs.table == base_alias:
|
|
296
|
+
lhs.set("table", None)
|
|
297
|
+
return node
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
# Cross-statement BIT-column registry, keyed (table_lower, column_lower). Built once
|
|
301
|
+
# per transpile pass by `_collect_bit_columns` because a `CHECK (BitCol = 1)` often lives
|
|
302
|
+
# in a separate `ALTER TABLE … ADD CONSTRAINT` whose target column was declared BIT in an
|
|
303
|
+
# earlier CREATE TABLE / ALTER ADD COLUMN batch (so single-statement context isn't enough).
|
|
304
|
+
_BIT_COLS: set[tuple[str, str]] = set()
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _table_name_of(stmt: exp.Expression) -> str | None:
|
|
308
|
+
"""Lower-cased name of the table a CREATE/ALTER/UPDATE/DELETE statement targets, if any.
|
|
309
|
+
|
|
310
|
+
DML (UPDATE/DELETE) is included so the boolean-int rewriter can resolve a statement's
|
|
311
|
+
target table against the BIT-column registry and coerce `bitcol = 1/0` → `TRUE/FALSE`
|
|
312
|
+
in SET assignments and WHERE predicates — not just in CREATE/ALTER CHECK constraints.
|
|
313
|
+
For UPDATE/DELETE the target lives in `stmt.this`; if that is an aliased subtree we fall
|
|
314
|
+
back to the first Table node."""
|
|
315
|
+
if isinstance(stmt, exp.Create):
|
|
316
|
+
t = stmt.this.find(exp.Table) if stmt.this else None
|
|
317
|
+
elif isinstance(stmt, (exp.Alter, exp.Update, exp.Delete)):
|
|
318
|
+
t = stmt.this if isinstance(stmt.this, exp.Table) else stmt.find(exp.Table)
|
|
319
|
+
else:
|
|
320
|
+
t = None
|
|
321
|
+
return t.name.lower() if isinstance(t, exp.Table) and t.name else None
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _collect_bit_columns(sql: str) -> set[tuple[str, str]]:
|
|
325
|
+
"""One pass over the whole file collecting (table, column) for every BIT column
|
|
326
|
+
declared in a CREATE TABLE or ALTER ADD COLUMN — used to resolve boolean CHECKs
|
|
327
|
+
that sit in separate ALTER statements."""
|
|
328
|
+
out: set[tuple[str, str]] = set()
|
|
329
|
+
protected = sql.replace(FLYWAY_MACRO, FLYWAY_SENTINEL)
|
|
330
|
+
# Parse per GO batch via the resilient splitter — a single unparseable statement
|
|
331
|
+
# (common in baselines) must only drop itself, not wipe the registry for every
|
|
332
|
+
# other table declared in the same batch.
|
|
333
|
+
for batch in _GO_SPLIT.split(protected):
|
|
334
|
+
if not batch.strip():
|
|
335
|
+
continue
|
|
336
|
+
for st, _raw in _parse_resilient(batch):
|
|
337
|
+
if st is None:
|
|
338
|
+
continue
|
|
339
|
+
tbl = _table_name_of(st)
|
|
340
|
+
if not tbl:
|
|
341
|
+
continue
|
|
342
|
+
for cd in st.find_all(exp.ColumnDef):
|
|
343
|
+
if cd.kind is not None and cd.kind.this in (exp.DataType.Type.BIT, exp.DataType.Type.BOOLEAN):
|
|
344
|
+
out.add((tbl, cd.name.lower()))
|
|
345
|
+
return out
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _boolean_columns_for(stmt: exp.Expression) -> set[str]:
|
|
349
|
+
"""Lower-cased names of columns in scope for `stmt` that are BIT/BOOLEAN: those declared
|
|
350
|
+
in the statement itself, plus the `_BIT_COLS` registry entries (file-level declarations +
|
|
351
|
+
cross-file baselines) for the statement's target table. Shared by the coercion pass and
|
|
352
|
+
the residual self-check so both reason about the same type set."""
|
|
353
|
+
cols = {
|
|
354
|
+
cd.name.lower()
|
|
355
|
+
for cd in stmt.find_all(exp.ColumnDef)
|
|
356
|
+
if cd.kind is not None and cd.kind.this in (exp.DataType.Type.BIT, exp.DataType.Type.BOOLEAN)
|
|
357
|
+
}
|
|
358
|
+
tbl = _table_name_of(stmt)
|
|
359
|
+
if tbl:
|
|
360
|
+
cols |= {col for (t, col) in _BIT_COLS if t == tbl}
|
|
361
|
+
return cols
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _unwrap_paren(x: exp.Expression) -> exp.Expression:
|
|
365
|
+
while isinstance(x, exp.Paren): # SS wraps the literal in parens: `IsActive = (1)`
|
|
366
|
+
x = x.this
|
|
367
|
+
return x
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _bool_literal(node: exp.Expression):
|
|
371
|
+
"""The paren-unwrapped node as exp.true()/false() if it is a `0`/`1` integer literal,
|
|
372
|
+
else None. Used to decide whether a comparand should be coerced to a PG boolean."""
|
|
373
|
+
node = _unwrap_paren(node)
|
|
374
|
+
if isinstance(node, exp.Literal) and not node.is_string and node.name in ("0", "1"):
|
|
375
|
+
return exp.true() if node.name == "1" else exp.false()
|
|
376
|
+
return None
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _is_bool_col(node: exp.Expression, bool_cols: set[str]) -> bool:
|
|
380
|
+
node = _unwrap_paren(node)
|
|
381
|
+
return isinstance(node, exp.Column) and node.name.lower() in bool_cols
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _rewrite_boolean_int_comparisons(stmt: exp.Expression) -> exp.Expression:
|
|
385
|
+
"""Coerce `<bitcol> <op> 0/1` → `<bitcol> <op> TRUE/FALSE` for any column known to be
|
|
386
|
+
BIT/BOOLEAN, in ANY syntactic context — CHECK, WHERE, SET assignment, JOIN ON — and across
|
|
387
|
+
`=`, `<>`/`!=`, and `IN (…)`. After BIT→BOOLEAN, a `boolean = integer` comparison/assignment
|
|
388
|
+
is a PG type error that aborts the statement; this is the single type-driven pass that
|
|
389
|
+
prevents it everywhere rather than per-construct. Type-aware via `_boolean_columns_for`, so
|
|
390
|
+
genuine integer columns (`Priority = 1`) are never touched."""
|
|
391
|
+
bool_cols = _boolean_columns_for(stmt)
|
|
392
|
+
if not bool_cols:
|
|
393
|
+
return stmt
|
|
394
|
+
|
|
395
|
+
def fix(node: exp.Expression) -> exp.Expression:
|
|
396
|
+
# Equality / inequality, either operand order: `col = 1`, `0 <> col`, `col != 1`.
|
|
397
|
+
if isinstance(node, (exp.EQ, exp.NEQ)):
|
|
398
|
+
left, right = node.this, node.expression
|
|
399
|
+
if _is_bool_col(left, bool_cols):
|
|
400
|
+
lit = _bool_literal(right)
|
|
401
|
+
if lit is not None:
|
|
402
|
+
return node.__class__(this=left, expression=lit)
|
|
403
|
+
elif _is_bool_col(right, bool_cols):
|
|
404
|
+
lit = _bool_literal(left)
|
|
405
|
+
if lit is not None:
|
|
406
|
+
return node.__class__(this=lit, expression=right)
|
|
407
|
+
return node
|
|
408
|
+
# Set membership: `col IN (0, 1)` → `col IN (FALSE, TRUE)` (coerce only the 0/1 elems).
|
|
409
|
+
if isinstance(node, exp.In) and _is_bool_col(node.this, bool_cols):
|
|
410
|
+
exprs = node.args.get("expressions")
|
|
411
|
+
if exprs:
|
|
412
|
+
node.set("expressions", [(_bool_literal(e) or e) for e in exprs])
|
|
413
|
+
return node
|
|
414
|
+
return node
|
|
415
|
+
|
|
416
|
+
return stmt.transform(fix)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _residual_boolean_int(stmt: exp.Expression) -> list[str]:
|
|
420
|
+
"""Self-check: after coercion + the full rewrite pipeline, find any surviving
|
|
421
|
+
`<bitcol> <op> 0/1` comparison/membership the coercion pass did not eliminate (e.g. an
|
|
422
|
+
operator/shape it doesn't model — CASE switch, BETWEEN). These are exactly the
|
|
423
|
+
`boolean = integer` errors that abort a PG migration, so they are surfaced as conversion
|
|
424
|
+
gaps rather than emitted silently — making the converter's gap count behavioral, not just
|
|
425
|
+
syntactic. Returns SQL snippets of each offending node."""
|
|
426
|
+
bool_cols = _boolean_columns_for(stmt)
|
|
427
|
+
if not bool_cols:
|
|
428
|
+
return []
|
|
429
|
+
hits: list[str] = []
|
|
430
|
+
for node in stmt.walk():
|
|
431
|
+
# Comparisons + membership the coercion pass targets (defence-in-depth — should be
|
|
432
|
+
# empty after coercion, but proves it).
|
|
433
|
+
if isinstance(node, (exp.EQ, exp.NEQ)):
|
|
434
|
+
if (_is_bool_col(node.this, bool_cols) and _bool_literal(node.expression) is not None) or (
|
|
435
|
+
_is_bool_col(node.expression, bool_cols) and _bool_literal(node.this) is not None
|
|
436
|
+
):
|
|
437
|
+
hits.append(node.sql(dialect=MJPostgres))
|
|
438
|
+
elif isinstance(node, exp.In) and _is_bool_col(node.this, bool_cols):
|
|
439
|
+
if any(_bool_literal(e) is not None for e in (node.args.get("expressions") or [])):
|
|
440
|
+
hits.append(node.sql(dialect=MJPostgres))
|
|
441
|
+
# Shapes the coercion pass intentionally does NOT model — surface them so they are
|
|
442
|
+
# hand-resolved instead of silently emitted as `boolean = integer`:
|
|
443
|
+
# CASE switch: `CASE bitcol WHEN 1 THEN …` (implicit equality, no EQ node)
|
|
444
|
+
elif isinstance(node, exp.Case) and node.this is not None and _is_bool_col(node.this, bool_cols):
|
|
445
|
+
if any(_bool_literal(w.this) is not None for w in node.args.get("ifs", [])):
|
|
446
|
+
hits.append(node.sql(dialect=MJPostgres))
|
|
447
|
+
# BETWEEN: `bitcol BETWEEN 0 AND 1`
|
|
448
|
+
elif isinstance(node, exp.Between) and _is_bool_col(node.this, bool_cols):
|
|
449
|
+
if _bool_literal(node.args.get("low")) is not None or _bool_literal(node.args.get("high")) is not None:
|
|
450
|
+
hits.append(node.sql(dialect=MJPostgres))
|
|
451
|
+
return hits
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _strip_national(node: exp.Expression) -> exp.Expression:
|
|
455
|
+
"""N'...' (exp.National) → plain string literal; PG has no N-prefixed literals."""
|
|
456
|
+
if isinstance(node, exp.National):
|
|
457
|
+
return exp.Literal.string(node.name)
|
|
458
|
+
return node
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _strip_nulls_ordering(node: exp.Expression) -> exp.Expression:
|
|
462
|
+
"""sqlglot adds NULLS FIRST to ordered PK/index columns; drop it to match PG defaults."""
|
|
463
|
+
if isinstance(node, exp.Ordered) and node.args.get("nulls_first") is not None:
|
|
464
|
+
node.set("nulls_first", None)
|
|
465
|
+
return node
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _strip_collate(node: exp.Expression) -> exp.Expression:
|
|
469
|
+
"""Drop SS collations on COLUMN DEFINITIONS (`CREATE TABLE c … COLLATE x`) — PG doesn't have
|
|
470
|
+
them and a column's storage collation carries no comparison semantics that must be preserved.
|
|
471
|
+
|
|
472
|
+
EXPRESSION collations (`c COLLATE Latin1_General_BIN2 <> 'y'`) are deliberately NOT handled
|
|
473
|
+
here: they change comparison semantics (a `_BIN2`/`_CS_` qualifier forces case-sensitive
|
|
474
|
+
matching that the surrounding default-collation `=` does not), so naively dropping them silently
|
|
475
|
+
breaks the statement. Those are reported as unhandled upstream so the migration falls back to a
|
|
476
|
+
hand-authored PG form."""
|
|
477
|
+
if isinstance(node, exp.ColumnDef):
|
|
478
|
+
kept = [
|
|
479
|
+
c for c in node.args.get("constraints", [])
|
|
480
|
+
if not (isinstance(c, exp.ColumnConstraint) and isinstance(c.kind, exp.CollateColumnConstraint))
|
|
481
|
+
]
|
|
482
|
+
node.set("constraints", kept)
|
|
483
|
+
return node
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _drop_isjson_checks(node: exp.Expression) -> exp.Expression:
|
|
487
|
+
"""Drop CHECK constraints using ISJSON() — PG has no ISJSON (validity enforced elsewhere).
|
|
488
|
+
|
|
489
|
+
Handles both column-level (a constraint on a ColumnDef) and table-level
|
|
490
|
+
(`CONSTRAINT ck CHECK (ISJSON(...))` in a CREATE, or `ADD CONSTRAINT ... CHECK
|
|
491
|
+
(ISJSON(...))` in an ALTER). Table-level forms are removed outright (return None);
|
|
492
|
+
an ALTER left with no actions is dropped downstream by the empty-ALTER guard.
|
|
493
|
+
"""
|
|
494
|
+
if isinstance(node, exp.ColumnDef):
|
|
495
|
+
kept = [
|
|
496
|
+
c for c in node.args.get("constraints", [])
|
|
497
|
+
if not (
|
|
498
|
+
isinstance(c, exp.ColumnConstraint)
|
|
499
|
+
and isinstance(c.kind, exp.CheckColumnConstraint)
|
|
500
|
+
and "ISJSON" in c.kind.sql(dialect="tsql").upper()
|
|
501
|
+
)
|
|
502
|
+
]
|
|
503
|
+
node.set("constraints", kept)
|
|
504
|
+
return node
|
|
505
|
+
if isinstance(node, (exp.Constraint, exp.AddConstraint)) and "ISJSON" in node.sql(dialect="tsql").upper():
|
|
506
|
+
return None
|
|
507
|
+
return node
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _fold_clustered_constraints(node: exp.Expression) -> exp.Expression:
|
|
511
|
+
"""`PRIMARY KEY CLUSTERED (cols)` / `UNIQUE NONCLUSTERED (cols)` → PG `PRIMARY KEY (cols)` / `UNIQUE (cols)`.
|
|
512
|
+
|
|
513
|
+
SQL Server's CLUSTERED/NONCLUSTERED qualifier parses into a sibling
|
|
514
|
+
Clustered/NonClusteredColumnConstraint that holds the columns; PG has no such
|
|
515
|
+
qualifier, so fold the columns into the PK/UNIQUE and drop the qualifier.
|
|
516
|
+
"""
|
|
517
|
+
def cols_of(clustered):
|
|
518
|
+
return [o.this if isinstance(o, exp.Ordered) else o for o in (clustered.this or [])]
|
|
519
|
+
|
|
520
|
+
# PK CLUSTERED: PrimaryKeyColumnConstraint + ClusteredColumnConstraint are siblings.
|
|
521
|
+
if isinstance(node, exp.Constraint):
|
|
522
|
+
exprs = node.args.get("expressions") or []
|
|
523
|
+
clustered = next((e for e in exprs if isinstance(e, exp.ClusteredColumnConstraint)), None)
|
|
524
|
+
if clustered is not None and any(isinstance(e, exp.PrimaryKeyColumnConstraint) for e in exprs):
|
|
525
|
+
node.set("expressions", [exp.PrimaryKey(expressions=cols_of(clustered))])
|
|
526
|
+
return node
|
|
527
|
+
|
|
528
|
+
# UNIQUE NONCLUSTERED: UniqueColumnConstraint wraps a NonClusteredColumnConstraint holding the cols.
|
|
529
|
+
if isinstance(node, exp.UniqueColumnConstraint) and isinstance(node.this, exp.NonClusteredColumnConstraint):
|
|
530
|
+
node.set("this", exp.Schema(expressions=cols_of(node.this)))
|
|
531
|
+
return node
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _column_fk_to_reference(node: exp.Expression) -> exp.Expression:
|
|
535
|
+
"""Column-level `CONSTRAINT fk FOREIGN KEY REFERENCES t(c)` → bare `REFERENCES t(c)`.
|
|
536
|
+
|
|
537
|
+
T-SQL allows the `FOREIGN KEY` keyword on an inline (column) FK constraint; PG does
|
|
538
|
+
not — a column constraint is just `REFERENCES`. sqlglot parses this into a
|
|
539
|
+
ColumnConstraint whose kind is ForeignKey(reference=Reference(...)) and the Postgres
|
|
540
|
+
generator keeps the `FOREIGN KEY` token, producing `syntax error at or near "FOREIGN"`
|
|
541
|
+
that aborts the whole `ALTER TABLE ... ADD` (losing every column in the statement).
|
|
542
|
+
Replace the ForeignKey kind with its inner Reference so PG sees a valid column FK.
|
|
543
|
+
"""
|
|
544
|
+
if isinstance(node, exp.ColumnConstraint) and isinstance(node.kind, exp.ForeignKey):
|
|
545
|
+
ref = node.kind.args.get("reference")
|
|
546
|
+
if ref is not None:
|
|
547
|
+
node.set("kind", ref)
|
|
548
|
+
return node
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _fix_misparsed_table_constraint(node: exp.Expression) -> exp.Expression:
|
|
552
|
+
"""A table-level `CONSTRAINT name CHECK(...)` listed inside a T-SQL multi-item
|
|
553
|
+
`ALTER TABLE ADD col1, col2, CONSTRAINT ... CHECK(...)` is mis-parsed by sqlglot
|
|
554
|
+
as a column named "CONSTRAINT" with a user-defined type (the constraint name),
|
|
555
|
+
e.g. `ADD COLUMN "CONSTRAINT" CK_x CHECK(...)` → PG `type "ck_x" does not exist`,
|
|
556
|
+
aborting the whole ADD. Rebuild it as a proper `ADD CONSTRAINT name <kind>`.
|
|
557
|
+
"""
|
|
558
|
+
if (isinstance(node, exp.ColumnDef)
|
|
559
|
+
and isinstance(node.this, exp.Identifier)
|
|
560
|
+
and node.this.name.upper() == "CONSTRAINT"
|
|
561
|
+
and node.kind is not None
|
|
562
|
+
and node.kind.this == exp.DataType.Type.USERDEFINED):
|
|
563
|
+
cname = node.kind.args.get("kind")
|
|
564
|
+
name_ident = exp.to_identifier(cname.name if isinstance(cname, exp.Expression) else str(cname))
|
|
565
|
+
kinds = [c.kind for c in node.constraints if isinstance(c, exp.ColumnConstraint) and c.kind is not None]
|
|
566
|
+
if kinds:
|
|
567
|
+
return exp.AddConstraint(expressions=[exp.Constraint(this=name_ident, expressions=kinds)])
|
|
568
|
+
return node
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _rewrite_boolean_defaults(node: exp.Expression) -> exp.Expression:
|
|
572
|
+
"""A BIT column defaulting to 1/0 becomes a BOOLEAN defaulting to TRUE/FALSE.
|
|
573
|
+
|
|
574
|
+
SQL Server wraps defaults in parens (`DEFAULT ((1))`), so unwrap before checking.
|
|
575
|
+
"""
|
|
576
|
+
if isinstance(node, exp.ColumnDef) and node.kind and node.kind.this == exp.DataType.Type.BIT:
|
|
577
|
+
for constraint in node.constraints:
|
|
578
|
+
ck = constraint.kind
|
|
579
|
+
if not isinstance(ck, exp.DefaultColumnConstraint):
|
|
580
|
+
continue
|
|
581
|
+
val = ck.this
|
|
582
|
+
while isinstance(val, exp.Paren):
|
|
583
|
+
val = val.this
|
|
584
|
+
if isinstance(val, exp.Literal) and not val.is_string and val.name in ("0", "1"):
|
|
585
|
+
ck.set("this", exp.true() if val.name == "1" else exp.false())
|
|
586
|
+
return node
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
import re as _re
|
|
590
|
+
|
|
591
|
+
# `GO` is a batch separator (SSMS/sqlcmd tooling), not SQL — split on it before parsing.
|
|
592
|
+
_GO_SPLIT = _re.compile(r"(?im)^\s*GO\s*;?\s*$")
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _first_keyword(text: str) -> str:
|
|
596
|
+
"""A short label for an unhandled passthrough statement (for reporting)."""
|
|
597
|
+
m = _re.search(r"[A-Za-z_][A-Za-z0-9_]*", text)
|
|
598
|
+
token = m.group(0).upper() if m else "?"
|
|
599
|
+
low = text.lower()
|
|
600
|
+
if "sp_addextendedproperty" in low:
|
|
601
|
+
return "sp_addextendedproperty"
|
|
602
|
+
if token in ("IF", "BEGIN", "DECLARE", "PRINT", "EXEC", "EXECUTE", "WHILE"):
|
|
603
|
+
return token
|
|
604
|
+
return token
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
# --- Fixed-shape T-SQL envelopes that sqlglot cannot parse -------------------
|
|
608
|
+
# These are not arbitrary procedural SQL — they are CodeGen/hand templates with a
|
|
609
|
+
# fixed shape. We recognize the envelope structurally and transpile the real SQL
|
|
610
|
+
# *inside* it (predicates, INSERT bodies, descriptions) through the AST dialect.
|
|
611
|
+
|
|
612
|
+
# CodeGen object naming convention (mirrors MigrationStatementSplitter.CODEGEN_NAME):
|
|
613
|
+
# views, the CRUD/recompile sprocs, fn* functions, trg* triggers — all regenerated by
|
|
614
|
+
# `mj codegen`, so their extended-property comments must be skipped (object is dropped).
|
|
615
|
+
_CODEGEN_OBJECT_NAME = _re.compile(r"^(spCreate|spUpdate|spDelete|spRecompile|vw|fn|trgUpdate|trgCreate|trgDelete|trg)", _re.IGNORECASE)
|
|
616
|
+
|
|
617
|
+
# EXEC sp_addextendedproperty @name=N'MS_Description', @value=N'...', ... ;
|
|
618
|
+
# The terminating `;` must be the one OUTSIDE the quoted args — description @values
|
|
619
|
+
# routinely contain semicolons ("applies to all apps; when set, …"). A naive `.*?;`
|
|
620
|
+
# cuts at the first in-string semicolon, corrupting this and every later envelope
|
|
621
|
+
# boundary. Match args as a run of (whole single-quoted string | non-quote/non-`;` char).
|
|
622
|
+
# The terminator itself is optional when the EXEC is the last statement of its chunk
|
|
623
|
+
# (end-of-input, a GO separator, or the END of a surrounding BEGIN block) — T-SQL does
|
|
624
|
+
# not require it, and a missing `;` must not skip the comment. Args are lazy so the
|
|
625
|
+
# match stops at the first such boundary rather than swallowing following statements.
|
|
626
|
+
_SP_EXTPROP = _re.compile(
|
|
627
|
+
r"EXEC(?:UTE)?\s+sp_addextendedproperty\b(?P<args>(?:'(?:[^']|'')*'|[^';])*?)"
|
|
628
|
+
r"(?:;|(?=\s*(?:\Z|GO\b|END\b)))",
|
|
629
|
+
_re.IGNORECASE | _re.DOTALL,
|
|
630
|
+
)
|
|
631
|
+
# IF [NOT] EXISTS (<select>) BEGIN <body> END — the idempotency wrapper.
|
|
632
|
+
# Recognized by a scanner, not a regex: the body must terminate at the END that matches
|
|
633
|
+
# the block's BEGIN, counting BEGIN/CASE nesting and skipping strings/comments/bracketed
|
|
634
|
+
# identifiers — a regex stopping at any `\bEND\b` truncates on `CASE … END` (or the word
|
|
635
|
+
# END inside a literal) and the leftover tail corrupts neighboring statements.
|
|
636
|
+
_IF_EXISTS_HEAD = _re.compile(r"\bIF\s+(?P<neg>NOT\s+)?EXISTS\s*\(", _re.IGNORECASE)
|
|
637
|
+
_WORD = _re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def _scan_atom(s: str, i: int) -> int:
|
|
641
|
+
"""If s[i] opens a string / comment / bracketed or quoted identifier, return the index
|
|
642
|
+
just past it; otherwise return i unchanged (plain code character)."""
|
|
643
|
+
c = s[i]
|
|
644
|
+
if c == "'":
|
|
645
|
+
i += 1
|
|
646
|
+
n = len(s)
|
|
647
|
+
while i < n:
|
|
648
|
+
if s[i] == "'":
|
|
649
|
+
if i + 1 < n and s[i + 1] == "'": # escaped '' inside string
|
|
650
|
+
i += 2
|
|
651
|
+
continue
|
|
652
|
+
return i + 1
|
|
653
|
+
i += 1
|
|
654
|
+
return n
|
|
655
|
+
if c == "-" and s.startswith("--", i):
|
|
656
|
+
j = s.find("\n", i)
|
|
657
|
+
return len(s) if j < 0 else j + 1
|
|
658
|
+
if c == "/" and s.startswith("/*", i):
|
|
659
|
+
j = s.find("*/", i)
|
|
660
|
+
return len(s) if j < 0 else j + 2
|
|
661
|
+
if c == "[":
|
|
662
|
+
j = s.find("]", i)
|
|
663
|
+
return len(s) if j < 0 else j + 1
|
|
664
|
+
if c == '"':
|
|
665
|
+
j = s.find('"', i + 1)
|
|
666
|
+
return len(s) if j < 0 else j + 1
|
|
667
|
+
return i
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _match_paren(s: str, i: int) -> int:
|
|
671
|
+
"""s[i] == '(' → index just past the matching ')' (atom-aware), or -1 if unbalanced."""
|
|
672
|
+
depth = 0
|
|
673
|
+
n = len(s)
|
|
674
|
+
while i < n:
|
|
675
|
+
j = _scan_atom(s, i)
|
|
676
|
+
if j != i:
|
|
677
|
+
i = j
|
|
678
|
+
continue
|
|
679
|
+
if s[i] == "(":
|
|
680
|
+
depth += 1
|
|
681
|
+
elif s[i] == ")":
|
|
682
|
+
depth -= 1
|
|
683
|
+
if depth == 0:
|
|
684
|
+
return i + 1
|
|
685
|
+
i += 1
|
|
686
|
+
return -1
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _peek_word(s: str, i: int) -> str:
|
|
690
|
+
"""Next word token at/after i (skipping whitespace/comments), uppercased; '' if none."""
|
|
691
|
+
n = len(s)
|
|
692
|
+
while i < n:
|
|
693
|
+
j = _scan_atom(s, i)
|
|
694
|
+
if j != i:
|
|
695
|
+
i = j
|
|
696
|
+
continue
|
|
697
|
+
if s[i].isspace():
|
|
698
|
+
i += 1
|
|
699
|
+
continue
|
|
700
|
+
m = _WORD.match(s, i)
|
|
701
|
+
return m.group(0).upper() if m else ""
|
|
702
|
+
return ""
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _match_block_end(s: str, i: int) -> tuple[int, int]:
|
|
706
|
+
"""From just after a block's BEGIN keyword, return (body_end, block_end): the start of
|
|
707
|
+
the matching END token and the index just past it (plus an optional trailing `;`).
|
|
708
|
+
BEGIN…END and CASE…END nest; BEGIN TRAN[SACTION] pairs with COMMIT, not END, so it
|
|
709
|
+
does not count. Returns (-1, -1) when the block never terminates."""
|
|
710
|
+
depth = 1
|
|
711
|
+
n = len(s)
|
|
712
|
+
while i < n:
|
|
713
|
+
j = _scan_atom(s, i)
|
|
714
|
+
if j != i:
|
|
715
|
+
i = j
|
|
716
|
+
continue
|
|
717
|
+
m = _WORD.match(s, i)
|
|
718
|
+
if not m:
|
|
719
|
+
i += 1
|
|
720
|
+
continue
|
|
721
|
+
word = m.group(0).upper()
|
|
722
|
+
if word == "BEGIN" and _peek_word(s, m.end()) not in ("TRAN", "TRANSACTION"):
|
|
723
|
+
depth += 1
|
|
724
|
+
elif word == "CASE":
|
|
725
|
+
depth += 1
|
|
726
|
+
elif word == "END":
|
|
727
|
+
depth -= 1
|
|
728
|
+
if depth == 0:
|
|
729
|
+
end = m.end()
|
|
730
|
+
t = _re.compile(r"\s*;").match(s, end)
|
|
731
|
+
return i, (t.end() if t else end)
|
|
732
|
+
i = m.end()
|
|
733
|
+
return -1, -1
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
class _IfExistsMatch:
|
|
737
|
+
"""Minimal re.Match-alike for _find_if_exists_begin results — start/end/group are
|
|
738
|
+
the only members the batch walk and _transpile_if_exists_begin use."""
|
|
739
|
+
__slots__ = ("_text", "_start", "_end", "_groups")
|
|
740
|
+
|
|
741
|
+
def __init__(self, text: str, start: int, end: int, neg: str | None, cond: str, body: str):
|
|
742
|
+
self._text, self._start, self._end = text, start, end
|
|
743
|
+
self._groups = {"neg": neg, "cond": cond, "body": body}
|
|
744
|
+
|
|
745
|
+
def start(self) -> int:
|
|
746
|
+
return self._start
|
|
747
|
+
|
|
748
|
+
def end(self) -> int:
|
|
749
|
+
return self._end
|
|
750
|
+
|
|
751
|
+
def group(self, key: str | int = 0) -> str | None:
|
|
752
|
+
return self._text[self._start:self._end] if key == 0 else self._groups[key]
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _find_if_exists_begin(text: str, pos: int = 0) -> _IfExistsMatch | None:
|
|
756
|
+
"""Find the next `IF [NOT] EXISTS (<select>) BEGIN <body> END` block at/after pos."""
|
|
757
|
+
for head in _IF_EXISTS_HEAD.finditer(text, pos):
|
|
758
|
+
cond_close = _match_paren(text, head.end() - 1)
|
|
759
|
+
if cond_close < 0:
|
|
760
|
+
continue
|
|
761
|
+
cond = text[head.end():cond_close - 1]
|
|
762
|
+
if not _re.match(r"\s*SELECT\b", _strip_leading_sql_comments(cond), _re.IGNORECASE):
|
|
763
|
+
continue
|
|
764
|
+
# Expect the block's BEGIN next (skipping whitespace/comments).
|
|
765
|
+
i, n = cond_close, len(text)
|
|
766
|
+
while i < n:
|
|
767
|
+
if text[i].isspace():
|
|
768
|
+
i += 1
|
|
769
|
+
continue
|
|
770
|
+
j = _scan_atom(text, i)
|
|
771
|
+
if j != i and text[i] in ("-", "/"): # comments only; anything else breaks the shape
|
|
772
|
+
i = j
|
|
773
|
+
continue
|
|
774
|
+
break
|
|
775
|
+
m = _WORD.match(text, i)
|
|
776
|
+
if not m or m.group(0).upper() != "BEGIN":
|
|
777
|
+
continue
|
|
778
|
+
body_end, block_end = _match_block_end(text, m.end())
|
|
779
|
+
if body_end < 0:
|
|
780
|
+
continue
|
|
781
|
+
return _IfExistsMatch(text, head.start(), block_end, head.group("neg"), cond, text[m.end():body_end])
|
|
782
|
+
return None
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _unquote_tsql_string(val: str) -> str:
|
|
786
|
+
"""N'foo''bar' / 'foo''bar' → foo'bar (T-SQL string literal to raw text)."""
|
|
787
|
+
val = val.strip()
|
|
788
|
+
if val[:1] in ("N", "n") and val[1:2] == "'":
|
|
789
|
+
val = val[1:]
|
|
790
|
+
if val.startswith("'") and val.endswith("'"):
|
|
791
|
+
val = val[1:-1]
|
|
792
|
+
return val.replace("''", "'")
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _pg_string(raw: str) -> str:
|
|
796
|
+
"""Raw text → PG single-quoted literal (escape embedded quotes)."""
|
|
797
|
+
return "'" + raw.replace("'", "''") + "'"
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _extprop_arg(args: str, name: str) -> str | None:
|
|
801
|
+
m = _re.search(r"@" + name + r"\s*=\s*(N?'(?:[^']|'')*')", args, _re.IGNORECASE)
|
|
802
|
+
return _unquote_tsql_string(m.group(1)) if m else None
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _extprop_positional(args: str) -> dict | None:
|
|
806
|
+
"""Parse positional sp_addextendedproperty args: name, value, l0type, l0name, l1type, l1name[, l2type, l2name]."""
|
|
807
|
+
vals = [_unquote_tsql_string(m.group(1)) for m in _re.finditer(r"(N?'(?:[^']|'')*')", args)]
|
|
808
|
+
if len(vals) < 6:
|
|
809
|
+
return None
|
|
810
|
+
d = {"name": vals[0], "value": vals[1], "level0name": vals[3], "level1name": vals[5]}
|
|
811
|
+
if len(vals) >= 8:
|
|
812
|
+
d["level2type"], d["level2name"] = vals[6], vals[7]
|
|
813
|
+
return d
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _transpile_sp_addextendedproperty(args: str) -> str | None:
|
|
817
|
+
"""EXEC sp_addextendedproperty(MS_Description) → COMMENT ON COLUMN/TABLE. Handles named and positional forms."""
|
|
818
|
+
# Named form (@name=N'…'); fall back to positional (N'MS_Description', N'…', …).
|
|
819
|
+
if _extprop_arg(args, "name") is not None:
|
|
820
|
+
get = lambda k: _extprop_arg(args, k) # noqa: E731
|
|
821
|
+
else:
|
|
822
|
+
p = _extprop_positional(args)
|
|
823
|
+
if p is None:
|
|
824
|
+
return None
|
|
825
|
+
get = lambda k: p.get(k) # noqa: E731
|
|
826
|
+
|
|
827
|
+
if (get("name") or "").upper() != "MS_DESCRIPTION":
|
|
828
|
+
return None
|
|
829
|
+
value = get("value")
|
|
830
|
+
schema = get("level0name") or FLYWAY_MACRO
|
|
831
|
+
table = get("level1name")
|
|
832
|
+
col = get("level2name")
|
|
833
|
+
col_type = (get("level2type") or ("COLUMN" if col else "")).upper()
|
|
834
|
+
if value is None or not table:
|
|
835
|
+
return None
|
|
836
|
+
# Comments on CodeGen objects (views vw*, sprocs spCreate/spUpdate/spDelete/spRecompile,
|
|
837
|
+
# functions fn*, triggers trg*) are regenerated by `mj codegen` — and the object doesn't
|
|
838
|
+
# exist at apply time (it's dropped), so a COMMENT ON would fail "relation does not exist".
|
|
839
|
+
# Skip them; CodeGen re-emits the object and its description.
|
|
840
|
+
if _CODEGEN_OBJECT_NAME.match(table):
|
|
841
|
+
return ""
|
|
842
|
+
if col and col_type == "COLUMN":
|
|
843
|
+
return f'COMMENT ON COLUMN {schema}."{table}"."{col}" IS {_pg_string(value)};'
|
|
844
|
+
return f'COMMENT ON TABLE {schema}."{table}" IS {_pg_string(value)};'
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
# T-SQL emits column defaults as standalone constraint statements that sqlglot
|
|
848
|
+
# CANNOT parse (it falls back to an opaque exp.Command and the default is lost):
|
|
849
|
+
# ALTER TABLE [t] ADD CONSTRAINT [DF_…] DEFAULT (<expr>) FOR [col];
|
|
850
|
+
# These carry ~all of MJ's column defaults (969 newsequentialid() ID defaults alone),
|
|
851
|
+
# so dropping them leaves the PG schema without them and breaks seed INSERTs that omit
|
|
852
|
+
# a defaulted column (e.g. ApplicationEntity rows relying on the ID default). Recognize
|
|
853
|
+
# the fixed shape on raw text and emit the PG form: `ALTER TABLE t ALTER COLUMN "col"
|
|
854
|
+
# SET DEFAULT <expr>` — the same "structural envelope" approach used for
|
|
855
|
+
# sp_addextendedproperty / IF EXISTS BEGIN. The trailing `;` is optional because
|
|
856
|
+
# sqlglot's Command node text drops it.
|
|
857
|
+
_DEFAULT_CONSTRAINT = _re.compile(
|
|
858
|
+
r"^\s*ALTER\s+TABLE\s+(?P<tbl>.+?)\s+ADD\s+CONSTRAINT\s+(?:\[[^\]]+\]|\"[^\"]+\"|\S+)\s+"
|
|
859
|
+
r"DEFAULT\s+(?P<expr>.+?)\s+FOR\s+(?P<col>\[[^\]]+\]|\"[^\"]+\"|\w+)\s*;?\s*$",
|
|
860
|
+
_re.IGNORECASE | _re.DOTALL,
|
|
861
|
+
)
|
|
862
|
+
# Identifier tokens inside a table reference: bracketed, double-quoted, the Flyway
|
|
863
|
+
# sentinel/macro, or a bare PascalCase name.
|
|
864
|
+
_IDENT_TOKEN = _re.compile(
|
|
865
|
+
r'\[[^\]]+\]|"[^"]+"|' + _re.escape(FLYWAY_SENTINEL) + r'|' + _re.escape(FLYWAY_MACRO) + r'|[A-Za-z_]\w*'
|
|
866
|
+
)
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def _ident_token_to_pg(tok: str) -> str:
|
|
870
|
+
"""One identifier token → PG form. `[x]`/`"x"`/`x` → `"x"`; Flyway sentinel/macro → macro verbatim."""
|
|
871
|
+
name = tok.strip()
|
|
872
|
+
if name.startswith("[") and name.endswith("]"):
|
|
873
|
+
name = name[1:-1]
|
|
874
|
+
elif name.startswith('"') and name.endswith('"'):
|
|
875
|
+
name = name[1:-1]
|
|
876
|
+
if name in (FLYWAY_SENTINEL, FLYWAY_MACRO):
|
|
877
|
+
return FLYWAY_MACRO
|
|
878
|
+
return f'"{name}"'
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _emit_table_ref(tbl_raw: str) -> str:
|
|
882
|
+
"""`[__mj].[X]` → `"__mj"."X"`; `${flyway:defaultSchema}.[X]` → `${flyway:defaultSchema}."X"`."""
|
|
883
|
+
toks = _IDENT_TOKEN.findall(tbl_raw)
|
|
884
|
+
return ".".join(_ident_token_to_pg(t) for t in toks) if toks else tbl_raw.strip()
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def _balanced_parens(s: str) -> bool:
|
|
888
|
+
"""True if every paren in s closes before the end (i.e. the whole string is one group)."""
|
|
889
|
+
depth = 0
|
|
890
|
+
for i, c in enumerate(s):
|
|
891
|
+
if c == "(":
|
|
892
|
+
depth += 1
|
|
893
|
+
elif c == ")":
|
|
894
|
+
depth -= 1
|
|
895
|
+
if depth == 0 and i != len(s) - 1:
|
|
896
|
+
return False
|
|
897
|
+
return depth == 0
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _convert_default_expr(expr_raw: str, table: str, col: str) -> str:
|
|
901
|
+
"""Convert a T-SQL DEFAULT expression to PG. Unwraps SS's `((…))` wrapping, maps
|
|
902
|
+
BIT 1/0 → TRUE/FALSE (via the file-level bit registry), and routes the rest through
|
|
903
|
+
the AST (newsequentialid()→gen_random_uuid(), getutcdate()→now(), N'…'→'…')."""
|
|
904
|
+
e = expr_raw.strip().rstrip(";").strip()
|
|
905
|
+
while len(e) >= 2 and e[0] == "(" and e[-1] == ")" and _balanced_parens(e):
|
|
906
|
+
e = e[1:-1].strip()
|
|
907
|
+
if table and (table.lower(), col.lower()) in _BIT_COLS and e in ("0", "1"):
|
|
908
|
+
return "TRUE" if e == "1" else "FALSE"
|
|
909
|
+
try:
|
|
910
|
+
node = sqlglot.parse_one(e.replace(FLYWAY_MACRO, FLYWAY_SENTINEL), read="tsql")
|
|
911
|
+
node = node.transform(_strip_national).transform(_rewrite_functions)
|
|
912
|
+
return node.sql(dialect=MJPostgres, identify=True)
|
|
913
|
+
except Exception: # noqa: BLE001 — fall back to the raw expr (still valid for simple literals)
|
|
914
|
+
return e
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def _strip_leading_sql_comments(s: str) -> str:
|
|
918
|
+
"""Drop leading `--` / `/* */` comments. sqlglot attaches a preceding comment to the
|
|
919
|
+
statement node, so a Command's text can be `/* note */ ALTER …` — which defeats a
|
|
920
|
+
`^ALTER` anchor. The comment is non-essential metadata; drop it for matching/emit."""
|
|
921
|
+
s = s.strip()
|
|
922
|
+
while True:
|
|
923
|
+
if s.startswith("--"):
|
|
924
|
+
nl = s.find("\n")
|
|
925
|
+
s = ("" if nl < 0 else s[nl + 1:]).lstrip()
|
|
926
|
+
elif s.startswith("/*"):
|
|
927
|
+
end = s.find("*/")
|
|
928
|
+
s = ("" if end < 0 else s[end + 2:]).lstrip()
|
|
929
|
+
else:
|
|
930
|
+
return s
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def _transpile_default_constraint(txt: str) -> str | None:
|
|
934
|
+
"""`ALTER TABLE t ADD CONSTRAINT df DEFAULT (expr) FOR [col]` → PG `ALTER COLUMN SET DEFAULT`.
|
|
935
|
+
Returns None if txt isn't a standalone default constraint."""
|
|
936
|
+
m = _DEFAULT_CONSTRAINT.match(txt.strip())
|
|
937
|
+
if not m:
|
|
938
|
+
return None
|
|
939
|
+
col = m.group("col").strip().strip('[]"')
|
|
940
|
+
toks = _IDENT_TOKEN.findall(m.group("tbl"))
|
|
941
|
+
tname = (toks[-1].strip('[]"') if toks else "")
|
|
942
|
+
expr = _convert_default_expr(m.group("expr"), tname, col)
|
|
943
|
+
return f'ALTER TABLE {_emit_table_ref(m.group("tbl"))} ALTER COLUMN "{col}" SET DEFAULT {expr}'
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
# T-SQL `ALTER TABLE t ALTER COLUMN c <type> [NOT NULL|NULL]` — also unparseable by
|
|
947
|
+
# sqlglot (→ opaque Command). PG splits this into a TYPE change and a separate
|
|
948
|
+
# nullability action: `ALTER COLUMN c TYPE <pgtype>, ALTER COLUMN c SET/DROP NOT NULL`.
|
|
949
|
+
# Dropped silently today, so a post-baseline column type/nullability change would
|
|
950
|
+
# no-op; emit the PG form (reusing the dialect's type mapping).
|
|
951
|
+
_ALTER_COLUMN = _re.compile(
|
|
952
|
+
r"^\s*ALTER\s+TABLE\s+(?P<tbl>.+?)\s+ALTER\s+COLUMN\s+(?P<col>\[[^\]]+\]|\"[^\"]+\"|\w+)\s+(?P<rest>.+?)\s*;?\s*$",
|
|
953
|
+
_re.IGNORECASE | _re.DOTALL,
|
|
954
|
+
)
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _transpile_alter_column(txt: str) -> str | None:
|
|
958
|
+
"""`ALTER TABLE t ALTER COLUMN c <type> [NOT NULL|NULL]` → PG TYPE change + nullability.
|
|
959
|
+
Returns None if txt isn't an ALTER COLUMN (or its type can't be parsed)."""
|
|
960
|
+
m = _ALTER_COLUMN.match(txt.strip())
|
|
961
|
+
if not m:
|
|
962
|
+
return None
|
|
963
|
+
rest = m.group("rest").strip()
|
|
964
|
+
try:
|
|
965
|
+
node = sqlglot.parse_one(f"CREATE TABLE t (c {rest})", read="tsql")
|
|
966
|
+
cd = node.find(exp.ColumnDef)
|
|
967
|
+
if cd is None or cd.kind is None:
|
|
968
|
+
return None
|
|
969
|
+
pgtype = cd.kind.sql(dialect=MJPostgres, identify=True)
|
|
970
|
+
except Exception: # noqa: BLE001
|
|
971
|
+
return None
|
|
972
|
+
col = m.group("col").strip().strip('[]"')
|
|
973
|
+
tbl_ref = _emit_table_ref(m.group("tbl"))
|
|
974
|
+
# T-SQL: omitting the NULL/NOT NULL spec on ALTER COLUMN makes the column NULLable.
|
|
975
|
+
if _re.search(r"\bNOT\s+NULL\b", rest, _re.IGNORECASE):
|
|
976
|
+
nullability = f'ALTER COLUMN "{col}" SET NOT NULL'
|
|
977
|
+
else:
|
|
978
|
+
nullability = f'ALTER COLUMN "{col}" DROP NOT NULL'
|
|
979
|
+
# Emit the TYPE change for every type — atomic ones included (an INT→BIGINT widening
|
|
980
|
+
# must not vanish) — plus the nullability action. PG rejects an `ALTER … TYPE` (even a
|
|
981
|
+
# no-op restate, which SS produces on every ALTER COLUMN) while a view depends on the
|
|
982
|
+
# column, so drop dependent CodeGen views first (regenerated by `mj codegen`).
|
|
983
|
+
actions = [f'ALTER COLUMN "{col}" TYPE {pgtype}', nullability]
|
|
984
|
+
bare_tbl = m.group("tbl").strip().split(".")[-1].strip().strip('[]"')
|
|
985
|
+
return _drop_dependent_views_block(bare_tbl, col) + "\n" + f'ALTER TABLE {tbl_ref} ' + ", ".join(actions)
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _drop_dependent_views_block(table: str, col: str) -> str:
|
|
989
|
+
"""DO block that drops every view depending on <table>.<col> (CodeGen regenerates them).
|
|
990
|
+
Deterministic and a no-op when nothing depends on the column."""
|
|
991
|
+
tbl = table.replace("'", "''")
|
|
992
|
+
cn = col.replace("'", "''")
|
|
993
|
+
return (
|
|
994
|
+
"DO $$\nDECLARE r RECORD;\nBEGIN\n"
|
|
995
|
+
" FOR r IN\n"
|
|
996
|
+
" SELECT DISTINCT ns.nspname AS sch, dv.relname AS vw\n"
|
|
997
|
+
" FROM pg_depend d\n"
|
|
998
|
+
" JOIN pg_rewrite rw ON rw.oid = d.objid\n"
|
|
999
|
+
" JOIN pg_class dv ON dv.oid = rw.ev_class AND dv.relkind = 'v'\n"
|
|
1000
|
+
" JOIN pg_namespace ns ON ns.oid = dv.relnamespace\n"
|
|
1001
|
+
" JOIN pg_class tc ON tc.oid = d.refobjid\n"
|
|
1002
|
+
" JOIN pg_attribute a ON a.attrelid = tc.oid AND a.attnum = d.refobjsubid\n"
|
|
1003
|
+
f" WHERE tc.relname = '{tbl}' AND a.attname = '{cn}'\n"
|
|
1004
|
+
" LOOP\n"
|
|
1005
|
+
" EXECUTE format('DROP VIEW IF EXISTS %I.%I CASCADE', r.sch, r.vw);\n"
|
|
1006
|
+
" END LOOP;\nEND $$;"
|
|
1007
|
+
)
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
# EXEC sp_dropextendedproperty @name=N'MS_Description', … @level1name=…, @level2name=… ;
|
|
1011
|
+
# Same envelope shape as sp_addextendedproperty but with NO @value (it removes a comment).
|
|
1012
|
+
# Map to `COMMENT ON … IS NULL`; dropped today, so 7 stale column comments survive.
|
|
1013
|
+
# Terminator semantics match _SP_EXTPROP (optional at chunk boundaries).
|
|
1014
|
+
_SP_DROPEXTPROP = _re.compile(
|
|
1015
|
+
r"EXEC(?:UTE)?\s+sp_dropextendedproperty\b(?P<args>(?:'(?:[^']|'')*'|[^';])*?)"
|
|
1016
|
+
r"(?:;|(?=\s*(?:\Z|GO\b|END\b)))",
|
|
1017
|
+
_re.IGNORECASE | _re.DOTALL,
|
|
1018
|
+
)
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
def _dropextprop_positional(args: str) -> dict | None:
|
|
1022
|
+
"""Positional sp_dropextendedproperty args: name, l0type, l0name, l1type, l1name[, l2type, l2name] (no value)."""
|
|
1023
|
+
vals = [_unquote_tsql_string(m.group(1)) for m in _re.finditer(r"(N?'(?:[^']|'')*')", args)]
|
|
1024
|
+
if len(vals) < 5:
|
|
1025
|
+
return None
|
|
1026
|
+
d = {"name": vals[0], "level0name": vals[2], "level1name": vals[4]}
|
|
1027
|
+
if len(vals) >= 7:
|
|
1028
|
+
d["level2type"], d["level2name"] = vals[5], vals[6]
|
|
1029
|
+
return d
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def _transpile_sp_dropextendedproperty(args: str) -> str | None:
|
|
1033
|
+
"""EXEC sp_dropextendedproperty(MS_Description) → COMMENT ON COLUMN/TABLE … IS NULL."""
|
|
1034
|
+
if _extprop_arg(args, "name") is not None:
|
|
1035
|
+
get = lambda k: _extprop_arg(args, k) # noqa: E731
|
|
1036
|
+
else:
|
|
1037
|
+
p = _dropextprop_positional(args)
|
|
1038
|
+
if p is None:
|
|
1039
|
+
return None
|
|
1040
|
+
get = lambda k: p.get(k) # noqa: E731
|
|
1041
|
+
if (get("name") or "").upper() != "MS_DESCRIPTION":
|
|
1042
|
+
return None
|
|
1043
|
+
schema = get("level0name") or FLYWAY_MACRO
|
|
1044
|
+
table = get("level1name")
|
|
1045
|
+
col = get("level2name")
|
|
1046
|
+
col_type = (get("level2type") or ("COLUMN" if col else "")).upper()
|
|
1047
|
+
if not table:
|
|
1048
|
+
return None
|
|
1049
|
+
if _CODEGEN_OBJECT_NAME.match(table): # CodeGen object — regenerated, skip
|
|
1050
|
+
return ""
|
|
1051
|
+
if col and col_type == "COLUMN":
|
|
1052
|
+
return f'COMMENT ON COLUMN {schema}."{table}"."{col}" IS NULL;'
|
|
1053
|
+
return f'COMMENT ON TABLE {schema}."{table}" IS NULL;'
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _split_top_level_statements(sql: str) -> list[str]:
|
|
1057
|
+
"""Split SQL on top-level `;`, ignoring semicolons inside single-quoted strings and
|
|
1058
|
+
`--` / `/* */` comments. Used to recover good statements when a whole-gap parse fails
|
|
1059
|
+
on one bad statement (a poison statement must not drop its neighbors)."""
|
|
1060
|
+
out: list[str] = []
|
|
1061
|
+
buf: list[str] = []
|
|
1062
|
+
i, n, in_str = 0, len(sql), False
|
|
1063
|
+
while i < n:
|
|
1064
|
+
c = sql[i]
|
|
1065
|
+
if in_str:
|
|
1066
|
+
buf.append(c)
|
|
1067
|
+
if c == "'":
|
|
1068
|
+
if i + 1 < n and sql[i + 1] == "'": # escaped '' inside string
|
|
1069
|
+
buf.append("'"); i += 2; continue
|
|
1070
|
+
in_str = False
|
|
1071
|
+
i += 1; continue
|
|
1072
|
+
if c == "'":
|
|
1073
|
+
in_str = True; buf.append(c); i += 1; continue
|
|
1074
|
+
if c == "-" and i + 1 < n and sql[i + 1] == "-": # line comment
|
|
1075
|
+
j = sql.find("\n", i); j = n if j < 0 else j
|
|
1076
|
+
buf.append(sql[i:j]); i = j; continue
|
|
1077
|
+
if c == "/" and i + 1 < n and sql[i + 1] == "*": # block comment
|
|
1078
|
+
j = sql.find("*/", i); j = n if j < 0 else j + 2
|
|
1079
|
+
buf.append(sql[i:j]); i = j; continue
|
|
1080
|
+
if c == ";":
|
|
1081
|
+
buf.append(";"); out.append("".join(buf)); buf = []; i += 1; continue
|
|
1082
|
+
buf.append(c); i += 1
|
|
1083
|
+
tail = "".join(buf)
|
|
1084
|
+
if tail.strip():
|
|
1085
|
+
out.append(tail)
|
|
1086
|
+
return out
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _parse_resilient(protected: str) -> list[tuple[exp.Expression | None, str]]:
|
|
1090
|
+
"""Parse a SQL chunk into (statement, raw_text) pairs. Fast path: one `sqlglot.parse`.
|
|
1091
|
+
On failure, fall back to per-statement parsing so a single unparseable statement only
|
|
1092
|
+
drops itself, not the valid DDL around it (returns (None, raw) for the failures)."""
|
|
1093
|
+
try:
|
|
1094
|
+
return [(s, "") for s in sqlglot.parse(protected, read="tsql") if s is not None]
|
|
1095
|
+
except Exception: # noqa: BLE001
|
|
1096
|
+
pass
|
|
1097
|
+
results: list[tuple[exp.Expression | None, str]] = []
|
|
1098
|
+
for piece in _split_top_level_statements(protected):
|
|
1099
|
+
if not piece.strip():
|
|
1100
|
+
continue
|
|
1101
|
+
try:
|
|
1102
|
+
for s in sqlglot.parse(piece, read="tsql"):
|
|
1103
|
+
if s is not None:
|
|
1104
|
+
results.append((s, piece))
|
|
1105
|
+
except Exception: # noqa: BLE001
|
|
1106
|
+
results.append((None, piece))
|
|
1107
|
+
return results
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def _transpile_plain(sql: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
1111
|
+
"""Transpile a chunk of regular SQL via the AST dialect; report unparseable bits."""
|
|
1112
|
+
out, unhandled = [], []
|
|
1113
|
+
protected = sql.replace(FLYWAY_MACRO, FLYWAY_SENTINEL)
|
|
1114
|
+
# Set after reporting a CREATE PROCEDURE/FUNCTION/TRIGGER: the routine's closing
|
|
1115
|
+
# `END` often parses as its own dangling statement — it belongs to the routine we
|
|
1116
|
+
# just reported, not to a new gap.
|
|
1117
|
+
swallow_routine_end = False
|
|
1118
|
+
for stmt, raw in _parse_resilient(protected):
|
|
1119
|
+
if swallow_routine_end:
|
|
1120
|
+
swallow_routine_end = False
|
|
1121
|
+
tail = (raw or (stmt.sql(dialect="tsql") if stmt is not None else "")).strip().rstrip(";").strip()
|
|
1122
|
+
if tail.upper() == "END":
|
|
1123
|
+
continue
|
|
1124
|
+
if stmt is None:
|
|
1125
|
+
unhandled.append({"kind": "parse-error", "snippet": raw.strip()[:80]})
|
|
1126
|
+
continue
|
|
1127
|
+
# Standalone seed of schema-derived metadata → drop; CodeGen regenerates it.
|
|
1128
|
+
if isinstance(stmt, exp.Insert) and _METADATA_TABLES.search(stmt.sql(dialect="tsql")):
|
|
1129
|
+
continue
|
|
1130
|
+
# T-SQL procedural glue with no standalone PG equivalent — DECLARE @v / SET @v /
|
|
1131
|
+
# SELECT @v = ... / IF @v ... EXEC('...'). In Category-B (regular DDL/DML) these
|
|
1132
|
+
# only appear as leaked imperative logic (e.g. dynamic auto-named-constraint
|
|
1133
|
+
# drops); real proc/function bodies are classified hand-procedural upstream and
|
|
1134
|
+
# never reach here. Emitting them produces invalid `$v` SQL — report, don't emit.
|
|
1135
|
+
# Detect by the presence of a T-SQL `@v` (which parses to exp.Parameter) rather than
|
|
1136
|
+
# by statement type: sqlglot's node type for these varies across versions (e.g. `IF …`
|
|
1137
|
+
# is exp.If in some, exp.IfBlock in others), and an enumerated list silently leaks the
|
|
1138
|
+
# ones it misses. The protected Flyway macro is an Identifier, not a Parameter, so this
|
|
1139
|
+
# never false-positives on `${flyway:defaultSchema}`.
|
|
1140
|
+
# An expression COLLATE (`c COLLATE Latin1_General_BIN2 <> 'y'`) encodes case-sensitivity
|
|
1141
|
+
# that PG can't express by mechanically dropping the qualifier without changing semantics
|
|
1142
|
+
# (see `_strip_collate`); report so the migration falls back to a hand-authored PG form.
|
|
1143
|
+
if isinstance(stmt, exp.Declare) or stmt.find(exp.Parameter) is not None or stmt.find(exp.Collate) is not None:
|
|
1144
|
+
txt = stmt.sql(dialect="tsql")
|
|
1145
|
+
unhandled.append({"kind": _first_keyword(txt), "snippet": txt[:80]})
|
|
1146
|
+
continue
|
|
1147
|
+
# Self-aliased UPDATE…FROM with an outer join: no semantics-preserving PG
|
|
1148
|
+
# rewrite exists (see `_update_alias_outer_join`) — report, don't emit.
|
|
1149
|
+
if isinstance(stmt, exp.Update) and _update_alias_outer_join(stmt):
|
|
1150
|
+
txt = stmt.sql(dialect="tsql")
|
|
1151
|
+
unhandled.append({"kind": "UPDATE-OUTER-JOIN", "snippet": txt[:80]})
|
|
1152
|
+
continue
|
|
1153
|
+
# Hand-written routines: a T-SQL PROCEDURE/FUNCTION/TRIGGER body cannot be
|
|
1154
|
+
# transpiled mechanically (parameter syntax, control flow, and the body itself
|
|
1155
|
+
# are all T-SQL) — naive emission produces invalid PG like `$x INT AS BEGIN …`.
|
|
1156
|
+
# The classifier flags these files needs-hand-authoring; here we report the
|
|
1157
|
+
# routine so it lands in the gap comments instead of half-translated output.
|
|
1158
|
+
if isinstance(stmt, exp.Create) and (stmt.args.get("kind") or "").upper() in (
|
|
1159
|
+
"PROCEDURE",
|
|
1160
|
+
"FUNCTION",
|
|
1161
|
+
"TRIGGER",
|
|
1162
|
+
):
|
|
1163
|
+
txt = stmt.sql(dialect="tsql")
|
|
1164
|
+
name = stmt.find(exp.Table)
|
|
1165
|
+
unhandled.append({
|
|
1166
|
+
"kind": f"CREATE-{(stmt.args.get('kind') or '').upper()}",
|
|
1167
|
+
"snippet": (name.sql(dialect="tsql") + " — " if name else "") + txt[:80],
|
|
1168
|
+
})
|
|
1169
|
+
swallow_routine_end = True
|
|
1170
|
+
continue
|
|
1171
|
+
if isinstance(stmt, exp.Command):
|
|
1172
|
+
# sqlglot may glom a preceding comment onto the Command (`/* note */ ALTER …`),
|
|
1173
|
+
# which defeats the `^ALTER` anchors below; match against the bare statement.
|
|
1174
|
+
txt = _strip_leading_sql_comments(stmt.sql(dialect="tsql"))
|
|
1175
|
+
# T-SQL standalone column-default constraint — sqlglot can't parse it, so it
|
|
1176
|
+
# lands here as a Command. Emit the PG `ALTER COLUMN … SET DEFAULT` form.
|
|
1177
|
+
dc = _transpile_default_constraint(txt)
|
|
1178
|
+
if dc is not None:
|
|
1179
|
+
out.append(dc)
|
|
1180
|
+
continue
|
|
1181
|
+
# T-SQL `ALTER TABLE … ALTER COLUMN c <type> [NOT] NULL` — also a Command.
|
|
1182
|
+
ac = _transpile_alter_column(txt)
|
|
1183
|
+
if ac is not None:
|
|
1184
|
+
out.append(ac)
|
|
1185
|
+
continue
|
|
1186
|
+
# SQL Server batch-control noise — not needed on PG, drop silently.
|
|
1187
|
+
if _re.match(r"^\s*(BEGIN\s+TRY|END\s+TRY|BEGIN\s+CATCH|END\s+CATCH|SET\s+NOEXEC|GO)\b", txt, _re.IGNORECASE):
|
|
1188
|
+
continue
|
|
1189
|
+
unhandled.append({"kind": _first_keyword(txt), "snippet": txt[:80]})
|
|
1190
|
+
continue
|
|
1191
|
+
# SS session/batch-control SETs have no PG equivalent and error as unrecognized
|
|
1192
|
+
# config params — drop them (NOEXEC, NOCOUNT, XACT_ABORT, QUOTED_IDENTIFIER, ANSI_*).
|
|
1193
|
+
if isinstance(stmt, exp.Set) and _re.search(
|
|
1194
|
+
r"\b(NOEXEC|NOCOUNT|XACT_ABORT|QUOTED_IDENTIFIER|ANSI_NULLS|ANSI_PADDING|ANSI_WARNINGS|"
|
|
1195
|
+
r"ARITHABORT|CONCAT_NULL_YIELDS_NULL|NUMERIC_ROUNDABORT)\b",
|
|
1196
|
+
stmt.sql(dialect="tsql"), _re.IGNORECASE):
|
|
1197
|
+
continue
|
|
1198
|
+
# RAISERROR(...) at statement level is invalid PG outside a function — drop it
|
|
1199
|
+
# (inside an IF…BEGIN guard it is handled as RAISE EXCEPTION by the DO-block path).
|
|
1200
|
+
if isinstance(stmt, exp.Anonymous) and (stmt.name or "").upper() == "RAISERROR":
|
|
1201
|
+
continue
|
|
1202
|
+
# `ALTER TABLE t ALTER COLUMN c <type>` with NO nullability spec parses cleanly
|
|
1203
|
+
# (unlike the `… NULL`/`… NOT NULL` forms, which land as opaque Commands). Route
|
|
1204
|
+
# it through the same structured emission as those, so it gets the dependent-view
|
|
1205
|
+
# drop and T-SQL's implied nullability reset (omitting the spec → NULLable).
|
|
1206
|
+
if isinstance(stmt, exp.Alter):
|
|
1207
|
+
acts = stmt.args.get("actions") or []
|
|
1208
|
+
if len(acts) == 1 and isinstance(acts[0], exp.AlterColumn) and acts[0].args.get("dtype"):
|
|
1209
|
+
ac = _transpile_alter_column(stmt.sql(dialect="tsql"))
|
|
1210
|
+
if ac is not None:
|
|
1211
|
+
out.append(ac)
|
|
1212
|
+
continue
|
|
1213
|
+
if isinstance(stmt, exp.Create) and (stmt.args.get("kind") or "").upper() == "NONCLUSTERED INDEX":
|
|
1214
|
+
stmt.set("kind", "INDEX") # PG has no NONCLUSTERED qualifier
|
|
1215
|
+
stmt = _rewrite_boolean_int_comparisons(stmt)
|
|
1216
|
+
stmt = (
|
|
1217
|
+
stmt.transform(_rewrite_update_from_alias) # restructures UPDATE before other rewrites
|
|
1218
|
+
.transform(_rewrite_insert_booleans) # seed-INSERT 1/0 → TRUE/FALSE for bit cols
|
|
1219
|
+
.transform(_fix_misparsed_table_constraint)
|
|
1220
|
+
.transform(_rewrite_functions)
|
|
1221
|
+
.transform(_rewrite_boolean_defaults)
|
|
1222
|
+
.transform(_rewrite_string_concat)
|
|
1223
|
+
.transform(_strip_national)
|
|
1224
|
+
.transform(_strip_collate)
|
|
1225
|
+
.transform(_drop_isjson_checks) # drop ISJSON CHECK constraints BEFORE…
|
|
1226
|
+
.transform(_rewrite_isjson_eq) # …rewriting surviving ISJSON predicates (WHERE)
|
|
1227
|
+
.transform(_rewrite_isjson_bare)
|
|
1228
|
+
.transform(_fold_clustered_constraints)
|
|
1229
|
+
.transform(_strip_nulls_ordering)
|
|
1230
|
+
.transform(_column_fk_to_reference)
|
|
1231
|
+
)
|
|
1232
|
+
# An ALTER TABLE whose only action was dropped (e.g. an ISJSON ADD CONSTRAINT)
|
|
1233
|
+
# is left actionless — emitting bare `ALTER TABLE x` is a PG syntax error. Skip.
|
|
1234
|
+
if isinstance(stmt, exp.Alter) and not stmt.args.get("actions"):
|
|
1235
|
+
continue
|
|
1236
|
+
# Behavioral self-check: any `boolean = integer` comparison the coercion pass didn't
|
|
1237
|
+
# eliminate would abort on PG. Surface it as a gap (not silent output) so the gap
|
|
1238
|
+
# count reflects type-correctness, not just parseability.
|
|
1239
|
+
residual = _residual_boolean_int(stmt)
|
|
1240
|
+
if residual:
|
|
1241
|
+
unhandled.append({"kind": "BOOL-INT-RESIDUAL", "snippet": "; ".join(residual)[:120]})
|
|
1242
|
+
continue
|
|
1243
|
+
out.append(stmt.sql(dialect=MJPostgres, pretty=pretty, identify=True))
|
|
1244
|
+
return (";\n".join(out) + (";" if out else "")), unhandled
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
_RAISERROR = _re.compile(r"RAISERROR\s*\(\s*(N?'(?:[^']|'')*'|@?\w+)", _re.IGNORECASE)
|
|
1248
|
+
|
|
1249
|
+
# Inline entity-metadata INSERTs (Entity / EntityField / ApplicationEntity / …) in a
|
|
1250
|
+
# FEATURE migration are KEPT and transpiled — NOT dropped. Empirically, `mj codegen` on
|
|
1251
|
+
# PostgreSQL regenerates SQL *objects* (views, CRUD functions, triggers — already dropped
|
|
1252
|
+
# by the splitter's CodeGen-block extraction) but does NOT introspect the schema to (re)create
|
|
1253
|
+
# Entity/EntityField *metadata rows* the way SQL-Server CodeGen does (the schema-management
|
|
1254
|
+
# sprocs that drive that — spUpdateExistingEntitiesFromSchema, … — are SQL-Server-only).
|
|
1255
|
+
# So a new entity's registration rows have NO other source: dropping them leaves the entity
|
|
1256
|
+
# absent from metadata, and CodeGen then generates no view/sproc for it (a parity gap of
|
|
1257
|
+
# exactly the new-entity views/sprocs). Pure-metadata `*_Metadata_Sync` migrations are still
|
|
1258
|
+
# fully re-seeded via `mj sync push` — the SPLITTER routes those to reseed (empty kept-TSQL),
|
|
1259
|
+
# so they never reach here. This matcher is therefore intentionally disabled (matches nothing).
|
|
1260
|
+
_METADATA_TABLES = _re.compile(r"(?!x)x")
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def _transpile_extprop_segment(text: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
1264
|
+
"""Transpile a text segment that may interleave sp_add/dropextendedproperty envelopes
|
|
1265
|
+
with plain SQL — used for IF…BEGIN bodies, where a guarded INSERT can sit next to an
|
|
1266
|
+
extprop EXEC (the top-level batch walker handles the same mix outside blocks)."""
|
|
1267
|
+
out: list[str] = []
|
|
1268
|
+
unhandled: list[dict] = []
|
|
1269
|
+
pos = 0
|
|
1270
|
+
while pos < len(text):
|
|
1271
|
+
ext = _SP_EXTPROP.search(text, pos)
|
|
1272
|
+
dxp = _SP_DROPEXTPROP.search(text, pos)
|
|
1273
|
+
nxt = min([x for x in (ext, dxp) if x], key=lambda x: x.start(), default=None)
|
|
1274
|
+
gap = text[pos:] if nxt is None else text[pos:nxt.start()]
|
|
1275
|
+
if gap.strip():
|
|
1276
|
+
s, u = _transpile_plain(gap, pretty)
|
|
1277
|
+
if s.strip():
|
|
1278
|
+
out.append(s)
|
|
1279
|
+
unhandled.extend(u)
|
|
1280
|
+
if nxt is None:
|
|
1281
|
+
break
|
|
1282
|
+
if nxt is ext:
|
|
1283
|
+
comment = _transpile_sp_addextendedproperty(nxt.group("args"))
|
|
1284
|
+
if comment:
|
|
1285
|
+
out.append(comment)
|
|
1286
|
+
elif comment is None:
|
|
1287
|
+
unhandled.append({"kind": "sp_addextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
1288
|
+
else:
|
|
1289
|
+
comment = _transpile_sp_dropextendedproperty(nxt.group("args"))
|
|
1290
|
+
if comment:
|
|
1291
|
+
out.append(comment)
|
|
1292
|
+
elif comment is None:
|
|
1293
|
+
unhandled.append({"kind": "sp_dropextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
1294
|
+
pos = nxt.end()
|
|
1295
|
+
return "\n".join(out), unhandled
|
|
1296
|
+
|
|
1297
|
+
|
|
1298
|
+
# SS catalog references in a guard condition (sys.* views / OBJECT_ID()) — meaningless
|
|
1299
|
+
# on PG; the common shapes are translated to PG catalog equivalents, the rest reported.
|
|
1300
|
+
_SYS_CATALOG_REF = _re.compile(r"\bsys\s*\.\s*\w+|\bOBJECT_ID\s*\(", _re.IGNORECASE)
|
|
1301
|
+
_SYS_TABLE = _re.compile(r"\bsys\s*\.\s*(\w+)", _re.IGNORECASE)
|
|
1302
|
+
_OBJECT_ID_ARG = _re.compile(
|
|
1303
|
+
r"\bOBJECT_ID\s*\(\s*N?'(?P<obj>[^']+)'\s*(?:,\s*N?'(?P<type>[^']*)'\s*)?\)", _re.IGNORECASE
|
|
1304
|
+
)
|
|
1305
|
+
_NAME_EQ = _re.compile(r"\bname\s*=\s*N?'(?P<name>[^']+)'", _re.IGNORECASE)
|
|
1306
|
+
# Predicate shapes the translator understands; anything left over in the WHERE clause
|
|
1307
|
+
# beyond these (plus AND/whitespace/parens) makes the guard untranslatable.
|
|
1308
|
+
_KNOWN_SYS_PREDS = (
|
|
1309
|
+
_re.compile(r"\b\w+(?:\s*\.\s*\w+)?\s*=\s*OBJECT_ID\s*\([^)]*\)", _re.IGNORECASE),
|
|
1310
|
+
_re.compile(r"\b(?:\w+\s*\.\s*)?name\s*=\s*N?'[^']*'", _re.IGNORECASE),
|
|
1311
|
+
_re.compile(r"\btype\s*(?:=\s*N?'U'|IN\s*\(\s*N?'U'\s*\))", _re.IGNORECASE),
|
|
1312
|
+
_re.compile(r"\bschema_id\s*=\s*SCHEMA_ID\s*\([^)]*\)", _re.IGNORECASE),
|
|
1313
|
+
)
|
|
1314
|
+
|
|
1315
|
+
|
|
1316
|
+
def _qualified_obj(obj: str) -> tuple[str, str]:
|
|
1317
|
+
"""Split an OBJECT_ID('sch.Tbl') argument into raw (schema, table) names; a missing
|
|
1318
|
+
schema qualifier defaults to the Flyway macro (MJ's default schema)."""
|
|
1319
|
+
toks = _IDENT_TOKEN.findall(obj)
|
|
1320
|
+
names = [t[1:-1] if t[:1] in ("[", '"') else t for t in toks]
|
|
1321
|
+
table = names[-1] if names else obj
|
|
1322
|
+
schema = names[-2] if len(names) >= 2 else FLYWAY_MACRO
|
|
1323
|
+
return schema, table
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
def _sys_guard_residue_ok(cond: str) -> bool:
|
|
1327
|
+
"""True when the guard's WHERE clause consists only of recognized predicates joined
|
|
1328
|
+
by AND — i.e. nothing semantically load-bearing would be dropped in translation."""
|
|
1329
|
+
parts = _re.split(r"\bWHERE\b", cond, maxsplit=1, flags=_re.IGNORECASE)
|
|
1330
|
+
if len(parts) != 2 or _re.search(r"\bJOIN\b", cond, _re.IGNORECASE):
|
|
1331
|
+
return False
|
|
1332
|
+
residue = parts[1]
|
|
1333
|
+
for p in _KNOWN_SYS_PREDS:
|
|
1334
|
+
residue = p.sub(" ", residue)
|
|
1335
|
+
return not _re.search(r"[=<>']|\b(?:OR|IN|LIKE|EXISTS|NOT|SELECT)\b", residue, _re.IGNORECASE)
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
def _translate_sys_guard(cond: str, neg: bool) -> str | None:
|
|
1339
|
+
"""Translate the common SQL-Server catalog guard conditions to a full PG predicate
|
|
1340
|
+
(the text between `IF` and `THEN`). Handled shapes:
|
|
1341
|
+
* sys.columns + OBJECT_ID(tbl) + name='col' → EXISTS (information_schema.columns …)
|
|
1342
|
+
* sys.tables / sys.objects table existence → to_regclass(…) IS [NOT] NULL
|
|
1343
|
+
* sys.indexes + OBJECT_ID(tbl) + name='idx' → EXISTS (pg_indexes …)
|
|
1344
|
+
Returns None when the shape isn't confidently recognized — the caller then routes the
|
|
1345
|
+
whole IF block to unhandled rather than emitting sys.* references PG rejects."""
|
|
1346
|
+
sys_tables = {t.lower() for t in _SYS_TABLE.findall(cond)}
|
|
1347
|
+
obj = _OBJECT_ID_ARG.search(cond)
|
|
1348
|
+
name = _NAME_EQ.search(cond)
|
|
1349
|
+
exists = "NOT EXISTS" if neg else "EXISTS"
|
|
1350
|
+
if not _sys_guard_residue_ok(cond):
|
|
1351
|
+
return None
|
|
1352
|
+
if sys_tables == {"columns"} and obj and name:
|
|
1353
|
+
sch, tbl = _qualified_obj(obj.group("obj"))
|
|
1354
|
+
return (
|
|
1355
|
+
f"{exists} (SELECT 1 FROM information_schema.columns WHERE table_schema = '{sch}' "
|
|
1356
|
+
f"AND table_name = '{tbl}' AND column_name = '{name.group('name')}')"
|
|
1357
|
+
)
|
|
1358
|
+
if sys_tables == {"indexes"} and obj and name:
|
|
1359
|
+
sch, tbl = _qualified_obj(obj.group("obj"))
|
|
1360
|
+
return (
|
|
1361
|
+
f"{exists} (SELECT 1 FROM pg_indexes WHERE schemaname = '{sch}' "
|
|
1362
|
+
f"AND tablename = '{tbl}' AND indexname = '{name.group('name')}')"
|
|
1363
|
+
)
|
|
1364
|
+
if sys_tables == {"objects"} and obj:
|
|
1365
|
+
# Only a user-table check ('U' — in OBJECT_ID's 2nd arg or a type predicate) maps
|
|
1366
|
+
# to to_regclass; procs/views/triggers are CodeGen objects handled out of band.
|
|
1367
|
+
type_arg = (obj.group("type") or "").strip().upper()
|
|
1368
|
+
if type_arg != "U" and not _re.search(r"\btype\s*(?:=\s*N?'U'|IN\s*\(\s*N?'U'\s*\))", cond, _re.IGNORECASE):
|
|
1369
|
+
return None
|
|
1370
|
+
sch, tbl = _qualified_obj(obj.group("obj"))
|
|
1371
|
+
return f"to_regclass('{sch}.\"{tbl}\"') IS {'NULL' if neg else 'NOT NULL'}"
|
|
1372
|
+
if sys_tables == {"tables"} and (obj or name):
|
|
1373
|
+
if obj:
|
|
1374
|
+
sch, tbl = _qualified_obj(obj.group("obj"))
|
|
1375
|
+
else:
|
|
1376
|
+
tbl = name.group("name")
|
|
1377
|
+
ms = _re.search(r"\bSCHEMA_ID\s*\(\s*N?'([^']+)'\s*\)", cond, _re.IGNORECASE)
|
|
1378
|
+
sch = ms.group(1) if ms else FLYWAY_MACRO
|
|
1379
|
+
return f"to_regclass('{sch}.\"{tbl}\"') IS {'NULL' if neg else 'NOT NULL'}"
|
|
1380
|
+
return None
|
|
1381
|
+
|
|
1382
|
+
|
|
1383
|
+
def _transpile_if_exists_begin(m: _IfExistsMatch) -> tuple[str, list[dict]]:
|
|
1384
|
+
"""IF [NOT] EXISTS(<sel>) BEGIN <body> END → PG DO $$ … IF … THEN … END IF; … $$;"""
|
|
1385
|
+
raw_body = m.group("body").strip()
|
|
1386
|
+
# Idempotent seed of schema-derived metadata → drop; CodeGen regenerates it.
|
|
1387
|
+
if _METADATA_TABLES.search(raw_body):
|
|
1388
|
+
return "", []
|
|
1389
|
+
# Extended-property comment dance: a guard whose body consists EXCLUSIVELY of
|
|
1390
|
+
# sp_add/dropextendedproperty EXECs (plus PRINT/comment noise) is SQL-Server-only —
|
|
1391
|
+
# PG `COMMENT ON … IS …` overwrites unconditionally and `mj codegen` re-syncs every
|
|
1392
|
+
# MS_Description from EntityField.Description, so the drop-then-re-add dance is a
|
|
1393
|
+
# no-op on PG. Drop the whole guard. A body that ALSO carries real statements (e.g.
|
|
1394
|
+
# a guarded INSERT before the EXEC) is NOT dropped — it flows through the normal
|
|
1395
|
+
# path below, where the extprop-aware segment walker handles the mix.
|
|
1396
|
+
if _SP_EXTPROP.search(raw_body) or _SP_DROPEXTPROP.search(raw_body):
|
|
1397
|
+
rest = _SP_DROPEXTPROP.sub("", _SP_EXTPROP.sub("", raw_body))
|
|
1398
|
+
leftover = [
|
|
1399
|
+
s for s in _split_top_level_statements(rest)
|
|
1400
|
+
if _strip_leading_sql_comments(s).strip(" \t\r\n;")
|
|
1401
|
+
and not _re.match(r"\s*PRINT\b", _strip_leading_sql_comments(s), _re.IGNORECASE)
|
|
1402
|
+
]
|
|
1403
|
+
if not leftover:
|
|
1404
|
+
return "", []
|
|
1405
|
+
neg = "NOT " if m.group("neg") else ""
|
|
1406
|
+
cond_raw = m.group("cond").strip()
|
|
1407
|
+
u1: list[dict] = []
|
|
1408
|
+
# SS catalog guards (sys.* / OBJECT_ID()) fail at apply on PG — translate the common
|
|
1409
|
+
# shapes; anything unrecognized is reported whole, never emitted as sys.* SQL.
|
|
1410
|
+
if _SYS_CATALOG_REF.search(cond_raw):
|
|
1411
|
+
cond_full = _translate_sys_guard(cond_raw, bool(m.group("neg")))
|
|
1412
|
+
if cond_full is None:
|
|
1413
|
+
return "", [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}]
|
|
1414
|
+
else:
|
|
1415
|
+
cond_sql, u1 = _transpile_plain(cond_raw)
|
|
1416
|
+
cond_inner = cond_sql.rstrip(";").strip()
|
|
1417
|
+
cond_full = f"{neg}EXISTS ({cond_inner})" if cond_inner else None
|
|
1418
|
+
# Guard blocks (IF EXISTS(...) BEGIN RAISERROR('conflict') END) → RAISE EXCEPTION.
|
|
1419
|
+
rr = _RAISERROR.search(raw_body)
|
|
1420
|
+
if rr:
|
|
1421
|
+
msg = rr.group(1)
|
|
1422
|
+
msg = _pg_string(_unquote_tsql_string(msg)) if msg.lstrip("Nn").startswith("'") else "'migration guard failed'"
|
|
1423
|
+
body_sql, u2 = f"RAISE EXCEPTION {msg};", []
|
|
1424
|
+
else:
|
|
1425
|
+
body_sql, u2 = _transpile_extprop_segment(raw_body)
|
|
1426
|
+
if not cond_full or not body_sql.strip():
|
|
1427
|
+
return "", (u1 + u2 + [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}])
|
|
1428
|
+
do = (
|
|
1429
|
+
"DO $$\nBEGIN\n"
|
|
1430
|
+
f" IF {cond_full} THEN\n"
|
|
1431
|
+
f" {body_sql.strip()}\n"
|
|
1432
|
+
" END IF;\nEND $$;"
|
|
1433
|
+
)
|
|
1434
|
+
return do, (u1 + u2)
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
def mj_transpile(sql: str, *, pretty: bool = True, identify: bool = True) -> dict:
|
|
1438
|
+
"""
|
|
1439
|
+
Transpile SS T-SQL (Category-B DDL/DML) to MJ-flavored PostgreSQL via the AST.
|
|
1440
|
+
|
|
1441
|
+
Handles three fixed-shape envelopes sqlglot can't parse — `sp_addextendedproperty`
|
|
1442
|
+
(→ COMMENT ON), `IF [NOT] EXISTS(...) BEGIN ... END` (→ DO $$ ... $$), and GO
|
|
1443
|
+
batch separators — by recognizing the envelope structurally and routing the real
|
|
1444
|
+
SQL inside through the AST dialect. Anything still unparseable is *reported* in
|
|
1445
|
+
"unhandled" rather than emitted as invalid passthrough.
|
|
1446
|
+
"""
|
|
1447
|
+
out: list[str] = []
|
|
1448
|
+
unhandled: list[dict] = []
|
|
1449
|
+
|
|
1450
|
+
# File-level pass: register every BIT column so boolean comparisons in separate
|
|
1451
|
+
# statements can be resolved (see `_rewrite_boolean_int_comparisons`). Augment with
|
|
1452
|
+
# boolean columns of tables declared OUTSIDE this file (the baseline / earlier
|
|
1453
|
+
# migrations), supplied as JSON [["table","col"],…] in MJ_EXTRA_BIT_COLS — needed to
|
|
1454
|
+
# coerce 1/0 → TRUE/FALSE in seed INSERTs that target core tables (e.g. User.IsActive).
|
|
1455
|
+
global _BIT_COLS
|
|
1456
|
+
_BIT_COLS = _collect_bit_columns(sql)
|
|
1457
|
+
extra = _os.environ.get("MJ_EXTRA_BIT_COLS")
|
|
1458
|
+
if extra:
|
|
1459
|
+
try:
|
|
1460
|
+
_BIT_COLS |= {(t.lower(), c.lower()) for t, c in _json.loads(extra)}
|
|
1461
|
+
except Exception: # noqa: BLE001
|
|
1462
|
+
pass
|
|
1463
|
+
|
|
1464
|
+
for batch in _GO_SPLIT.split(sql):
|
|
1465
|
+
if not batch.strip():
|
|
1466
|
+
continue
|
|
1467
|
+
out_sql, u = _transpile_batch(batch, pretty)
|
|
1468
|
+
out.extend(out_sql)
|
|
1469
|
+
unhandled.extend(u)
|
|
1470
|
+
|
|
1471
|
+
# Final safety net: the macro is protected to FLYWAY_SENTINEL by a blanket text
|
|
1472
|
+
# replace before parsing, and restored at the AST level in identifier_sql (identifier
|
|
1473
|
+
# position) and literal_sql (string-literal content). Anything sqlglot carries as
|
|
1474
|
+
# opaque text — chiefly trailing `/* … */` comments attached to a statement — never
|
|
1475
|
+
# passes through either hook, so a sentinel can survive there. Restoring it in the
|
|
1476
|
+
# emitted SQL guarantees no `__mj_flyway_default_schema__` ever leaks into output
|
|
1477
|
+
# (it is harmless in a comment, but would be a real "schema does not exist" error if
|
|
1478
|
+
# it ever appeared in an unhandled executable position). Text-level, post-AST.
|
|
1479
|
+
out = [s.replace(FLYWAY_SENTINEL, FLYWAY_MACRO) for s in out]
|
|
1480
|
+
# Same restoration for the gap report — snippets are shown to humans/LLMs and must
|
|
1481
|
+
# read as the original macro, not the internal sentinel.
|
|
1482
|
+
unhandled = [
|
|
1483
|
+
{**u, "snippet": u["snippet"].replace(FLYWAY_SENTINEL, FLYWAY_MACRO)}
|
|
1484
|
+
for u in unhandled
|
|
1485
|
+
]
|
|
1486
|
+
|
|
1487
|
+
return {"sql": out, "unhandled": unhandled}
|
|
1488
|
+
|
|
1489
|
+
|
|
1490
|
+
# Baseline extended-property EXECs come wrapped in per-statement error handling:
|
|
1491
|
+
# BEGIN TRY <EXEC sp_addextendedproperty …> END TRY
|
|
1492
|
+
# BEGIN CATCH DECLARE @msg…; SELECT @msg=ERROR_MESSAGE()…; RAISERROR(…); SET NOEXEC ON END CATCH
|
|
1493
|
+
# The TRY/CATCH delimiters are dropped as batch noise downstream, but the CATCH-body
|
|
1494
|
+
# DECLARE/SELECT would surface as bogus "unhandled" procedural leaks. Strip a CATCH
|
|
1495
|
+
# block only when its body is purely that plumbing — anything substantive stays put
|
|
1496
|
+
# and is transpiled/reported by the batch walk.
|
|
1497
|
+
_CATCH_BLOCK = _re.compile(r"BEGIN\s+CATCH\b(?P<body>.*?)\bEND\s+CATCH\b\s*;?", _re.IGNORECASE | _re.DOTALL)
|
|
1498
|
+
_CATCH_NOISE_STMT = _re.compile(r"^\s*(DECLARE\b|SELECT\s+@|RAISERROR\s*\(|PRINT\b|SET\s+NOEXEC\b|THROW\b)", _re.IGNORECASE)
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
def _strip_catch_noise(batch: str) -> str:
|
|
1502
|
+
"""Remove BEGIN CATCH…END CATCH wrappers whose body is purely error-reporting
|
|
1503
|
+
plumbing (DECLARE / SELECT @x=ERROR_*() / RAISERROR / PRINT / SET NOEXEC / THROW)."""
|
|
1504
|
+
def repl(m: "_re.Match") -> str:
|
|
1505
|
+
stmts = [s for s in _split_top_level_statements(m.group("body")) if _strip_leading_sql_comments(s).strip(" \t\r\n;")]
|
|
1506
|
+
return "" if all(_CATCH_NOISE_STMT.match(_strip_leading_sql_comments(s)) for s in stmts) else m.group(0)
|
|
1507
|
+
return _CATCH_BLOCK.sub(repl, batch)
|
|
1508
|
+
|
|
1509
|
+
|
|
1510
|
+
def _transpile_batch(batch: str, pretty: bool = False) -> tuple[list[str], list[dict]]:
|
|
1511
|
+
"""Scan one GO batch into envelope chunks + plain SQL, transpiling each in order."""
|
|
1512
|
+
out: list[str] = []
|
|
1513
|
+
unhandled: list[dict] = []
|
|
1514
|
+
|
|
1515
|
+
# Extended-property batches: drop the SS error-handling plumbing around the EXECs
|
|
1516
|
+
# (see _strip_catch_noise), then let the walk below handle EVERYTHING in the batch —
|
|
1517
|
+
# a CREATE INDEX / INSERT sharing the batch must transpile, never silently vanish.
|
|
1518
|
+
if _SP_EXTPROP.search(batch) or _SP_DROPEXTPROP.search(batch):
|
|
1519
|
+
batch = _strip_catch_noise(batch)
|
|
1520
|
+
|
|
1521
|
+
pos = 0
|
|
1522
|
+
# Walk the batch, alternating between recognized envelopes and plain SQL gaps.
|
|
1523
|
+
while pos < len(batch):
|
|
1524
|
+
ext = _SP_EXTPROP.search(batch, pos)
|
|
1525
|
+
dxp = _SP_DROPEXTPROP.search(batch, pos)
|
|
1526
|
+
ife = _find_if_exists_begin(batch, pos)
|
|
1527
|
+
nxt = min([m for m in (ext, dxp, ife) if m], key=lambda m: m.start(), default=None)
|
|
1528
|
+
if nxt is None:
|
|
1529
|
+
gap = batch[pos:]
|
|
1530
|
+
if gap.strip():
|
|
1531
|
+
s, u = _transpile_plain(gap, pretty)
|
|
1532
|
+
if s.strip():
|
|
1533
|
+
out.append(s)
|
|
1534
|
+
unhandled.extend(u)
|
|
1535
|
+
break
|
|
1536
|
+
gap = batch[pos:nxt.start()]
|
|
1537
|
+
if gap.strip():
|
|
1538
|
+
s, u = _transpile_plain(gap, pretty)
|
|
1539
|
+
if s.strip():
|
|
1540
|
+
out.append(s)
|
|
1541
|
+
unhandled.extend(u)
|
|
1542
|
+
if nxt is ext:
|
|
1543
|
+
comment = _transpile_sp_addextendedproperty(nxt.group("args"))
|
|
1544
|
+
if comment:
|
|
1545
|
+
out.append(comment)
|
|
1546
|
+
elif comment is None: # "" is an intentional CodeGen-object skip, not a failure
|
|
1547
|
+
unhandled.append({"kind": "sp_addextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
1548
|
+
elif nxt is dxp:
|
|
1549
|
+
comment = _transpile_sp_dropextendedproperty(nxt.group("args"))
|
|
1550
|
+
if comment:
|
|
1551
|
+
out.append(comment)
|
|
1552
|
+
elif comment is None:
|
|
1553
|
+
unhandled.append({"kind": "sp_dropextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
1554
|
+
else:
|
|
1555
|
+
do, u = _transpile_if_exists_begin(nxt)
|
|
1556
|
+
if do.strip():
|
|
1557
|
+
out.append(do)
|
|
1558
|
+
unhandled.extend(u)
|
|
1559
|
+
pos = nxt.end()
|
|
1560
|
+
return out, unhandled
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
if __name__ == "__main__":
|
|
1564
|
+
import sys, json
|
|
1565
|
+
if "--collect-bitcols" in sys.argv:
|
|
1566
|
+
# Emit [["table","col"],…] for every BIT/BOOLEAN column in the piped SQL — lets the
|
|
1567
|
+
# convert driver build a cross-file registry (baseline tables) for INSERT coercion.
|
|
1568
|
+
print(json.dumps(sorted(list(_collect_bit_columns(sys.stdin.read())))))
|
|
1569
|
+
else:
|
|
1570
|
+
result = mj_transpile(sys.stdin.read())
|
|
1571
|
+
print(json.dumps(result, indent=2))
|