@memberjunction/sqlglot-ts 5.48.0 → 5.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -31,6 +31,13 @@ from sqlglot import exp
|
|
|
31
31
|
from sqlglot.dialects.postgres import Postgres
|
|
32
32
|
from sqlglot.dialects.tsql import TSQL
|
|
33
33
|
|
|
34
|
+
# sqlglot's node type for a T-SQL `IF …` statement varies across versions — `exp.If` in
|
|
35
|
+
# older releases, `exp.IfBlock` only from 29.0.0. The committed pin (requirements.txt,
|
|
36
|
+
# sqlglot~=27.18.0) has NO IfBlock, so a bare `exp.IfBlock` attribute access would raise
|
|
37
|
+
# AttributeError on every statement reaching the plain-SQL path. Resolve whichever names
|
|
38
|
+
# exist ONCE at import, so the isinstance guard is version-tolerant (issue #3252 review).
|
|
39
|
+
_IF_NODE_TYPES = tuple(t for t in (getattr(exp, "If", None), getattr(exp, "IfBlock", None)) if t is not None)
|
|
40
|
+
|
|
34
41
|
# The Flyway schema macro is not SQL; protect it to a sentinel identifier for the
|
|
35
42
|
# parse, then the Generator restores it verbatim.
|
|
36
43
|
FLYWAY_MACRO = "${flyway:defaultSchema}"
|
|
@@ -197,6 +204,14 @@ def _is_isjson(node: exp.Expression) -> bool:
|
|
|
197
204
|
return isinstance(node, exp.Anonymous) and (node.name or "").upper() == "ISJSON"
|
|
198
205
|
|
|
199
206
|
|
|
207
|
+
def _contains_isjson_call(node: exp.Expression) -> bool:
|
|
208
|
+
"""True if `node`'s subtree contains a genuine ISJSON(...) call. Distinguishes the CHECK
|
|
209
|
+
predicate `ISJSON(x)` from a column merely NAMED `IsJsonEnabled` or the string literal
|
|
210
|
+
'ISJSON' — neither parses to an exp.Anonymous, so a substring match would false-drop them
|
|
211
|
+
(issue #3252 review). See _is_isjson."""
|
|
212
|
+
return any(_is_isjson(n) for n in node.find_all(exp.Anonymous))
|
|
213
|
+
|
|
214
|
+
|
|
200
215
|
def _rewrite_isjson_eq(node: exp.Expression) -> exp.Expression:
|
|
201
216
|
"""SS `ISJSON(x) = 1` / `ISJSON(x) = 0` → PG `x IS JSON` / `x IS NOT JSON`.
|
|
202
217
|
PG has no ISJSON function; it has the SQL:2016 `IS JSON` predicate (PG16+)."""
|
|
@@ -486,10 +501,15 @@ def _strip_collate(node: exp.Expression) -> exp.Expression:
|
|
|
486
501
|
def _drop_isjson_checks(node: exp.Expression) -> exp.Expression:
|
|
487
502
|
"""Drop CHECK constraints using ISJSON() — PG has no ISJSON (validity enforced elsewhere).
|
|
488
503
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
504
|
+
Detection is by the ISJSON *function call* (exp.Anonymous via _contains_isjson_call), never a
|
|
505
|
+
substring of rendered SQL — so a CHECK on a column NAMED `IsJsonEnabled` or comparing to the
|
|
506
|
+
literal 'ISJSON' is preserved (issue #3252 review). Filtering is per INDIVIDUAL constraint: an
|
|
507
|
+
`ADD CONSTRAINT pk PRIMARY KEY (...), CONSTRAINT ck CHECK (ISJSON(...))` drops only the ISJSON
|
|
508
|
+
check and KEEPS the sibling PK/FK — previously the whole AddConstraint (hence the sibling) was
|
|
509
|
+
silently lost. Column-level checks are filtered on the ColumnDef; a table-level named CHECK
|
|
510
|
+
that is entirely ISJSON is removed outright (return None); an ALTER left with no actions is
|
|
511
|
+
dropped downstream by the empty-ALTER guard. Runs BEFORE _rewrite_isjson_eq/_bare, so a
|
|
512
|
+
CHECK-context ISJSON is still an exp.Anonymous when this pass inspects it.
|
|
493
513
|
"""
|
|
494
514
|
if isinstance(node, exp.ColumnDef):
|
|
495
515
|
kept = [
|
|
@@ -497,40 +517,99 @@ def _drop_isjson_checks(node: exp.Expression) -> exp.Expression:
|
|
|
497
517
|
if not (
|
|
498
518
|
isinstance(c, exp.ColumnConstraint)
|
|
499
519
|
and isinstance(c.kind, exp.CheckColumnConstraint)
|
|
500
|
-
and
|
|
520
|
+
and _contains_isjson_call(c.kind)
|
|
501
521
|
)
|
|
502
522
|
]
|
|
503
523
|
node.set("constraints", kept)
|
|
504
524
|
return node
|
|
505
|
-
if isinstance(node,
|
|
525
|
+
if isinstance(node, exp.AddConstraint):
|
|
526
|
+
exprs = node.args.get("expressions") or []
|
|
527
|
+
kept = [e for e in exprs if not _contains_isjson_call(e)]
|
|
528
|
+
if not kept:
|
|
529
|
+
return None # every action was an ISJSON check → drop the whole ADD (ALTER-ACTIONLESS)
|
|
530
|
+
node.set("expressions", kept)
|
|
531
|
+
return node
|
|
532
|
+
if isinstance(node, exp.Constraint) and _contains_isjson_call(node):
|
|
506
533
|
return None
|
|
507
534
|
return node
|
|
508
535
|
|
|
509
536
|
|
|
510
537
|
def _fold_clustered_constraints(node: exp.Expression) -> exp.Expression:
|
|
511
|
-
"""`PRIMARY KEY CLUSTERED (cols)` / `UNIQUE NONCLUSTERED (cols)`
|
|
538
|
+
"""`PRIMARY KEY {CLUSTERED|NONCLUSTERED} (cols)` / `UNIQUE {CLUSTERED|NONCLUSTERED} (cols)`
|
|
539
|
+
→ PG `PRIMARY KEY (cols)` / `UNIQUE (cols)`.
|
|
512
540
|
|
|
513
541
|
SQL Server's CLUSTERED/NONCLUSTERED qualifier parses into a sibling
|
|
514
|
-
Clustered/NonClusteredColumnConstraint that holds the columns; PG has no such
|
|
515
|
-
|
|
542
|
+
Clustered/NonClusteredColumnConstraint that holds the columns; PG has no such qualifier, so
|
|
543
|
+
fold the columns into the PK/UNIQUE and drop the qualifier. BOTH qualifiers must be handled
|
|
544
|
+
for BOTH constraint kinds: sqlglot folds PK+CLUSTERED and UNIQUE+NONCLUSTERED on its own, but
|
|
545
|
+
leaks the cross pairs — PK+NONCLUSTERED emits the invalid `PRIMARY KEY, NONCLUSTERED (cols)`
|
|
546
|
+
(spurious comma) and UNIQUE+CLUSTERED keeps the CLUSTERED keyword. Treat the two qualifier
|
|
547
|
+
node types identically so all four combinations fold.
|
|
516
548
|
"""
|
|
517
|
-
|
|
518
|
-
|
|
549
|
+
_QUALIFIER = (exp.ClusteredColumnConstraint, exp.NonClusteredColumnConstraint)
|
|
550
|
+
|
|
551
|
+
def cols_of(qualifier):
|
|
552
|
+
return [o.this if isinstance(o, exp.Ordered) else o for o in (qualifier.this or [])]
|
|
519
553
|
|
|
520
|
-
# PK CLUSTERED: PrimaryKeyColumnConstraint +
|
|
554
|
+
# PK {CLUSTERED|NONCLUSTERED}: PrimaryKeyColumnConstraint + a qualifier node are siblings.
|
|
521
555
|
if isinstance(node, exp.Constraint):
|
|
522
556
|
exprs = node.args.get("expressions") or []
|
|
523
|
-
|
|
524
|
-
if
|
|
525
|
-
node.set("expressions", [exp.PrimaryKey(expressions=cols_of(
|
|
557
|
+
qualifier = next((e for e in exprs if isinstance(e, _QUALIFIER)), None)
|
|
558
|
+
if qualifier is not None and any(isinstance(e, exp.PrimaryKeyColumnConstraint) for e in exprs):
|
|
559
|
+
node.set("expressions", [exp.PrimaryKey(expressions=cols_of(qualifier))])
|
|
526
560
|
return node
|
|
527
561
|
|
|
528
|
-
# UNIQUE NONCLUSTERED: UniqueColumnConstraint wraps
|
|
529
|
-
if isinstance(node, exp.UniqueColumnConstraint) and isinstance(node.this,
|
|
562
|
+
# UNIQUE {CLUSTERED|NONCLUSTERED}: UniqueColumnConstraint wraps the qualifier holding the cols.
|
|
563
|
+
if isinstance(node, exp.UniqueColumnConstraint) and isinstance(node.this, _QUALIFIER):
|
|
530
564
|
node.set("this", exp.Schema(expressions=cols_of(node.this)))
|
|
531
565
|
return node
|
|
532
566
|
|
|
533
567
|
|
|
568
|
+
def _split_multi_add_constraint(node: exp.Expression) -> exp.Expression:
|
|
569
|
+
"""`ALTER TABLE t ADD CONSTRAINT a ..., CONSTRAINT b ...` → one `ADD` per constraint.
|
|
570
|
+
|
|
571
|
+
T-SQL lets a single `ADD` govern a comma-separated constraint list; PG requires `ADD`
|
|
572
|
+
before EACH action in a multi-action ALTER. sqlglot parses the list into ONE
|
|
573
|
+
AddConstraint holding N Constraint children and emits one `ADD` + comma list — PG rejects
|
|
574
|
+
the trailing bare `CONSTRAINT` with `syntax error at or near "CONSTRAINT"`, a SILENT
|
|
575
|
+
invalid emission (no unhandled). Split into one AddConstraint per constraint so the
|
|
576
|
+
generator repeats `ADD`. (sqlglot already repeats `ADD` for multiple ADD COLUMN actions;
|
|
577
|
+
only the constraint list leaks. Real occurrences: v5.49 Compaction two CHECKs on
|
|
578
|
+
AIAgentType, v5.24 KnowledgeHub CHECK + FK on Tag.)
|
|
579
|
+
"""
|
|
580
|
+
if not isinstance(node, exp.Alter):
|
|
581
|
+
return node
|
|
582
|
+
actions = node.args.get("actions") or []
|
|
583
|
+
new_actions = []
|
|
584
|
+
changed = False
|
|
585
|
+
for action in actions:
|
|
586
|
+
exprs = action.args.get("expressions") if isinstance(action, exp.AddConstraint) else None
|
|
587
|
+
if exprs and len(exprs) > 1:
|
|
588
|
+
new_actions.extend(exp.AddConstraint(expressions=[e.copy()]) for e in exprs)
|
|
589
|
+
changed = True
|
|
590
|
+
else:
|
|
591
|
+
new_actions.append(action)
|
|
592
|
+
if changed:
|
|
593
|
+
node.set("actions", new_actions)
|
|
594
|
+
return node
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def _sanitize_nested_comments(node: exp.Expression) -> exp.Expression:
|
|
598
|
+
"""Neutralize `/*` and `*/` inside a node's attached comment text.
|
|
599
|
+
|
|
600
|
+
sqlglot relocates T-SQL `--` line comments into inline `/* ... */` block comments on AST
|
|
601
|
+
nodes. PostgreSQL block comments NEST, so a comment body containing `/*` (e.g. `image/*`)
|
|
602
|
+
opens a nested comment and one containing `*/` closes early — either yields
|
|
603
|
+
`unterminated /* comment` and aborts the whole file. sqlglot 30.x sanitizes this at emit,
|
|
604
|
+
but the committed pin (27.18) emits comment text VERBATIM — a version-skew invalid
|
|
605
|
+
emission. Insert a space to break the sequence (identical to sqlglot 30's own behavior, so
|
|
606
|
+
the emitted SQL matches across versions). Comment text only — never touches SQL.
|
|
607
|
+
"""
|
|
608
|
+
if node.comments:
|
|
609
|
+
node.comments = [c.replace("/*", "/ *").replace("*/", "* /") for c in node.comments]
|
|
610
|
+
return node
|
|
611
|
+
|
|
612
|
+
|
|
534
613
|
def _column_fk_to_reference(node: exp.Expression) -> exp.Expression:
|
|
535
614
|
"""Column-level `CONSTRAINT fk FOREIGN KEY REFERENCES t(c)` → bare `REFERENCES t(c)`.
|
|
536
615
|
|
|
@@ -586,6 +665,18 @@ def _rewrite_boolean_defaults(node: exp.Expression) -> exp.Expression:
|
|
|
586
665
|
return node
|
|
587
666
|
|
|
588
667
|
|
|
668
|
+
def _strip_default_constraint_names(node: exp.Expression) -> exp.Expression:
|
|
669
|
+
"""T-SQL permits a NAME on a column default (`CONSTRAINT [DF_x] DEFAULT (75)`); PG does
|
|
670
|
+
NOT — it errors at `CONSTRAINT`. Strip the name from a DEFAULT column-constraint, leaving
|
|
671
|
+
a bare unnamed `DEFAULT (75)` (issue #3252 RC3, the inline-column form; the standalone
|
|
672
|
+
`ADD CONSTRAINT ... DEFAULT ... FOR col` form is handled by _transpile_default_constraint).
|
|
673
|
+
Named CHECK/FK/UNIQUE column constraints are valid PG and are left untouched."""
|
|
674
|
+
if isinstance(node, exp.ColumnConstraint) and isinstance(node.kind, exp.DefaultColumnConstraint) \
|
|
675
|
+
and node.args.get("this") is not None:
|
|
676
|
+
node.set("this", None)
|
|
677
|
+
return node
|
|
678
|
+
|
|
679
|
+
|
|
589
680
|
import re as _re
|
|
590
681
|
|
|
591
682
|
# `GO` is a batch separator (SSMS/sqlcmd tooling), not SQL — split on it before parsing.
|
|
@@ -609,10 +700,14 @@ def _first_keyword(text: str) -> str:
|
|
|
609
700
|
# fixed shape. We recognize the envelope structurally and transpile the real SQL
|
|
610
701
|
# *inside* it (predicates, INSERT bodies, descriptions) through the AST dialect.
|
|
611
702
|
|
|
612
|
-
# CodeGen object naming convention
|
|
613
|
-
# views, the CRUD/recompile sprocs, fn* functions,
|
|
614
|
-
# `mj codegen`, so their extended-property comments
|
|
615
|
-
|
|
703
|
+
# CodeGen object naming convention — MUST mirror MigrationStatementSplitter.CODEGEN_NAME exactly:
|
|
704
|
+
# views (vw*), the CRUD/recompile sprocs, fn* functions, and the CodeGen triggers trgUpdate/
|
|
705
|
+
# trgCreate/trgDelete — all regenerated by `mj codegen`, so their extended-property comments are
|
|
706
|
+
# skipped (the object itself is dropped). Bare `trg` was REMOVED in RC2 (issue #3252): it matched
|
|
707
|
+
# EVERY trigger name, so a HAND-written trigger (e.g. trgConversationDetail_AssignSequence) was
|
|
708
|
+
# misclassified as CodeGen and its comment silently skipped. The two classifiers must not drift —
|
|
709
|
+
# a `trg*` object that isn't trgUpdate/trgCreate/trgDelete is hand-authored, kept, and reported.
|
|
710
|
+
_CODEGEN_OBJECT_NAME = _re.compile(r"^(spCreate|spUpdate|spDelete|spRecompile|vw|fn|trgUpdate|trgCreate|trgDelete)", _re.IGNORECASE)
|
|
616
711
|
|
|
617
712
|
# EXEC sp_addextendedproperty @name=N'MS_Description', @value=N'...', ... ;
|
|
618
713
|
# The terminating `;` must be the one OUTSIDE the quoted args — description @values
|
|
@@ -752,16 +847,208 @@ class _IfExistsMatch:
|
|
|
752
847
|
return self._text[self._start:self._end] if key == 0 else self._groups[key]
|
|
753
848
|
|
|
754
849
|
|
|
850
|
+
def _scan_block_less_body(text: str, i: int) -> tuple[int, bool]:
|
|
851
|
+
"""From i (start of a block-less IF's single governed statement), scan atom-aware to the
|
|
852
|
+
statement's terminating top-level ';'. Returns (end, is_if_else):
|
|
853
|
+
- end: index just past ';' (or at a top-level ELSE, or len(text) if it runs to the end).
|
|
854
|
+
- is_if_else: True when a top-level ELSE (outside any CASE…END) is reached before ';'
|
|
855
|
+
— i.e. this is an IF … ELSE, which we don't model and must NOT capture.
|
|
856
|
+
CASE…END nesting is tracked so a CASE-expression ELSE does not falsely trip the bail."""
|
|
857
|
+
n = len(text)
|
|
858
|
+
case_depth = 0
|
|
859
|
+
while i < n:
|
|
860
|
+
j = _scan_atom(text, i)
|
|
861
|
+
if j != i:
|
|
862
|
+
i = j
|
|
863
|
+
continue
|
|
864
|
+
w = _WORD.match(text, i)
|
|
865
|
+
if w:
|
|
866
|
+
kw = w.group(0).upper()
|
|
867
|
+
if kw == "CASE":
|
|
868
|
+
case_depth += 1
|
|
869
|
+
elif kw == "END" and case_depth > 0:
|
|
870
|
+
case_depth -= 1
|
|
871
|
+
elif kw == "ELSE" and case_depth == 0:
|
|
872
|
+
return i, True
|
|
873
|
+
i = w.end()
|
|
874
|
+
continue
|
|
875
|
+
if text[i] == ";":
|
|
876
|
+
return i + 1, False
|
|
877
|
+
i += 1
|
|
878
|
+
return n, False
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _next_keyword(text: str, i: int) -> str:
|
|
882
|
+
"""The next SQL word at/after i, skipping whitespace and comments, uppercased ('' if none)."""
|
|
883
|
+
n = len(text)
|
|
884
|
+
while i < n:
|
|
885
|
+
if text[i].isspace():
|
|
886
|
+
i += 1
|
|
887
|
+
continue
|
|
888
|
+
j = _scan_atom(text, i)
|
|
889
|
+
if j != i and text[i] in ("-", "/"): # comments
|
|
890
|
+
i = j
|
|
891
|
+
continue
|
|
892
|
+
break
|
|
893
|
+
m = _WORD.match(text, i)
|
|
894
|
+
return m.group(0).upper() if m else ""
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def _next_keyword_pos(text: str, i: int) -> int:
|
|
898
|
+
"""Index of the next SQL word at/after i, skipping whitespace and comments (n if none)."""
|
|
899
|
+
n = len(text)
|
|
900
|
+
while i < n:
|
|
901
|
+
if text[i].isspace():
|
|
902
|
+
i += 1
|
|
903
|
+
continue
|
|
904
|
+
j = _scan_atom(text, i)
|
|
905
|
+
if j != i and text[i] in ("-", "/"):
|
|
906
|
+
i = j
|
|
907
|
+
continue
|
|
908
|
+
break
|
|
909
|
+
return i
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def _has_code_word(text: str, word: str) -> bool:
|
|
913
|
+
"""True when `word` appears as a standalone SQL keyword OUTSIDE strings / comments /
|
|
914
|
+
bracketed identifiers (atom-aware scan)."""
|
|
915
|
+
target = word.upper()
|
|
916
|
+
i, n = 0, len(text)
|
|
917
|
+
while i < n:
|
|
918
|
+
j = _scan_atom(text, i)
|
|
919
|
+
if j != i:
|
|
920
|
+
i = j
|
|
921
|
+
continue
|
|
922
|
+
m = _WORD.match(text, i)
|
|
923
|
+
if m:
|
|
924
|
+
if m.group(0).upper() == target:
|
|
925
|
+
return True
|
|
926
|
+
i = m.end()
|
|
927
|
+
continue
|
|
928
|
+
i += 1
|
|
929
|
+
return False
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
# ── Routine envelope (issue #3252 heavy smoke — the routine-body containment fix) ──────
|
|
933
|
+
# A hand-written CREATE PROCEDURE/FUNCTION/TRIGGER must be absorbed WHOLE, from raw text,
|
|
934
|
+
# BEFORE any parsing: sqlglot (both the pinned 27.x and 30.x, in different ways) fragments
|
|
935
|
+
# routine bodies at inner `;` boundaries, letting body statements ESCAPE the routine gap —
|
|
936
|
+
# a body UPDATE then emits as top-level migration SQL (executing against the whole table at
|
|
937
|
+
# apply time), a dangling `END` emits as `END;` (which PostgreSQL parses as COMMIT — ending
|
|
938
|
+
# Flyway's migration transaction mid-flight), and cursor fragments emit as invalid PG.
|
|
939
|
+
# The envelope reports the whole routine as ONE gap and resumes after its balanced END.
|
|
940
|
+
_ROUTINE_HEAD = _re.compile(r"CREATE\s+(?:OR\s+ALTER\s+)?(?P<kind>PROC(?:EDURE)?|FUNCTION|TRIGGER)\b", _re.IGNORECASE)
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
class _RoutineMatch:
|
|
944
|
+
"""Minimal match-alike for a routine envelope (start/end/kind/snippet)."""
|
|
945
|
+
__slots__ = ("_start", "_end", "kind", "snippet")
|
|
946
|
+
|
|
947
|
+
def __init__(self, start: int, end: int, kind: str, snippet: str):
|
|
948
|
+
self._start, self._end, self.kind, self.snippet = start, end, kind, snippet
|
|
949
|
+
|
|
950
|
+
def start(self) -> int:
|
|
951
|
+
return self._start
|
|
952
|
+
|
|
953
|
+
def end(self) -> int:
|
|
954
|
+
return self._end
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _find_routine_envelope(text: str, pos: int = 0) -> _RoutineMatch | None:
|
|
958
|
+
"""Find the next hand-routine header at a CODE position (never inside a string literal —
|
|
959
|
+
baseline metadata seeds embed 'CREATE PROCEDURE' in prompt-template strings) and span it
|
|
960
|
+
to its balanced `AS BEGIN … END` end. A body with no `AS BEGIN` (block-less T-SQL proc
|
|
961
|
+
body) absorbs the rest of the chunk — per T-SQL semantics the body IS the rest of the
|
|
962
|
+
batch. Returns None when no routine head exists at/after pos."""
|
|
963
|
+
i, n = pos, len(text)
|
|
964
|
+
while i < n:
|
|
965
|
+
j = _scan_atom(text, i)
|
|
966
|
+
if j != i:
|
|
967
|
+
i = j
|
|
968
|
+
continue
|
|
969
|
+
if text[i] not in ("C", "c"):
|
|
970
|
+
i += 1
|
|
971
|
+
continue
|
|
972
|
+
m = _ROUTINE_HEAD.match(text, i)
|
|
973
|
+
if not m:
|
|
974
|
+
i += 1
|
|
975
|
+
continue
|
|
976
|
+
end = _routine_body_end(text, m.end())
|
|
977
|
+
kind = m.group("kind").upper()
|
|
978
|
+
kind = "PROCEDURE" if kind == "PROC" else kind
|
|
979
|
+
snippet = " ".join(text[m.start():m.start() + 120].split())[:80]
|
|
980
|
+
return _RoutineMatch(m.start(), end, kind, snippet)
|
|
981
|
+
return None
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def _routine_body_end(text: str, head_end: int) -> int:
|
|
985
|
+
"""From just after a routine header, find the index past the body's balanced END.
|
|
986
|
+
Looks for the first `AS` whose next keyword is `BEGIN` (skipping `@p AS INT` param
|
|
987
|
+
forms, whose next word is a type, not BEGIN); no such `AS BEGIN` → the body is
|
|
988
|
+
block-less and runs to the end of the chunk (T-SQL batch semantics)."""
|
|
989
|
+
i, n = head_end, len(text)
|
|
990
|
+
while i < n:
|
|
991
|
+
j = _scan_atom(text, i)
|
|
992
|
+
if j != i:
|
|
993
|
+
i = j
|
|
994
|
+
continue
|
|
995
|
+
w = _WORD.match(text, i)
|
|
996
|
+
if not w:
|
|
997
|
+
i += 1
|
|
998
|
+
continue
|
|
999
|
+
if w.group(0).upper() == "AS" and _next_keyword(text, w.end()) == "BEGIN":
|
|
1000
|
+
k = _next_keyword_pos(text, w.end()) # start of BEGIN
|
|
1001
|
+
bw = _WORD.match(text, k) # the BEGIN word
|
|
1002
|
+
_, block_end = _match_block_end(text, bw.end())
|
|
1003
|
+
return block_end if block_end >= 0 else n # unterminated → absorb rest
|
|
1004
|
+
i = w.end()
|
|
1005
|
+
return n
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _atom_masked_spans(text: str) -> list[tuple[int, int]]:
|
|
1009
|
+
"""(start, end) spans of `text` occupied by a string literal, comment, or quoted/bracketed
|
|
1010
|
+
identifier — everything `_scan_atom` skips as a non-code atom. A regex head that STARTS
|
|
1011
|
+
inside one of these is not real SQL (e.g. the phrase `IF EXISTS (SELECT ...)` mentioned in a
|
|
1012
|
+
seeded template/prompt body or an example-SQL literal) and must never be mistaken for a
|
|
1013
|
+
statement head — doing so shattered the surrounding INSERT and fabricated a live DO-block
|
|
1014
|
+
(issue #3252 RC1/RC2 regression). Spans are returned sorted and non-overlapping."""
|
|
1015
|
+
spans: list[tuple[int, int]] = []
|
|
1016
|
+
i, n = 0, len(text)
|
|
1017
|
+
while i < n:
|
|
1018
|
+
j = _scan_atom(text, i)
|
|
1019
|
+
if j != i:
|
|
1020
|
+
spans.append((i, j))
|
|
1021
|
+
i = j
|
|
1022
|
+
else:
|
|
1023
|
+
i += 1
|
|
1024
|
+
return spans
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def _pos_in_spans(pos: int, spans: list[tuple[int, int]]) -> bool:
|
|
1028
|
+
"""True if pos falls within any (sorted, non-overlapping) masked span from _atom_masked_spans."""
|
|
1029
|
+
for s, e in spans:
|
|
1030
|
+
if s > pos:
|
|
1031
|
+
return False
|
|
1032
|
+
if pos < e:
|
|
1033
|
+
return True
|
|
1034
|
+
return False
|
|
1035
|
+
|
|
1036
|
+
|
|
755
1037
|
def _find_if_exists_begin(text: str, pos: int = 0) -> _IfExistsMatch | None:
|
|
756
|
-
"""Find the next `IF [NOT] EXISTS (<select>) BEGIN <body> END` block
|
|
1038
|
+
"""Find the next `IF [NOT] EXISTS (<select>) BEGIN <body> END` block — OR the block-less
|
|
1039
|
+
`IF [NOT] EXISTS (<select>) <single-statement>;` form (issue #3252 RC1) — at/after pos.
|
|
1040
|
+
Heads inside a string literal or comment are skipped (see _atom_masked_spans)."""
|
|
1041
|
+
masked = _atom_masked_spans(text)
|
|
757
1042
|
for head in _IF_EXISTS_HEAD.finditer(text, pos):
|
|
1043
|
+
if _pos_in_spans(head.start(), masked):
|
|
1044
|
+
continue # the phrase lives inside a string literal / comment — not a statement
|
|
758
1045
|
cond_close = _match_paren(text, head.end() - 1)
|
|
759
1046
|
if cond_close < 0:
|
|
760
1047
|
continue
|
|
761
1048
|
cond = text[head.end():cond_close - 1]
|
|
762
1049
|
if not _re.match(r"\s*SELECT\b", _strip_leading_sql_comments(cond), _re.IGNORECASE):
|
|
763
1050
|
continue
|
|
764
|
-
#
|
|
1051
|
+
# Skip whitespace/comments after the condition to the first real token.
|
|
765
1052
|
i, n = cond_close, len(text)
|
|
766
1053
|
while i < n:
|
|
767
1054
|
if text[i].isspace():
|
|
@@ -773,12 +1060,21 @@ def _find_if_exists_begin(text: str, pos: int = 0) -> _IfExistsMatch | None:
|
|
|
773
1060
|
continue
|
|
774
1061
|
break
|
|
775
1062
|
m = _WORD.match(text, i)
|
|
776
|
-
if
|
|
1063
|
+
if m and m.group(0).upper() == "BEGIN":
|
|
1064
|
+
body_end, block_end = _match_block_end(text, m.end())
|
|
1065
|
+
if body_end < 0:
|
|
1066
|
+
continue
|
|
1067
|
+
return _IfExistsMatch(text, head.start(), block_end, head.group("neg"), cond, text[m.end():body_end])
|
|
1068
|
+
# Block-less form: capture the single governed statement (to its terminating ';').
|
|
1069
|
+
# IF/ELSE is not modeled — bail so it falls to the plain path, where the 1b If/IfBlock
|
|
1070
|
+
# guard reports it and never silently drops it. Detect ELSE both mid-statement (the
|
|
1071
|
+
# governed statement has no trailing ';' before ELSE) and after a terminating ';'.
|
|
1072
|
+
if not m or m.group(0).upper() in ("ELSE", "END"):
|
|
777
1073
|
continue
|
|
778
|
-
|
|
779
|
-
if
|
|
1074
|
+
stmt_end, is_if_else = _scan_block_less_body(text, i)
|
|
1075
|
+
if is_if_else or _next_keyword(text, stmt_end) == "ELSE":
|
|
780
1076
|
continue
|
|
781
|
-
return _IfExistsMatch(text, head.start(),
|
|
1077
|
+
return _IfExistsMatch(text, head.start(), stmt_end, head.group("neg"), cond, text[i:stmt_end])
|
|
782
1078
|
return None
|
|
783
1079
|
|
|
784
1080
|
|
|
@@ -1054,30 +1350,33 @@ def _transpile_sp_dropextendedproperty(args: str) -> str | None:
|
|
|
1054
1350
|
|
|
1055
1351
|
|
|
1056
1352
|
def _split_top_level_statements(sql: str) -> list[str]:
|
|
1057
|
-
"""Split SQL on top-level `;`, ignoring semicolons inside
|
|
1058
|
-
|
|
1059
|
-
|
|
1353
|
+
"""Split SQL on top-level `;`, ignoring semicolons inside strings, `--`/`/* */` comments,
|
|
1354
|
+
`[bracketed]`/`"quoted"` identifiers, and BEGIN…END / CASE…END blocks (BEGIN TRAN pairs
|
|
1355
|
+
with COMMIT, not END, so it does not open a block). Used to (a) recover good statements
|
|
1356
|
+
when a whole-gap parse fails on one poison statement (a poison must not drop its neighbors)
|
|
1357
|
+
and (b) measure the true top-level statement count so a MERGED whole-parse — sqlglot 30.x
|
|
1358
|
+
absorbing the statement that FOLLOWS a block-less IF into the IF node (issue #3252 heavy
|
|
1359
|
+
smoke) — can be detected and re-split. Block-awareness keeps a legitimate multi-statement
|
|
1360
|
+
BEGIN…END intact instead of shattering it at its interior `;`."""
|
|
1060
1361
|
out: list[str] = []
|
|
1061
1362
|
buf: list[str] = []
|
|
1062
|
-
i, n,
|
|
1363
|
+
i, n, depth = 0, len(sql), 0
|
|
1063
1364
|
while i < n:
|
|
1064
|
-
|
|
1065
|
-
if
|
|
1066
|
-
buf.append(c)
|
|
1067
|
-
if c == "'":
|
|
1068
|
-
if i + 1 < n and sql[i + 1] == "'": # escaped '' inside string
|
|
1069
|
-
buf.append("'"); i += 2; continue
|
|
1070
|
-
in_str = False
|
|
1071
|
-
i += 1; continue
|
|
1072
|
-
if c == "'":
|
|
1073
|
-
in_str = True; buf.append(c); i += 1; continue
|
|
1074
|
-
if c == "-" and i + 1 < n and sql[i + 1] == "-": # line comment
|
|
1075
|
-
j = sql.find("\n", i); j = n if j < 0 else j
|
|
1076
|
-
buf.append(sql[i:j]); i = j; continue
|
|
1077
|
-
if c == "/" and i + 1 < n and sql[i + 1] == "*": # block comment
|
|
1078
|
-
j = sql.find("*/", i); j = n if j < 0 else j + 2
|
|
1365
|
+
j = _scan_atom(sql, i)
|
|
1366
|
+
if j != i: # string / comment / [bracketed] / "quoted" — copy verbatim
|
|
1079
1367
|
buf.append(sql[i:j]); i = j; continue
|
|
1080
|
-
|
|
1368
|
+
w = _WORD.match(sql, i)
|
|
1369
|
+
if w:
|
|
1370
|
+
kw = w.group(0).upper()
|
|
1371
|
+
if kw == "BEGIN" and _peek_word(sql, w.end()) not in ("TRAN", "TRANSACTION"):
|
|
1372
|
+
depth += 1
|
|
1373
|
+
elif kw == "CASE":
|
|
1374
|
+
depth += 1
|
|
1375
|
+
elif kw == "END" and depth > 0:
|
|
1376
|
+
depth -= 1
|
|
1377
|
+
buf.append(w.group(0)); i = w.end(); continue
|
|
1378
|
+
c = sql[i]
|
|
1379
|
+
if c == ";" and depth == 0:
|
|
1081
1380
|
buf.append(";"); out.append("".join(buf)); buf = []; i += 1; continue
|
|
1082
1381
|
buf.append(c); i += 1
|
|
1083
1382
|
tail = "".join(buf)
|
|
@@ -1086,18 +1385,44 @@ def _split_top_level_statements(sql: str) -> list[str]:
|
|
|
1086
1385
|
return out
|
|
1087
1386
|
|
|
1088
1387
|
|
|
1388
|
+
def _has_sql_content(piece: str) -> bool:
|
|
1389
|
+
"""True if `piece` holds any non-whitespace, non-comment SQL. Used to count REAL statements
|
|
1390
|
+
(a trailing comment-only or blank split fragment must not inflate the count and falsely
|
|
1391
|
+
trip the merged-whole-parse fallback)."""
|
|
1392
|
+
i, n = 0, len(piece)
|
|
1393
|
+
while i < n:
|
|
1394
|
+
c = piece[i]
|
|
1395
|
+
if c.isspace():
|
|
1396
|
+
i += 1; continue
|
|
1397
|
+
if c == "-" and piece.startswith("--", i):
|
|
1398
|
+
j = piece.find("\n", i); i = n if j < 0 else j + 1; continue
|
|
1399
|
+
if c == "/" and piece.startswith("/*", i):
|
|
1400
|
+
j = piece.find("*/", i); i = n if j < 0 else j + 2; continue
|
|
1401
|
+
return True
|
|
1402
|
+
return False
|
|
1403
|
+
|
|
1404
|
+
|
|
1089
1405
|
def _parse_resilient(protected: str) -> list[tuple[exp.Expression | None, str]]:
|
|
1090
1406
|
"""Parse a SQL chunk into (statement, raw_text) pairs. Fast path: one `sqlglot.parse`.
|
|
1091
|
-
|
|
1092
|
-
|
|
1407
|
+
Fall back to per-statement parsing in TWO cases, so no statement is lost:
|
|
1408
|
+
1. The whole-parse raised — a single unparseable statement must only drop itself, not
|
|
1409
|
+
the valid DDL around it (returns (None, raw) for the failures).
|
|
1410
|
+
2. The whole-parse SILENTLY MERGED statements — sqlglot 30.x absorbs the statement that
|
|
1411
|
+
follows a block-less `IF <cond> <stmt>` into the IF node, so an unconditional rider
|
|
1412
|
+
(e.g. an `ALTER … ADD CONSTRAINT` after an `IF @c … EXEC(…)` guard) neither emits nor
|
|
1413
|
+
is reported. Detected by comparing the whole-parse statement count against the
|
|
1414
|
+
block-aware top-level split: fewer statements than real `;`-delimited units means a
|
|
1415
|
+
merge happened. (27.x splits these correctly, so this is a no-op there.)
|
|
1416
|
+
Per-piece re-parsing restores the true statement boundaries in both cases."""
|
|
1417
|
+
pieces = [p for p in _split_top_level_statements(protected) if _has_sql_content(p)]
|
|
1093
1418
|
try:
|
|
1094
|
-
|
|
1419
|
+
fast = [s for s in sqlglot.parse(protected, read="tsql") if s is not None]
|
|
1420
|
+
if len(fast) >= len(pieces): # no merge — trust the (faster) whole-parse
|
|
1421
|
+
return [(s, "") for s in fast]
|
|
1095
1422
|
except Exception: # noqa: BLE001
|
|
1096
1423
|
pass
|
|
1097
1424
|
results: list[tuple[exp.Expression | None, str]] = []
|
|
1098
|
-
for piece in
|
|
1099
|
-
if not piece.strip():
|
|
1100
|
-
continue
|
|
1425
|
+
for piece in pieces:
|
|
1101
1426
|
try:
|
|
1102
1427
|
for s in sqlglot.parse(piece, read="tsql"):
|
|
1103
1428
|
if s is not None:
|
|
@@ -1107,25 +1432,111 @@ def _parse_resilient(protected: str) -> list[tuple[exp.Expression | None, str]]:
|
|
|
1107
1432
|
return results
|
|
1108
1433
|
|
|
1109
1434
|
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1435
|
+
# T-SQL `ALTER TABLE t WITH [NO]CHECK ADD CONSTRAINT …` — the enforcement toggle has no PG
|
|
1436
|
+
# form. Stripped pre-parse (see _transpile_plain) so `WITH CHECK` (which otherwise emits the
|
|
1437
|
+
# invalid `… WITH CHECK ADD …`) and `WITH NOCHECK` (an opaque Command → unhandled) both become
|
|
1438
|
+
# a plain, validating `ADD CONSTRAINT …`. The `(?=ADD\b)` lookahead keeps it from touching a
|
|
1439
|
+
# view's `WITH CHECK OPTION` (there the next token is OPTION, not ADD).
|
|
1440
|
+
_WITH_CHECK_ADD = _re.compile(r"\bWITH\s+(?:NOCHECK|CHECK)\s+(?=ADD\b)", _re.IGNORECASE)
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
def _has_computed_column(stmt: exp.Expression) -> bool:
|
|
1444
|
+
"""True if `stmt` declares a computed/generated column (T-SQL `col AS (expr) [PERSISTED]`).
|
|
1445
|
+
Such columns cannot be transpiled mechanically: sqlglot emits `GENERATED ALWAYS AS (...)
|
|
1446
|
+
STORED` WITHOUT the PG-required column type (invalid), and a non-persisted T-SQL computed
|
|
1447
|
+
column has no STORED equivalent at all. Callers report the statement rather than emit it."""
|
|
1448
|
+
for cd in stmt.find_all(exp.ColumnDef):
|
|
1449
|
+
for c in cd.args.get("constraints") or []:
|
|
1450
|
+
if isinstance(c.args.get("kind"), exp.ComputedColumnConstraint):
|
|
1451
|
+
return True
|
|
1452
|
+
return False
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
def _transpile_plain(sql: str, pretty: bool = False) -> tuple[str, list[dict], list[dict]]:
|
|
1456
|
+
"""Transpile a chunk of regular SQL via the AST dialect; report unparseable bits.
|
|
1457
|
+
|
|
1458
|
+
Returns (sql, unhandled, dropped). `dropped` records every INTENTIONAL drop (batch-control
|
|
1459
|
+
noise, swallowed routine `END`, statement-level RAISERROR, actionless ALTER, …) so that
|
|
1460
|
+
accounting reconciles: parsed == emitted + unhandled + dropped (issue #3252 1d). The
|
|
1461
|
+
reconciliation is SOFT — a mismatch appends an ACCOUNTING-LEAK gap to `unhandled`, it
|
|
1462
|
+
never raises (a raise would crash the transpiler and, via the CLI catch, lose all
|
|
1463
|
+
artifacts — the exact RC3 pathology this fix eliminates)."""
|
|
1464
|
+
out, unhandled, dropped = [], [], []
|
|
1465
|
+
parsed = 0
|
|
1113
1466
|
protected = sql.replace(FLYWAY_MACRO, FLYWAY_SENTINEL)
|
|
1467
|
+
# Strip the SQL Server `WITH [NO]CHECK` enforcement toggle before `ADD CONSTRAINT` — it has
|
|
1468
|
+
# no PG equivalent and otherwise leaks as invalid `… WITH CHECK ADD …` (see _WITH_CHECK_ADD).
|
|
1469
|
+
# ATOM-AWARE: rewrite the toggle ONLY at a real code position. The identical phrase occurring
|
|
1470
|
+
# inside a string literal (e.g. a seeded `N'… WITH CHECK ADD …'`) or a comment is data/prose,
|
|
1471
|
+
# not the toggle — stripping it there silently corrupts emitted content (issue #3252: never
|
|
1472
|
+
# silently alter output). Matches whose head sits in a masked atom are left verbatim.
|
|
1473
|
+
_wca_spans = _atom_masked_spans(protected)
|
|
1474
|
+
protected = _WITH_CHECK_ADD.sub(
|
|
1475
|
+
lambda m: m.group(0) if _pos_in_spans(m.start(), _wca_spans) else "",
|
|
1476
|
+
protected,
|
|
1477
|
+
)
|
|
1114
1478
|
# Set after reporting a CREATE PROCEDURE/FUNCTION/TRIGGER: the routine's closing
|
|
1115
1479
|
# `END` often parses as its own dangling statement — it belongs to the routine we
|
|
1116
1480
|
# just reported, not to a new gap.
|
|
1117
1481
|
swallow_routine_end = False
|
|
1118
1482
|
for stmt, raw in _parse_resilient(protected):
|
|
1483
|
+
parsed += 1
|
|
1119
1484
|
if swallow_routine_end:
|
|
1120
1485
|
swallow_routine_end = False
|
|
1121
1486
|
tail = (raw or (stmt.sql(dialect="tsql") if stmt is not None else "")).strip().rstrip(";").strip()
|
|
1122
1487
|
if tail.upper() == "END":
|
|
1488
|
+
dropped.append({"kind": "ROUTINE-END", "snippet": tail[:80]})
|
|
1123
1489
|
continue
|
|
1124
1490
|
if stmt is None:
|
|
1125
1491
|
unhandled.append({"kind": "parse-error", "snippet": raw.strip()[:80]})
|
|
1126
1492
|
continue
|
|
1493
|
+
# A T-SQL `IF …` statement that the IF-EXISTS envelope did NOT capture (e.g. a
|
|
1494
|
+
# block-less IF/ELSE — issue #3252 RC1) reaches here as exp.If/exp.IfBlock. sqlglot's
|
|
1495
|
+
# PG generator emits an EMPTY string for these, which is a SILENT DROP (the six bare
|
|
1496
|
+
# `;` of the original bug). Report it as a gap; never let it emit nothing.
|
|
1497
|
+
if isinstance(stmt, _IF_NODE_TYPES):
|
|
1498
|
+
txt = stmt.sql(dialect="tsql")
|
|
1499
|
+
unhandled.append({"kind": "IF-BLOCK", "snippet": txt[:80]})
|
|
1500
|
+
continue
|
|
1127
1501
|
# Standalone seed of schema-derived metadata → drop; CodeGen regenerates it.
|
|
1502
|
+
# (The _METADATA_TABLES matcher is intentionally disabled today; instrumented for
|
|
1503
|
+
# accounting completeness should it ever be re-enabled.)
|
|
1128
1504
|
if isinstance(stmt, exp.Insert) and _METADATA_TABLES.search(stmt.sql(dialect="tsql")):
|
|
1505
|
+
dropped.append({"kind": "METADATA-INSERT", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1506
|
+
continue
|
|
1507
|
+
# Hand-written routines MUST be detected BEFORE the Declare/Parameter branch below:
|
|
1508
|
+
# a routine body is full of `@params`, and version-dependently the whole routine can
|
|
1509
|
+
# carry exp.Parameter nodes (the pinned 27.x models proc params that way; 30.x does
|
|
1510
|
+
# not). If the Parameter branch caught it first, the routine would be reported WITHOUT
|
|
1511
|
+
# `swallow_routine_end`, leaving the resilient split's dangling `END` as a spurious
|
|
1512
|
+
# second gap (27.x) or a bogus emitted `END;` (30.x). Two forms:
|
|
1513
|
+
# (a) well-parsed exp.Create PROCEDURE/FUNCTION/TRIGGER — the dedicated branch just
|
|
1514
|
+
# below (hoisted above the Parameter branch for exactly this reason);
|
|
1515
|
+
# (b) a routine header sqlglot mis-parsed as an opaque Command (e.g. some
|
|
1516
|
+
# `CREATE TRIGGER … ON …` forms) — the text-based fallback here.
|
|
1517
|
+
if not isinstance(stmt, exp.Create):
|
|
1518
|
+
head_txt = _strip_leading_sql_comments(stmt.sql(dialect="tsql"))
|
|
1519
|
+
if _re.match(r"^\s*CREATE\s+(?:OR\s+ALTER\s+)?(?:PROC(?:EDURE)?|FUNCTION|TRIGGER)\b", head_txt, _re.IGNORECASE):
|
|
1520
|
+
unhandled.append({"kind": _first_keyword(head_txt), "snippet": head_txt[:80]})
|
|
1521
|
+
swallow_routine_end = True
|
|
1522
|
+
continue
|
|
1523
|
+
# Hand-written routines: a T-SQL PROCEDURE/FUNCTION/TRIGGER body cannot be
|
|
1524
|
+
# transpiled mechanically (parameter syntax, control flow, and the body itself
|
|
1525
|
+
# are all T-SQL) — naive emission produces invalid PG like `$x INT AS BEGIN …`.
|
|
1526
|
+
# The classifier flags these files needs-hand-authoring; here we report the
|
|
1527
|
+
# routine so it lands in the gap comments instead of half-translated output.
|
|
1528
|
+
if isinstance(stmt, exp.Create) and (stmt.args.get("kind") or "").upper() in (
|
|
1529
|
+
"PROCEDURE",
|
|
1530
|
+
"FUNCTION",
|
|
1531
|
+
"TRIGGER",
|
|
1532
|
+
):
|
|
1533
|
+
txt = stmt.sql(dialect="tsql")
|
|
1534
|
+
name = stmt.find(exp.Table)
|
|
1535
|
+
unhandled.append({
|
|
1536
|
+
"kind": f"CREATE-{(stmt.args.get('kind') or '').upper()}",
|
|
1537
|
+
"snippet": (name.sql(dialect="tsql") + " — " if name else "") + txt[:80],
|
|
1538
|
+
})
|
|
1539
|
+
swallow_routine_end = True
|
|
1129
1540
|
continue
|
|
1130
1541
|
# T-SQL procedural glue with no standalone PG equivalent — DECLARE @v / SET @v /
|
|
1131
1542
|
# SELECT @v = ... / IF @v ... EXEC('...'). In Category-B (regular DDL/DML) these
|
|
@@ -1150,23 +1561,21 @@ def _transpile_plain(sql: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
|
1150
1561
|
txt = stmt.sql(dialect="tsql")
|
|
1151
1562
|
unhandled.append({"kind": "UPDATE-OUTER-JOIN", "snippet": txt[:80]})
|
|
1152
1563
|
continue
|
|
1153
|
-
#
|
|
1154
|
-
#
|
|
1155
|
-
#
|
|
1156
|
-
#
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
"
|
|
1161
|
-
|
|
1162
|
-
)
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
})
|
|
1169
|
-
swallow_routine_end = True
|
|
1564
|
+
# T-SQL `DELETE TOP (n) FROM t` misparses on BOTH versions into a spurious multi-table
|
|
1565
|
+
# DELETE (`DELETE "TOP" AS _t0(n) FROM t`) — invalid PG with no mechanical rewrite (PG
|
|
1566
|
+
# would need a ctid/CTE form). The signature is a DELETE carrying a `tables` list (PG puts
|
|
1567
|
+
# the target in `this`, not `tables`) whose entry is the swallowed `TOP` keyword.
|
|
1568
|
+
if isinstance(stmt, exp.Delete) and any(
|
|
1569
|
+
isinstance(t, exp.Table) and (t.name or "").upper() == "TOP"
|
|
1570
|
+
for t in (stmt.args.get("tables") or [])):
|
|
1571
|
+
unhandled.append({"kind": "DELETE-TOP", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1572
|
+
continue
|
|
1573
|
+
# Computed/generated columns (`col AS (expr) [PERSISTED]`) can't be transpiled — report
|
|
1574
|
+
# (would emit a type-less `GENERATED ALWAYS AS (...) STORED`; see _has_computed_column).
|
|
1575
|
+
# Only CREATE/ALTER declare columns — gate on type so the subtree walk skips the huge
|
|
1576
|
+
# metadata INSERTs in baseline dumps (thousands of VALUES rows) it could never match.
|
|
1577
|
+
if isinstance(stmt, (exp.Create, exp.Alter)) and _has_computed_column(stmt):
|
|
1578
|
+
unhandled.append({"kind": "COMPUTED-COLUMN", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1170
1579
|
continue
|
|
1171
1580
|
if isinstance(stmt, exp.Command):
|
|
1172
1581
|
# sqlglot may glom a preceding comment onto the Command (`/* note */ ALTER …`),
|
|
@@ -1185,19 +1594,30 @@ def _transpile_plain(sql: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
|
1185
1594
|
continue
|
|
1186
1595
|
# SQL Server batch-control noise — not needed on PG, drop silently.
|
|
1187
1596
|
if _re.match(r"^\s*(BEGIN\s+TRY|END\s+TRY|BEGIN\s+CATCH|END\s+CATCH|SET\s+NOEXEC|GO)\b", txt, _re.IGNORECASE):
|
|
1597
|
+
dropped.append({"kind": "BATCH-CONTROL", "snippet": txt[:80]})
|
|
1188
1598
|
continue
|
|
1189
1599
|
unhandled.append({"kind": _first_keyword(txt), "snippet": txt[:80]})
|
|
1190
1600
|
continue
|
|
1601
|
+
# T-SQL `SET IDENTITY_INSERT t ON/OFF` — no PG equivalent (PG uses OVERRIDING SYSTEM
|
|
1602
|
+
# VALUE per-INSERT). `…ON` lands as an unhandled Command; `…OFF` parses to exp.Set and
|
|
1603
|
+
# would emit invalid `SET "IDENTITY_INSERT" = t AS "OFF"`. Report (must precede the
|
|
1604
|
+
# SET-NOISE drop below so it lands as a gap, not a silent drop).
|
|
1605
|
+
if isinstance(stmt, exp.Set) and _re.search(
|
|
1606
|
+
r"\bIDENTITY_INSERT\b", stmt.sql(dialect="tsql"), _re.IGNORECASE):
|
|
1607
|
+
unhandled.append({"kind": "SET-IDENTITY-INSERT", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1608
|
+
continue
|
|
1191
1609
|
# SS session/batch-control SETs have no PG equivalent and error as unrecognized
|
|
1192
1610
|
# config params — drop them (NOEXEC, NOCOUNT, XACT_ABORT, QUOTED_IDENTIFIER, ANSI_*).
|
|
1193
1611
|
if isinstance(stmt, exp.Set) and _re.search(
|
|
1194
1612
|
r"\b(NOEXEC|NOCOUNT|XACT_ABORT|QUOTED_IDENTIFIER|ANSI_NULLS|ANSI_PADDING|ANSI_WARNINGS|"
|
|
1195
1613
|
r"ARITHABORT|CONCAT_NULL_YIELDS_NULL|NUMERIC_ROUNDABORT)\b",
|
|
1196
1614
|
stmt.sql(dialect="tsql"), _re.IGNORECASE):
|
|
1615
|
+
dropped.append({"kind": "SET-NOISE", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1197
1616
|
continue
|
|
1198
1617
|
# RAISERROR(...) at statement level is invalid PG outside a function — drop it
|
|
1199
1618
|
# (inside an IF…BEGIN guard it is handled as RAISE EXCEPTION by the DO-block path).
|
|
1200
1619
|
if isinstance(stmt, exp.Anonymous) and (stmt.name or "").upper() == "RAISERROR":
|
|
1620
|
+
dropped.append({"kind": "RAISERROR", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1201
1621
|
continue
|
|
1202
1622
|
# `ALTER TABLE t ALTER COLUMN c <type>` with NO nullability spec parses cleanly
|
|
1203
1623
|
# (unlike the `… NULL`/`… NOT NULL` forms, which land as opaque Commands). Route
|
|
@@ -1210,28 +1630,59 @@ def _transpile_plain(sql: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
|
1210
1630
|
if ac is not None:
|
|
1211
1631
|
out.append(ac)
|
|
1212
1632
|
continue
|
|
1213
|
-
|
|
1214
|
-
|
|
1633
|
+
# DROP kind normalization / gating (issue #3252 heavy smoke):
|
|
1634
|
+
# • `DROP PROC` — sqlglot keeps the T-SQL abbreviation, which is invalid PG. Spell it
|
|
1635
|
+
# out to PROCEDURE. (The ONE real invalid-PG emission in the v5 ledger — V202605091143.)
|
|
1636
|
+
# • `DROP TRIGGER` — PG requires an `ON <table>` clause; sqlglot (30.x) emits it WITHOUT
|
|
1637
|
+
# one (invalid). 27.x parses it as an opaque Command (already unhandled). Report so
|
|
1638
|
+
# both versions converge on a gap rather than emit the ON-less form.
|
|
1639
|
+
if isinstance(stmt, exp.Drop):
|
|
1640
|
+
dk = (stmt.args.get("kind") or "").upper()
|
|
1641
|
+
if dk == "PROC":
|
|
1642
|
+
stmt.set("kind", "PROCEDURE")
|
|
1643
|
+
elif dk == "TRIGGER":
|
|
1644
|
+
unhandled.append({"kind": "DROP-TRIGGER", "snippet": stmt.sql(dialect="tsql")[:80]})
|
|
1645
|
+
continue
|
|
1646
|
+
if isinstance(stmt, exp.Create) and (stmt.args.get("kind") or "").upper() in (
|
|
1647
|
+
"NONCLUSTERED INDEX", "CLUSTERED INDEX"):
|
|
1648
|
+
stmt.set("kind", "INDEX") # PG has no CLUSTERED/NONCLUSTERED qualifier
|
|
1215
1649
|
stmt = _rewrite_boolean_int_comparisons(stmt)
|
|
1650
|
+
# copy=False mutates the parsed tree IN PLACE across all passes. Each pass otherwise
|
|
1651
|
+
# deep-copies the whole statement AST (sqlglot's transform default), so a 15-pass chain
|
|
1652
|
+
# was 15 full tree deep-copies per statement — the dominant cost on 50 MB baselines
|
|
1653
|
+
# (deepcopy alone was ~40% of total runtime; see the perf note in _next_match). The
|
|
1654
|
+
# parsed `stmt` is used once here and discarded, so in-place mutation is safe, and the
|
|
1655
|
+
# pass ordering is unchanged: each pass still sees the prior pass's edits.
|
|
1216
1656
|
stmt = (
|
|
1217
|
-
stmt.transform(_rewrite_update_from_alias) # restructures UPDATE before other rewrites
|
|
1218
|
-
.transform(_rewrite_insert_booleans) # seed-INSERT 1/0 → TRUE/FALSE for bit cols
|
|
1219
|
-
.transform(_fix_misparsed_table_constraint)
|
|
1220
|
-
.transform(_rewrite_functions)
|
|
1221
|
-
.transform(_rewrite_boolean_defaults)
|
|
1222
|
-
.transform(
|
|
1223
|
-
.transform(
|
|
1224
|
-
.transform(
|
|
1225
|
-
.transform(
|
|
1226
|
-
.transform(
|
|
1227
|
-
.transform(
|
|
1228
|
-
.transform(
|
|
1229
|
-
.transform(
|
|
1230
|
-
.transform(
|
|
1657
|
+
stmt.transform(_rewrite_update_from_alias, copy=False) # restructures UPDATE before other rewrites
|
|
1658
|
+
.transform(_rewrite_insert_booleans, copy=False) # seed-INSERT 1/0 → TRUE/FALSE for bit cols
|
|
1659
|
+
.transform(_fix_misparsed_table_constraint, copy=False)
|
|
1660
|
+
.transform(_rewrite_functions, copy=False)
|
|
1661
|
+
.transform(_rewrite_boolean_defaults, copy=False)
|
|
1662
|
+
.transform(_strip_default_constraint_names, copy=False) # PG has no named column defaults
|
|
1663
|
+
.transform(_rewrite_string_concat, copy=False)
|
|
1664
|
+
.transform(_strip_national, copy=False)
|
|
1665
|
+
.transform(_strip_collate, copy=False)
|
|
1666
|
+
.transform(_drop_isjson_checks, copy=False) # drop ISJSON CHECK constraints BEFORE…
|
|
1667
|
+
.transform(_rewrite_isjson_eq, copy=False) # …rewriting surviving ISJSON predicates (WHERE)
|
|
1668
|
+
.transform(_rewrite_isjson_bare, copy=False)
|
|
1669
|
+
.transform(_fold_clustered_constraints, copy=False)
|
|
1670
|
+
.transform(_strip_nulls_ordering, copy=False)
|
|
1671
|
+
.transform(_column_fk_to_reference, copy=False)
|
|
1672
|
+
.transform(_split_multi_add_constraint, copy=False) # PG needs ADD per constraint; run last
|
|
1673
|
+
.transform(_sanitize_nested_comments, copy=False) # break /* */ in comments (pin version-skew)
|
|
1231
1674
|
)
|
|
1232
1675
|
# An ALTER TABLE whose only action was dropped (e.g. an ISJSON ADD CONSTRAINT)
|
|
1233
1676
|
# is left actionless — emitting bare `ALTER TABLE x` is a PG syntax error. Skip.
|
|
1677
|
+
# NOTE: do NOT render the actionless ALTER to T-SQL for the snippet — sqlglot's
|
|
1678
|
+
# tsql alter_sql does actions[0] and raises IndexError on an empty action list.
|
|
1679
|
+
# Use the (safe) table name instead.
|
|
1234
1680
|
if isinstance(stmt, exp.Alter) and not stmt.args.get("actions"):
|
|
1681
|
+
tbl = stmt.find(exp.Table)
|
|
1682
|
+
dropped.append({
|
|
1683
|
+
"kind": "ALTER-ACTIONLESS",
|
|
1684
|
+
"snippet": (tbl.sql(dialect="tsql") if tbl is not None else "ALTER TABLE")[:80],
|
|
1685
|
+
})
|
|
1235
1686
|
continue
|
|
1236
1687
|
# Behavioral self-check: any `boolean = integer` comparison the coercion pass didn't
|
|
1237
1688
|
# eliminate would abort on PG. Surface it as a gap (not silent output) so the gap
|
|
@@ -1240,12 +1691,123 @@ def _transpile_plain(sql: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
|
1240
1691
|
if residual:
|
|
1241
1692
|
unhandled.append({"kind": "BOOL-INT-RESIDUAL", "snippet": "; ".join(residual)[:120]})
|
|
1242
1693
|
continue
|
|
1243
|
-
|
|
1244
|
-
|
|
1694
|
+
# EMPTY-EMISSION postcondition (issue #3252 1b): sqlglot's generator treats an
|
|
1695
|
+
# unsupported node as a WARNING and returns "" — which would append a bare `;` and
|
|
1696
|
+
# vanish the source statement silently. Any non-empty source statement that renders
|
|
1697
|
+
# to empty/whitespace is REPORTED as a gap, never emitted. This closes the whole
|
|
1698
|
+
# class of "generator warned and returned nothing", not just the IfBlock instance.
|
|
1699
|
+
rendered = stmt.sql(dialect=MJPostgres, pretty=pretty, identify=True)
|
|
1700
|
+
if not rendered.strip().strip(";").strip():
|
|
1701
|
+
txt = stmt.sql(dialect="tsql")
|
|
1702
|
+
unhandled.append({"kind": "EMPTY-EMISSION", "snippet": txt[:80]})
|
|
1703
|
+
continue
|
|
1704
|
+
out.append(rendered)
|
|
1705
|
+
# SOFT reconciliation (issue #3252 1d): every parsed statement must land in exactly one
|
|
1706
|
+
# bucket. A mismatch means a drop site was missed — surface it as a gap so it is loud and
|
|
1707
|
+
# gets artifacts, but NEVER raise (a raise would abort the whole conversion run with zero
|
|
1708
|
+
# artifacts). `emitted` is counted at source-statement granularity (len(out)), NOT by
|
|
1709
|
+
# re-splitting the emitted body — a single statement can expand to a multi-`;` DO block.
|
|
1710
|
+
accounted = len(out) + len(unhandled) + len(dropped)
|
|
1711
|
+
if parsed != accounted:
|
|
1712
|
+
unhandled.append({
|
|
1713
|
+
"kind": "ACCOUNTING-LEAK",
|
|
1714
|
+
"snippet": f"parsed={parsed} but emitted={len(out)}+unhandled={len(unhandled)}+dropped={len(dropped)}={accounted}",
|
|
1715
|
+
})
|
|
1716
|
+
return (";\n".join(out) + (";" if out else "")), unhandled, dropped
|
|
1245
1717
|
|
|
1246
1718
|
|
|
1247
1719
|
_RAISERROR = _re.compile(r"RAISERROR\s*\(\s*(N?'(?:[^']|'')*'|@?\w+)", _re.IGNORECASE)
|
|
1248
1720
|
|
|
1721
|
+
# Statement-starter keywords that begin a NEW T-SQL statement. PRINT and RETURN are EXCLUDED:
|
|
1722
|
+
# after a RAISERROR aborts (→ RAISE EXCEPTION) they are moot, so the guard fast path treats a
|
|
1723
|
+
# trailing PRINT/RETURN as benign. Any OTHER starter glued on with no `;` (T-SQL makes the
|
|
1724
|
+
# terminator optional) is a REAL statement the fast path must NOT swallow (issue #3252 code review:
|
|
1725
|
+
# `RAISERROR(...) PRINT 'x' UPDATE …` — the UPDATE rode in behind the PRINT and vanished silently).
|
|
1726
|
+
_REAL_STMT_STARTERS = frozenset({
|
|
1727
|
+
"UPDATE", "INSERT", "DELETE", "MERGE", "SELECT", "EXEC", "EXECUTE", "CREATE",
|
|
1728
|
+
"ALTER", "DROP", "TRUNCATE", "DECLARE", "SET", "WHILE", "RAISERROR", "THROW",
|
|
1729
|
+
"GRANT", "REVOKE", "DENY", "BEGIN", "COMMIT", "ROLLBACK", "SAVE", "WAITFOR",
|
|
1730
|
+
"GOTO", "USE", "BREAK", "CONTINUE", "IF",
|
|
1731
|
+
})
|
|
1732
|
+
|
|
1733
|
+
|
|
1734
|
+
def _has_real_stmt_starter(s: str) -> bool:
|
|
1735
|
+
"""True if `s` has a real statement-starter (see _REAL_STMT_STARTERS) at paren depth 0,
|
|
1736
|
+
atom-aware. A scalar PRINT/RETURN argument never places one at depth 0 — a subquery's SELECT
|
|
1737
|
+
lives inside parens (depth > 0) — so a depth-0 hit means a real statement is glued on with no
|
|
1738
|
+
semicolon and must NOT be swallowed by the PRINT/RETURN exemption (issue #3252 code review)."""
|
|
1739
|
+
i, n, depth = 0, len(s), 0
|
|
1740
|
+
while i < n:
|
|
1741
|
+
j = _scan_atom(s, i)
|
|
1742
|
+
if j != i: # string / comment / [bracketed] / "quoted" — a keyword inside is data, not code
|
|
1743
|
+
i = j
|
|
1744
|
+
continue
|
|
1745
|
+
c = s[i]
|
|
1746
|
+
if c == "(":
|
|
1747
|
+
depth += 1
|
|
1748
|
+
i += 1
|
|
1749
|
+
continue
|
|
1750
|
+
if c == ")":
|
|
1751
|
+
depth = max(0, depth - 1)
|
|
1752
|
+
i += 1
|
|
1753
|
+
continue
|
|
1754
|
+
m = _WORD.match(s, i)
|
|
1755
|
+
if m:
|
|
1756
|
+
if depth == 0 and m.group(0).upper() in _REAL_STMT_STARTERS:
|
|
1757
|
+
return True
|
|
1758
|
+
i = m.end()
|
|
1759
|
+
continue
|
|
1760
|
+
i += 1
|
|
1761
|
+
return False
|
|
1762
|
+
|
|
1763
|
+
|
|
1764
|
+
def _is_raiserror_only(stmt: str) -> bool:
|
|
1765
|
+
"""True if `stmt` is a lone RAISERROR(...) call with no REAL statement glued after it. T-SQL
|
|
1766
|
+
makes the terminating `;` optional, so a sibling routinely follows with no separator
|
|
1767
|
+
(`RAISERROR(...) \n UPDATE ...`); _split_top_level_statements breaks on `;` only, so it hands
|
|
1768
|
+
that whole run in as ONE piece. Scan past the RAISERROR call's matched `)` (atom-aware, so a
|
|
1769
|
+
`(` inside the message string doesn't fool it): a trailing RETURN / PRINT / comment / blank is
|
|
1770
|
+
NOT a sibling (a RETURN is moot after the RAISE aborts), but a REAL statement riding after them
|
|
1771
|
+
— even glued behind a semicolon-less PRINT (`… PRINT 'x' UPDATE …`) — makes this NOT
|
|
1772
|
+
raiserror-only (#3252 code review). The PRINT/RETURN exemption is narrow, not a blanket skip."""
|
|
1773
|
+
if not _RAISERROR.match(stmt):
|
|
1774
|
+
return False
|
|
1775
|
+
open_paren = stmt.find("(")
|
|
1776
|
+
if open_paren < 0:
|
|
1777
|
+
return False
|
|
1778
|
+
close = _match_paren(stmt, open_paren)
|
|
1779
|
+
if close < 0:
|
|
1780
|
+
return False
|
|
1781
|
+
return not _has_real_stmt_starter(stmt[close:])
|
|
1782
|
+
|
|
1783
|
+
|
|
1784
|
+
def _raiserror_has_siblings(body: str) -> bool:
|
|
1785
|
+
"""True if `body` contains a meaningful statement OTHER than a single RAISERROR (a benign
|
|
1786
|
+
trailing PRINT / RETURN / comment / blank doesn't count). The RAISERROR→RAISE-EXCEPTION guard
|
|
1787
|
+
fast path collapses the guard to ONE `RAISE EXCEPTION`, so it is only safe when exactly ONE
|
|
1788
|
+
RAISERROR stands alone; a paired real statement — OR a SECOND RAISERROR — would be silently
|
|
1789
|
+
dropped (issue #3252). Because T-SQL's `;` is optional, a real statement can ride glued after a
|
|
1790
|
+
PRINT with no separator, so a piece that merely STARTS with PRINT is NOT automatically benign:
|
|
1791
|
+
consume the PRINT/RETURN and check the remainder for a real starter. _is_raiserror_only handles
|
|
1792
|
+
the semicolon-less RAISERROR shape a bare `;`-split misses; the count guards the semicolon-
|
|
1793
|
+
SEPARATED double-RAISERROR (`RAISERROR('a'); RAISERROR('b')`), whose second call splits into its
|
|
1794
|
+
own lone-raiserror piece and must not fold away unaccounted (issue #3252 code review 2)."""
|
|
1795
|
+
raiserrors = 0
|
|
1796
|
+
for s in _split_top_level_statements(body):
|
|
1797
|
+
stmt = _strip_leading_sql_comments(s).strip(" \t\r\n;")
|
|
1798
|
+
if not stmt:
|
|
1799
|
+
continue
|
|
1800
|
+
if _is_raiserror_only(stmt):
|
|
1801
|
+
raiserrors += 1
|
|
1802
|
+
if raiserrors > 1:
|
|
1803
|
+
return True # a 2nd RAISERROR can't fold into the single RAISE EXCEPTION — a sibling
|
|
1804
|
+
continue
|
|
1805
|
+
head = _re.match(r"(?:PRINT|RETURN)\b", stmt, _re.IGNORECASE)
|
|
1806
|
+
if head and not _has_real_stmt_starter(stmt[head.end():]):
|
|
1807
|
+
continue # a lone benign PRINT/RETURN — nothing real glued after it
|
|
1808
|
+
return True
|
|
1809
|
+
return False
|
|
1810
|
+
|
|
1249
1811
|
# Inline entity-metadata INSERTs (Entity / EntityField / ApplicationEntity / …) in a
|
|
1250
1812
|
# FEATURE migration are KEPT and transpiled — NOT dropped. Empirically, `mj codegen` on
|
|
1251
1813
|
# PostgreSQL regenerates SQL *objects* (views, CRUD functions, triggers — already dropped
|
|
@@ -1260,12 +1822,14 @@ _RAISERROR = _re.compile(r"RAISERROR\s*\(\s*(N?'(?:[^']|'')*'|@?\w+)", _re.IGNOR
|
|
|
1260
1822
|
_METADATA_TABLES = _re.compile(r"(?!x)x")
|
|
1261
1823
|
|
|
1262
1824
|
|
|
1263
|
-
def _transpile_extprop_segment(text: str, pretty: bool = False) -> tuple[str, list[dict]]:
|
|
1825
|
+
def _transpile_extprop_segment(text: str, pretty: bool = False) -> tuple[str, list[dict], list[dict]]:
|
|
1264
1826
|
"""Transpile a text segment that may interleave sp_add/dropextendedproperty envelopes
|
|
1265
1827
|
with plain SQL — used for IF…BEGIN bodies, where a guarded INSERT can sit next to an
|
|
1266
|
-
extprop EXEC (the top-level batch walker handles the same mix outside blocks).
|
|
1828
|
+
extprop EXEC (the top-level batch walker handles the same mix outside blocks).
|
|
1829
|
+
Returns (sql, unhandled, dropped) — `dropped` accumulated from the plain-SQL gaps."""
|
|
1267
1830
|
out: list[str] = []
|
|
1268
1831
|
unhandled: list[dict] = []
|
|
1832
|
+
dropped: list[dict] = []
|
|
1269
1833
|
pos = 0
|
|
1270
1834
|
while pos < len(text):
|
|
1271
1835
|
ext = _SP_EXTPROP.search(text, pos)
|
|
@@ -1273,10 +1837,11 @@ def _transpile_extprop_segment(text: str, pretty: bool = False) -> tuple[str, li
|
|
|
1273
1837
|
nxt = min([x for x in (ext, dxp) if x], key=lambda x: x.start(), default=None)
|
|
1274
1838
|
gap = text[pos:] if nxt is None else text[pos:nxt.start()]
|
|
1275
1839
|
if gap.strip():
|
|
1276
|
-
s, u = _transpile_plain(gap, pretty)
|
|
1840
|
+
s, u, d = _transpile_plain(gap, pretty)
|
|
1277
1841
|
if s.strip():
|
|
1278
1842
|
out.append(s)
|
|
1279
1843
|
unhandled.extend(u)
|
|
1844
|
+
dropped.extend(d)
|
|
1280
1845
|
if nxt is None:
|
|
1281
1846
|
break
|
|
1282
1847
|
if nxt is ext:
|
|
@@ -1292,7 +1857,7 @@ def _transpile_extprop_segment(text: str, pretty: bool = False) -> tuple[str, li
|
|
|
1292
1857
|
elif comment is None:
|
|
1293
1858
|
unhandled.append({"kind": "sp_dropextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
1294
1859
|
pos = nxt.end()
|
|
1295
|
-
return "\n".join(out), unhandled
|
|
1860
|
+
return "\n".join(out), unhandled, dropped
|
|
1296
1861
|
|
|
1297
1862
|
|
|
1298
1863
|
# SS catalog references in a guard condition (sys.* views / OBJECT_ID()) — meaningless
|
|
@@ -1380,12 +1945,13 @@ def _translate_sys_guard(cond: str, neg: bool) -> str | None:
|
|
|
1380
1945
|
return None
|
|
1381
1946
|
|
|
1382
1947
|
|
|
1383
|
-
def _transpile_if_exists_begin(m: _IfExistsMatch) -> tuple[str, list[dict]]:
|
|
1384
|
-
"""IF [NOT] EXISTS(<sel>) BEGIN <body> END → PG DO $$ … IF … THEN … END IF; … $$;
|
|
1948
|
+
def _transpile_if_exists_begin(m: _IfExistsMatch) -> tuple[str, list[dict], list[dict]]:
|
|
1949
|
+
"""IF [NOT] EXISTS(<sel>) BEGIN <body> END → PG DO $$ … IF … THEN … END IF; … $$;
|
|
1950
|
+
Returns (sql, unhandled, dropped) — see _transpile_plain for the accounting contract."""
|
|
1385
1951
|
raw_body = m.group("body").strip()
|
|
1386
1952
|
# Idempotent seed of schema-derived metadata → drop; CodeGen regenerates it.
|
|
1387
1953
|
if _METADATA_TABLES.search(raw_body):
|
|
1388
|
-
return "", []
|
|
1954
|
+
return "", [], [{"kind": "IF-GUARD-METADATA", "snippet": m.group(0)[:80]}]
|
|
1389
1955
|
# Extended-property comment dance: a guard whose body consists EXCLUSIVELY of
|
|
1390
1956
|
# sp_add/dropextendedproperty EXECs (plus PRINT/comment noise) is SQL-Server-only —
|
|
1391
1957
|
# PG `COMMENT ON … IS …` overwrites unconditionally and `mj codegen` re-syncs every
|
|
@@ -1401,37 +1967,43 @@ def _transpile_if_exists_begin(m: _IfExistsMatch) -> tuple[str, list[dict]]:
|
|
|
1401
1967
|
and not _re.match(r"\s*PRINT\b", _strip_leading_sql_comments(s), _re.IGNORECASE)
|
|
1402
1968
|
]
|
|
1403
1969
|
if not leftover:
|
|
1404
|
-
return "", []
|
|
1970
|
+
return "", [], [{"kind": "IF-GUARD-EXTPROP", "snippet": m.group(0)[:80]}]
|
|
1405
1971
|
neg = "NOT " if m.group("neg") else ""
|
|
1406
1972
|
cond_raw = m.group("cond").strip()
|
|
1407
1973
|
u1: list[dict] = []
|
|
1974
|
+
d1: list[dict] = []
|
|
1408
1975
|
# SS catalog guards (sys.* / OBJECT_ID()) fail at apply on PG — translate the common
|
|
1409
1976
|
# shapes; anything unrecognized is reported whole, never emitted as sys.* SQL.
|
|
1410
1977
|
if _SYS_CATALOG_REF.search(cond_raw):
|
|
1411
1978
|
cond_full = _translate_sys_guard(cond_raw, bool(m.group("neg")))
|
|
1412
1979
|
if cond_full is None:
|
|
1413
|
-
return "", [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}]
|
|
1980
|
+
return "", [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}], []
|
|
1414
1981
|
else:
|
|
1415
|
-
cond_sql, u1 = _transpile_plain(cond_raw)
|
|
1982
|
+
cond_sql, u1, d1 = _transpile_plain(cond_raw)
|
|
1416
1983
|
cond_inner = cond_sql.rstrip(";").strip()
|
|
1417
1984
|
cond_full = f"{neg}EXISTS ({cond_inner})" if cond_inner else None
|
|
1418
|
-
# Guard blocks (IF EXISTS(...) BEGIN RAISERROR('conflict') END) → RAISE EXCEPTION
|
|
1985
|
+
# Guard blocks (IF EXISTS(...) BEGIN RAISERROR('conflict') END) → RAISE EXCEPTION — but ONLY
|
|
1986
|
+
# when RAISERROR is the sole governed statement. A body pairing it with real statements (e.g.
|
|
1987
|
+
# RAISERROR(...) then INSERT ...) cannot collapse to one RAISE (which aborts) without silently
|
|
1988
|
+
# dropping the siblings; report the whole guard unhandled so it is hand-authored (#3252 review).
|
|
1419
1989
|
rr = _RAISERROR.search(raw_body)
|
|
1990
|
+
if rr and _raiserror_has_siblings(raw_body):
|
|
1991
|
+
return "", (u1 + [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}]), d1
|
|
1420
1992
|
if rr:
|
|
1421
1993
|
msg = rr.group(1)
|
|
1422
1994
|
msg = _pg_string(_unquote_tsql_string(msg)) if msg.lstrip("Nn").startswith("'") else "'migration guard failed'"
|
|
1423
|
-
body_sql, u2 = f"RAISE EXCEPTION {msg};", []
|
|
1995
|
+
body_sql, u2, d2 = f"RAISE EXCEPTION {msg};", [], []
|
|
1424
1996
|
else:
|
|
1425
|
-
body_sql, u2 = _transpile_extprop_segment(raw_body)
|
|
1997
|
+
body_sql, u2, d2 = _transpile_extprop_segment(raw_body)
|
|
1426
1998
|
if not cond_full or not body_sql.strip():
|
|
1427
|
-
return "", (u1 + u2 + [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}])
|
|
1999
|
+
return "", (u1 + u2 + [{"kind": "IF-EXISTS-BEGIN", "snippet": m.group(0)[:80]}]), (d1 + d2)
|
|
1428
2000
|
do = (
|
|
1429
2001
|
"DO $$\nBEGIN\n"
|
|
1430
2002
|
f" IF {cond_full} THEN\n"
|
|
1431
2003
|
f" {body_sql.strip()}\n"
|
|
1432
2004
|
" END IF;\nEND $$;"
|
|
1433
2005
|
)
|
|
1434
|
-
return do, (u1 + u2)
|
|
2006
|
+
return do, (u1 + u2), (d1 + d2)
|
|
1435
2007
|
|
|
1436
2008
|
|
|
1437
2009
|
def mj_transpile(sql: str, *, pretty: bool = True, identify: bool = True) -> dict:
|
|
@@ -1474,12 +2046,14 @@ def mj_transpile(sql: str, *, pretty: bool = True, identify: bool = True) -> dic
|
|
|
1474
2046
|
except Exception: # noqa: BLE001
|
|
1475
2047
|
pass
|
|
1476
2048
|
|
|
2049
|
+
dropped: list[dict] = []
|
|
1477
2050
|
for batch in _GO_SPLIT.split(sql):
|
|
1478
2051
|
if not batch.strip():
|
|
1479
2052
|
continue
|
|
1480
|
-
out_sql, u = _transpile_batch(batch, pretty)
|
|
2053
|
+
out_sql, u, d = _transpile_batch(batch, pretty)
|
|
1481
2054
|
out.extend(out_sql)
|
|
1482
2055
|
unhandled.extend(u)
|
|
2056
|
+
dropped.extend(d)
|
|
1483
2057
|
|
|
1484
2058
|
# Final safety net: the macro is protected to FLYWAY_SENTINEL by a blanket text
|
|
1485
2059
|
# replace before parsing, and restored at the AST level in identifier_sql (identifier
|
|
@@ -1496,8 +2070,16 @@ def mj_transpile(sql: str, *, pretty: bool = True, identify: bool = True) -> dic
|
|
|
1496
2070
|
{**u, "snippet": u["snippet"].replace(FLYWAY_SENTINEL, FLYWAY_MACRO)}
|
|
1497
2071
|
for u in unhandled
|
|
1498
2072
|
]
|
|
2073
|
+
dropped = [
|
|
2074
|
+
{**d, "snippet": d["snippet"].replace(FLYWAY_SENTINEL, FLYWAY_MACRO)}
|
|
2075
|
+
for d in dropped
|
|
2076
|
+
]
|
|
1499
2077
|
|
|
1500
|
-
|
|
2078
|
+
# `dropped` lists every INTENTIONALLY-discarded statement (batch-control noise, swallowed
|
|
2079
|
+
# routine `END`, metadata-guard drops, …) so the TS/CLI reconciliation layer (issue #3252
|
|
2080
|
+
# Phase 3) can account for what the dialect discarded. Backward-compatible: existing
|
|
2081
|
+
# consumers read only `sql`/`unhandled`.
|
|
2082
|
+
return {"sql": out, "unhandled": unhandled, "dropped": dropped}
|
|
1501
2083
|
|
|
1502
2084
|
|
|
1503
2085
|
# Baseline extended-property EXECs come wrapped in per-statement error handling:
|
|
@@ -1520,10 +2102,29 @@ def _strip_catch_noise(batch: str) -> str:
|
|
|
1520
2102
|
return _CATCH_BLOCK.sub(repl, batch)
|
|
1521
2103
|
|
|
1522
2104
|
|
|
1523
|
-
|
|
1524
|
-
|
|
2105
|
+
_UNSET = object() # "finder not yet run", distinct from a finder returning None (no match)
|
|
2106
|
+
|
|
2107
|
+
|
|
2108
|
+
def _next_match(cache: dict, key: str, finder, batch: str, pos: int):
|
|
2109
|
+
"""Memoized "next match at/after pos" for the batch walker. A cached match with
|
|
2110
|
+
start() >= pos is reused; a cached None is STICKY (a finder that found nothing from a
|
|
2111
|
+
lower pos finds nothing from a higher one); a finder is re-run only after pos advances
|
|
2112
|
+
PAST its last hit. Without this, the walker re-scans the whole batch with every finder on
|
|
2113
|
+
every iteration — O(statements × batch length), which on 50 MB baselines with thousands of
|
|
2114
|
+
sp_addextendedproperty envelopes turns the Python-level `_find_routine_envelope` scan into
|
|
2115
|
+
an O(N²) hang. With it, each finder runs O(number of its matches)."""
|
|
2116
|
+
cached = cache.get(key, _UNSET)
|
|
2117
|
+
if cached is _UNSET or (cached is not None and cached.start() < pos):
|
|
2118
|
+
cache[key] = finder(batch, pos)
|
|
2119
|
+
return cache[key]
|
|
2120
|
+
|
|
2121
|
+
|
|
2122
|
+
def _transpile_batch(batch: str, pretty: bool = False) -> tuple[list[str], list[dict], list[dict]]:
|
|
2123
|
+
"""Scan one GO batch into envelope chunks + plain SQL, transpiling each in order.
|
|
2124
|
+
Returns (sql_list, unhandled, dropped) — see _transpile_plain for the accounting contract."""
|
|
1525
2125
|
out: list[str] = []
|
|
1526
2126
|
unhandled: list[dict] = []
|
|
2127
|
+
dropped: list[dict] = []
|
|
1527
2128
|
|
|
1528
2129
|
# Extended-property batches: drop the SS error-handling plumbing around the EXECs
|
|
1529
2130
|
# (see _strip_catch_noise), then let the walk below handle EVERYTHING in the batch —
|
|
@@ -1532,45 +2133,64 @@ def _transpile_batch(batch: str, pretty: bool = False) -> tuple[list[str], list[
|
|
|
1532
2133
|
batch = _strip_catch_noise(batch)
|
|
1533
2134
|
|
|
1534
2135
|
pos = 0
|
|
2136
|
+
cache: dict = {} # memoizes each finder's next hit so the walk stays O(N) (see _next_match)
|
|
1535
2137
|
# Walk the batch, alternating between recognized envelopes and plain SQL gaps.
|
|
1536
2138
|
while pos < len(batch):
|
|
1537
|
-
ext = _SP_EXTPROP.search
|
|
1538
|
-
dxp = _SP_DROPEXTPROP.search
|
|
1539
|
-
ife = _find_if_exists_begin
|
|
1540
|
-
|
|
2139
|
+
ext = _next_match(cache, "ext", _SP_EXTPROP.search, batch, pos)
|
|
2140
|
+
dxp = _next_match(cache, "dxp", _SP_DROPEXTPROP.search, batch, pos)
|
|
2141
|
+
ife = _next_match(cache, "ife", _find_if_exists_begin, batch, pos)
|
|
2142
|
+
rtn = _next_match(cache, "rtn", _find_routine_envelope, batch, pos)
|
|
2143
|
+
nxt = min([m for m in (ext, dxp, ife, rtn) if m], key=lambda m: m.start(), default=None)
|
|
1541
2144
|
if nxt is None:
|
|
1542
2145
|
gap = batch[pos:]
|
|
1543
2146
|
if gap.strip():
|
|
1544
|
-
s, u = _transpile_plain(gap, pretty)
|
|
2147
|
+
s, u, d = _transpile_plain(gap, pretty)
|
|
1545
2148
|
if s.strip():
|
|
1546
2149
|
out.append(s)
|
|
1547
2150
|
unhandled.extend(u)
|
|
2151
|
+
dropped.extend(d)
|
|
1548
2152
|
break
|
|
1549
2153
|
gap = batch[pos:nxt.start()]
|
|
1550
2154
|
if gap.strip():
|
|
1551
|
-
s, u = _transpile_plain(gap, pretty)
|
|
2155
|
+
s, u, d = _transpile_plain(gap, pretty)
|
|
1552
2156
|
if s.strip():
|
|
1553
2157
|
out.append(s)
|
|
1554
2158
|
unhandled.extend(u)
|
|
2159
|
+
dropped.extend(d)
|
|
1555
2160
|
if nxt is ext:
|
|
1556
2161
|
comment = _transpile_sp_addextendedproperty(nxt.group("args"))
|
|
1557
2162
|
if comment:
|
|
1558
2163
|
out.append(comment)
|
|
1559
|
-
elif comment is None: #
|
|
2164
|
+
elif comment is None: # a genuine failure to transpile the envelope — report it
|
|
1560
2165
|
unhandled.append({"kind": "sp_addextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
2166
|
+
else: # "" — an intentional CodeGen-object comment skip: RECORD it as a drop so every
|
|
2167
|
+
# empty result is accounted for and a real drop can never hide here (issue #3252 P2)
|
|
2168
|
+
dropped.append({"kind": "sp_addextendedproperty-codegen-skip", "snippet": nxt.group(0)[:80]})
|
|
1561
2169
|
elif nxt is dxp:
|
|
1562
2170
|
comment = _transpile_sp_dropextendedproperty(nxt.group("args"))
|
|
1563
2171
|
if comment:
|
|
1564
2172
|
out.append(comment)
|
|
1565
2173
|
elif comment is None:
|
|
1566
2174
|
unhandled.append({"kind": "sp_dropextendedproperty", "snippet": nxt.group(0)[:80]})
|
|
2175
|
+
else: # "" — intentional CodeGen-object skip, accounted as a drop (issue #3252 P2)
|
|
2176
|
+
dropped.append({"kind": "sp_dropextendedproperty-codegen-skip", "snippet": nxt.group(0)[:80]})
|
|
2177
|
+
elif nxt is rtn:
|
|
2178
|
+
# A hand-written CREATE PROCEDURE/FUNCTION/TRIGGER can't be auto-transpiled to PG
|
|
2179
|
+
# (different procedural language + delimiter syntax). Report the routine WHOLE as
|
|
2180
|
+
# ONE gap and emit NOTHING for it — this is the containment that stops body
|
|
2181
|
+
# statements from escaping as top-level migration SQL (issue #3252 bugs #3/#4:
|
|
2182
|
+
# a body UPDATE running table-wide at apply time, a dangling END emitting as a
|
|
2183
|
+
# transaction-ending COMMIT, cursor fragments emitting as invalid PG). MJ's own
|
|
2184
|
+
# CodeGen routines are regenerated by the bake path, never routed through here.
|
|
2185
|
+
unhandled.append({"kind": f"CREATE-{rtn.kind}", "snippet": rtn.snippet})
|
|
1567
2186
|
else:
|
|
1568
|
-
do, u = _transpile_if_exists_begin(nxt)
|
|
2187
|
+
do, u, d = _transpile_if_exists_begin(nxt)
|
|
1569
2188
|
if do.strip():
|
|
1570
2189
|
out.append(do)
|
|
1571
2190
|
unhandled.extend(u)
|
|
2191
|
+
dropped.extend(d)
|
|
1572
2192
|
pos = nxt.end()
|
|
1573
|
-
return out, unhandled
|
|
2193
|
+
return out, unhandled, dropped
|
|
1574
2194
|
|
|
1575
2195
|
|
|
1576
2196
|
if __name__ == "__main__":
|