@sqlanvil/cli 1.27.2 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +2073 -58
- package/package.json +4 -2
- package/worker_bundle.js +356 -1
package/bundle.js
CHANGED
|
@@ -151,6 +151,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
151
151
|
* @property {string|null} [warehouse] WorkflowSettings warehouse
|
|
152
152
|
* @property {Object.<string,sqlanvil.IConnectionConfig>|null} [connections] WorkflowSettings connections
|
|
153
153
|
* @property {Object.<string,sqlanvil.IEnvironment>|null} [environments] WorkflowSettings environments
|
|
154
|
+
* @property {boolean|null} [preserveGovernanceControls] WorkflowSettings preserveGovernanceControls
|
|
154
155
|
*/
|
|
155
156
|
|
|
156
157
|
/**
|
|
@@ -323,6 +324,14 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
323
324
|
*/
|
|
324
325
|
WorkflowSettings.prototype.environments = $util.emptyObject;
|
|
325
326
|
|
|
327
|
+
/**
|
|
328
|
+
* WorkflowSettings preserveGovernanceControls.
|
|
329
|
+
* @member {boolean} preserveGovernanceControls
|
|
330
|
+
* @memberof sqlanvil.WorkflowSettings
|
|
331
|
+
* @instance
|
|
332
|
+
*/
|
|
333
|
+
WorkflowSettings.prototype.preserveGovernanceControls = false;
|
|
334
|
+
|
|
326
335
|
/**
|
|
327
336
|
* Creates a new WorkflowSettings instance using the specified properties.
|
|
328
337
|
* @function create
|
|
@@ -396,6 +405,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
396
405
|
writer.uint32(/* id 19, wireType 2 =*/154).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]);
|
|
397
406
|
$root.sqlanvil.Environment.encode(message.environments[keys[i]], writer.uint32(/* id 2, wireType 2 =*/18).fork(), q + 1).ldelim().ldelim();
|
|
398
407
|
}
|
|
408
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
409
|
+
writer.uint32(/* id 20, wireType 0 =*/160).bool(message.preserveGovernanceControls);
|
|
399
410
|
return writer;
|
|
400
411
|
};
|
|
401
412
|
|
|
@@ -575,6 +586,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
575
586
|
message.environments[key] = value;
|
|
576
587
|
break;
|
|
577
588
|
}
|
|
589
|
+
case 20: {
|
|
590
|
+
message.preserveGovernanceControls = reader.bool();
|
|
591
|
+
break;
|
|
592
|
+
}
|
|
578
593
|
default:
|
|
579
594
|
reader.skipType(tag & 7, long);
|
|
580
595
|
break;
|
|
@@ -696,6 +711,9 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
696
711
|
return "environments." + error;
|
|
697
712
|
}
|
|
698
713
|
}
|
|
714
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
715
|
+
if (typeof message.preserveGovernanceControls !== "boolean")
|
|
716
|
+
return "preserveGovernanceControls: boolean expected";
|
|
699
717
|
return null;
|
|
700
718
|
};
|
|
701
719
|
|
|
@@ -792,6 +810,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
792
810
|
message.environments[keys[i]] = $root.sqlanvil.Environment.fromObject(object.environments[keys[i]], long + 1);
|
|
793
811
|
}
|
|
794
812
|
}
|
|
813
|
+
if (object.preserveGovernanceControls != null)
|
|
814
|
+
message.preserveGovernanceControls = Boolean(object.preserveGovernanceControls);
|
|
795
815
|
return message;
|
|
796
816
|
};
|
|
797
817
|
|
|
@@ -834,6 +854,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
834
854
|
object.extension = null;
|
|
835
855
|
object.includeTestsInCompiledGraph = false;
|
|
836
856
|
object.warehouse = "";
|
|
857
|
+
object.preserveGovernanceControls = false;
|
|
837
858
|
}
|
|
838
859
|
if (message.sqlanvilCoreVersion != null && Object.hasOwnProperty.call(message, "sqlanvilCoreVersion"))
|
|
839
860
|
object.sqlanvilCoreVersion = message.sqlanvilCoreVersion;
|
|
@@ -892,6 +913,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
892
913
|
object.environments[keys2[j]] = $root.sqlanvil.Environment.toObject(message.environments[keys2[j]], options, q + 1);
|
|
893
914
|
}
|
|
894
915
|
}
|
|
916
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
917
|
+
object.preserveGovernanceControls = message.preserveGovernanceControls;
|
|
895
918
|
return object;
|
|
896
919
|
};
|
|
897
920
|
|
|
@@ -4643,6 +4666,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
4643
4666
|
* @property {sqlanvil.IPostgresOptions|null} [postgres] TableConfig postgres
|
|
4644
4667
|
* @property {sqlanvil.ISupabaseOptions|null} [supabase] TableConfig supabase
|
|
4645
4668
|
* @property {sqlanvil.IMysqlOptions|null} [mysql] TableConfig mysql
|
|
4669
|
+
* @property {boolean|null} [preserveGovernanceControls] TableConfig preserveGovernanceControls
|
|
4646
4670
|
*/
|
|
4647
4671
|
|
|
4648
4672
|
/**
|
|
@@ -4876,6 +4900,23 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
4876
4900
|
*/
|
|
4877
4901
|
TableConfig.prototype.mysql = null;
|
|
4878
4902
|
|
|
4903
|
+
/**
|
|
4904
|
+
* TableConfig preserveGovernanceControls.
|
|
4905
|
+
* @member {boolean|null|undefined} preserveGovernanceControls
|
|
4906
|
+
* @memberof sqlanvil.ActionConfig.TableConfig
|
|
4907
|
+
* @instance
|
|
4908
|
+
*/
|
|
4909
|
+
TableConfig.prototype.preserveGovernanceControls = null;
|
|
4910
|
+
|
|
4911
|
+
// OneOf field names bound to virtual getters and setters
|
|
4912
|
+
let $oneOfFields;
|
|
4913
|
+
|
|
4914
|
+
// Virtual OneOf for proto3 optional field
|
|
4915
|
+
Object.defineProperty(TableConfig.prototype, "_preserveGovernanceControls", {
|
|
4916
|
+
get: $util.oneOfGetter($oneOfFields = ["preserveGovernanceControls"]),
|
|
4917
|
+
set: $util.oneOfSetter($oneOfFields)
|
|
4918
|
+
});
|
|
4919
|
+
|
|
4879
4920
|
/**
|
|
4880
4921
|
* Creates a new TableConfig instance using the specified properties.
|
|
4881
4922
|
* @function create
|
|
@@ -4964,6 +5005,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
4964
5005
|
$root.sqlanvil.SupabaseOptions.encode(message.supabase, writer.uint32(/* id 26, wireType 2 =*/210).fork(), q + 1).ldelim();
|
|
4965
5006
|
if (message.mysql != null && Object.hasOwnProperty.call(message, "mysql"))
|
|
4966
5007
|
$root.sqlanvil.MysqlOptions.encode(message.mysql, writer.uint32(/* id 27, wireType 2 =*/218).fork(), q + 1).ldelim();
|
|
5008
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
5009
|
+
writer.uint32(/* id 28, wireType 0 =*/224).bool(message.preserveGovernanceControls);
|
|
4967
5010
|
return writer;
|
|
4968
5011
|
};
|
|
4969
5012
|
|
|
@@ -5162,6 +5205,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
5162
5205
|
message.mysql = $root.sqlanvil.MysqlOptions.decode(reader, reader.uint32(), undefined, long + 1);
|
|
5163
5206
|
break;
|
|
5164
5207
|
}
|
|
5208
|
+
case 28: {
|
|
5209
|
+
message.preserveGovernanceControls = reader.bool();
|
|
5210
|
+
break;
|
|
5211
|
+
}
|
|
5165
5212
|
default:
|
|
5166
5213
|
reader.skipType(tag & 7, long);
|
|
5167
5214
|
break;
|
|
@@ -5329,6 +5376,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
5329
5376
|
if (error)
|
|
5330
5377
|
return "mysql." + error;
|
|
5331
5378
|
}
|
|
5379
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls")) {
|
|
5380
|
+
if (typeof message.preserveGovernanceControls !== "boolean")
|
|
5381
|
+
return "preserveGovernanceControls: boolean expected";
|
|
5382
|
+
}
|
|
5332
5383
|
return null;
|
|
5333
5384
|
};
|
|
5334
5385
|
|
|
@@ -5472,6 +5523,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
5472
5523
|
throw TypeError(".sqlanvil.ActionConfig.TableConfig.mysql: object expected");
|
|
5473
5524
|
message.mysql = $root.sqlanvil.MysqlOptions.fromObject(object.mysql, long + 1);
|
|
5474
5525
|
}
|
|
5526
|
+
if (object.preserveGovernanceControls != null)
|
|
5527
|
+
message.preserveGovernanceControls = Boolean(object.preserveGovernanceControls);
|
|
5475
5528
|
return message;
|
|
5476
5529
|
};
|
|
5477
5530
|
|
|
@@ -5607,6 +5660,11 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
5607
5660
|
object.supabase = $root.sqlanvil.SupabaseOptions.toObject(message.supabase, options, q + 1);
|
|
5608
5661
|
if (message.mysql != null && Object.hasOwnProperty.call(message, "mysql"))
|
|
5609
5662
|
object.mysql = $root.sqlanvil.MysqlOptions.toObject(message.mysql, options, q + 1);
|
|
5663
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls")) {
|
|
5664
|
+
object.preserveGovernanceControls = message.preserveGovernanceControls;
|
|
5665
|
+
if (options.oneofs)
|
|
5666
|
+
object._preserveGovernanceControls = "preserveGovernanceControls";
|
|
5667
|
+
}
|
|
5610
5668
|
return object;
|
|
5611
5669
|
};
|
|
5612
5670
|
|
|
@@ -6862,6 +6920,22 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
6862
6920
|
return values;
|
|
6863
6921
|
})();
|
|
6864
6922
|
|
|
6923
|
+
/**
|
|
6924
|
+
* IncrementalStrategy enum.
|
|
6925
|
+
* @name sqlanvil.ActionConfig.IncrementalStrategy
|
|
6926
|
+
* @enum {number}
|
|
6927
|
+
* @property {number} INCREMENTAL_STRATEGY_UNSPECIFIED=0 INCREMENTAL_STRATEGY_UNSPECIFIED value
|
|
6928
|
+
* @property {number} INCREMENTAL_STRATEGY_MERGE=1 INCREMENTAL_STRATEGY_MERGE value
|
|
6929
|
+
* @property {number} INCREMENTAL_STRATEGY_INSERT_OVERWRITE=2 INCREMENTAL_STRATEGY_INSERT_OVERWRITE value
|
|
6930
|
+
*/
|
|
6931
|
+
ActionConfig.IncrementalStrategy = (function() {
|
|
6932
|
+
const valuesById = {}, values = Object.create(valuesById);
|
|
6933
|
+
values[valuesById[0] = "INCREMENTAL_STRATEGY_UNSPECIFIED"] = 0;
|
|
6934
|
+
values[valuesById[1] = "INCREMENTAL_STRATEGY_MERGE"] = 1;
|
|
6935
|
+
values[valuesById[2] = "INCREMENTAL_STRATEGY_INSERT_OVERWRITE"] = 2;
|
|
6936
|
+
return values;
|
|
6937
|
+
})();
|
|
6938
|
+
|
|
6865
6939
|
ActionConfig.IncrementalTableConfig = (function() {
|
|
6866
6940
|
|
|
6867
6941
|
/**
|
|
@@ -6898,6 +6972,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
6898
6972
|
* @property {sqlanvil.IPostgresOptions|null} [postgres] IncrementalTableConfig postgres
|
|
6899
6973
|
* @property {sqlanvil.ISupabaseOptions|null} [supabase] IncrementalTableConfig supabase
|
|
6900
6974
|
* @property {sqlanvil.IMysqlOptions|null} [mysql] IncrementalTableConfig mysql
|
|
6975
|
+
* @property {sqlanvil.ActionConfig.IncrementalStrategy|null} [incrementalStrategy] IncrementalTableConfig incrementalStrategy
|
|
6976
|
+
* @property {Array.<string>|null} [incrementalPredicates] IncrementalTableConfig incrementalPredicates
|
|
6901
6977
|
*/
|
|
6902
6978
|
|
|
6903
6979
|
/**
|
|
@@ -6918,6 +6994,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
6918
6994
|
this.clusterBy = [];
|
|
6919
6995
|
this.labels = {};
|
|
6920
6996
|
this.additionalOptions = {};
|
|
6997
|
+
this.incrementalPredicates = [];
|
|
6921
6998
|
if (properties)
|
|
6922
6999
|
for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
|
|
6923
7000
|
if (properties[keys[i]] != null && keys[i] !== "__proto__")
|
|
@@ -7164,6 +7241,22 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7164
7241
|
*/
|
|
7165
7242
|
IncrementalTableConfig.prototype.mysql = null;
|
|
7166
7243
|
|
|
7244
|
+
/**
|
|
7245
|
+
* IncrementalTableConfig incrementalStrategy.
|
|
7246
|
+
* @member {sqlanvil.ActionConfig.IncrementalStrategy} incrementalStrategy
|
|
7247
|
+
* @memberof sqlanvil.ActionConfig.IncrementalTableConfig
|
|
7248
|
+
* @instance
|
|
7249
|
+
*/
|
|
7250
|
+
IncrementalTableConfig.prototype.incrementalStrategy = 0;
|
|
7251
|
+
|
|
7252
|
+
/**
|
|
7253
|
+
* IncrementalTableConfig incrementalPredicates.
|
|
7254
|
+
* @member {Array.<string>} incrementalPredicates
|
|
7255
|
+
* @memberof sqlanvil.ActionConfig.IncrementalTableConfig
|
|
7256
|
+
* @instance
|
|
7257
|
+
*/
|
|
7258
|
+
IncrementalTableConfig.prototype.incrementalPredicates = $util.emptyArray;
|
|
7259
|
+
|
|
7167
7260
|
/**
|
|
7168
7261
|
* Creates a new IncrementalTableConfig instance using the specified properties.
|
|
7169
7262
|
* @function create
|
|
@@ -7261,6 +7354,11 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7261
7354
|
$root.sqlanvil.SupabaseOptions.encode(message.supabase, writer.uint32(/* id 29, wireType 2 =*/234).fork(), q + 1).ldelim();
|
|
7262
7355
|
if (message.mysql != null && Object.hasOwnProperty.call(message, "mysql"))
|
|
7263
7356
|
$root.sqlanvil.MysqlOptions.encode(message.mysql, writer.uint32(/* id 30, wireType 2 =*/242).fork(), q + 1).ldelim();
|
|
7357
|
+
if (message.incrementalStrategy != null && Object.hasOwnProperty.call(message, "incrementalStrategy"))
|
|
7358
|
+
writer.uint32(/* id 31, wireType 0 =*/248).int32(message.incrementalStrategy);
|
|
7359
|
+
if (message.incrementalPredicates != null && message.incrementalPredicates.length)
|
|
7360
|
+
for (let i = 0; i < message.incrementalPredicates.length; ++i)
|
|
7361
|
+
writer.uint32(/* id 32, wireType 2 =*/258).string(message.incrementalPredicates[i]);
|
|
7264
7362
|
return writer;
|
|
7265
7363
|
};
|
|
7266
7364
|
|
|
@@ -7477,6 +7575,16 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7477
7575
|
message.mysql = $root.sqlanvil.MysqlOptions.decode(reader, reader.uint32(), undefined, long + 1);
|
|
7478
7576
|
break;
|
|
7479
7577
|
}
|
|
7578
|
+
case 31: {
|
|
7579
|
+
message.incrementalStrategy = reader.int32();
|
|
7580
|
+
break;
|
|
7581
|
+
}
|
|
7582
|
+
case 32: {
|
|
7583
|
+
if (!(message.incrementalPredicates && message.incrementalPredicates.length))
|
|
7584
|
+
message.incrementalPredicates = [];
|
|
7585
|
+
message.incrementalPredicates.push(reader.string());
|
|
7586
|
+
break;
|
|
7587
|
+
}
|
|
7480
7588
|
default:
|
|
7481
7589
|
reader.skipType(tag & 7, long);
|
|
7482
7590
|
break;
|
|
@@ -7667,6 +7775,22 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7667
7775
|
if (error)
|
|
7668
7776
|
return "mysql." + error;
|
|
7669
7777
|
}
|
|
7778
|
+
if (message.incrementalStrategy != null && Object.hasOwnProperty.call(message, "incrementalStrategy"))
|
|
7779
|
+
switch (message.incrementalStrategy) {
|
|
7780
|
+
default:
|
|
7781
|
+
return "incrementalStrategy: enum value expected";
|
|
7782
|
+
case 0:
|
|
7783
|
+
case 1:
|
|
7784
|
+
case 2:
|
|
7785
|
+
break;
|
|
7786
|
+
}
|
|
7787
|
+
if (message.incrementalPredicates != null && Object.hasOwnProperty.call(message, "incrementalPredicates")) {
|
|
7788
|
+
if (!Array.isArray(message.incrementalPredicates))
|
|
7789
|
+
return "incrementalPredicates: array expected";
|
|
7790
|
+
for (let i = 0; i < message.incrementalPredicates.length; ++i)
|
|
7791
|
+
if (!$util.isString(message.incrementalPredicates[i]))
|
|
7792
|
+
return "incrementalPredicates: string[] expected";
|
|
7793
|
+
}
|
|
7670
7794
|
return null;
|
|
7671
7795
|
};
|
|
7672
7796
|
|
|
@@ -7845,6 +7969,33 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7845
7969
|
throw TypeError(".sqlanvil.ActionConfig.IncrementalTableConfig.mysql: object expected");
|
|
7846
7970
|
message.mysql = $root.sqlanvil.MysqlOptions.fromObject(object.mysql, long + 1);
|
|
7847
7971
|
}
|
|
7972
|
+
switch (object.incrementalStrategy) {
|
|
7973
|
+
default:
|
|
7974
|
+
if (typeof object.incrementalStrategy === "number") {
|
|
7975
|
+
message.incrementalStrategy = object.incrementalStrategy;
|
|
7976
|
+
break;
|
|
7977
|
+
}
|
|
7978
|
+
break;
|
|
7979
|
+
case "INCREMENTAL_STRATEGY_UNSPECIFIED":
|
|
7980
|
+
case 0:
|
|
7981
|
+
message.incrementalStrategy = 0;
|
|
7982
|
+
break;
|
|
7983
|
+
case "INCREMENTAL_STRATEGY_MERGE":
|
|
7984
|
+
case 1:
|
|
7985
|
+
message.incrementalStrategy = 1;
|
|
7986
|
+
break;
|
|
7987
|
+
case "INCREMENTAL_STRATEGY_INSERT_OVERWRITE":
|
|
7988
|
+
case 2:
|
|
7989
|
+
message.incrementalStrategy = 2;
|
|
7990
|
+
break;
|
|
7991
|
+
}
|
|
7992
|
+
if (object.incrementalPredicates) {
|
|
7993
|
+
if (!Array.isArray(object.incrementalPredicates))
|
|
7994
|
+
throw TypeError(".sqlanvil.ActionConfig.IncrementalTableConfig.incrementalPredicates: array expected");
|
|
7995
|
+
message.incrementalPredicates = [];
|
|
7996
|
+
for (let i = 0; i < object.incrementalPredicates.length; ++i)
|
|
7997
|
+
message.incrementalPredicates[i] = String(object.incrementalPredicates[i]);
|
|
7998
|
+
}
|
|
7848
7999
|
return message;
|
|
7849
8000
|
};
|
|
7850
8001
|
|
|
@@ -7873,6 +8024,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7873
8024
|
object.uniqueKey = [];
|
|
7874
8025
|
object.columns = [];
|
|
7875
8026
|
object.clusterBy = [];
|
|
8027
|
+
object.incrementalPredicates = [];
|
|
7876
8028
|
}
|
|
7877
8029
|
if (options.objects || options.defaults) {
|
|
7878
8030
|
object.labels = {};
|
|
@@ -7900,6 +8052,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7900
8052
|
object.postgres = null;
|
|
7901
8053
|
object.supabase = null;
|
|
7902
8054
|
object.mysql = null;
|
|
8055
|
+
object.incrementalStrategy = options.enums === String ? "INCREMENTAL_STRATEGY_UNSPECIFIED" : 0;
|
|
7903
8056
|
}
|
|
7904
8057
|
if (message.name != null && Object.hasOwnProperty.call(message, "name"))
|
|
7905
8058
|
object.name = message.name;
|
|
@@ -7995,6 +8148,13 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
7995
8148
|
object.supabase = $root.sqlanvil.SupabaseOptions.toObject(message.supabase, options, q + 1);
|
|
7996
8149
|
if (message.mysql != null && Object.hasOwnProperty.call(message, "mysql"))
|
|
7997
8150
|
object.mysql = $root.sqlanvil.MysqlOptions.toObject(message.mysql, options, q + 1);
|
|
8151
|
+
if (message.incrementalStrategy != null && Object.hasOwnProperty.call(message, "incrementalStrategy"))
|
|
8152
|
+
object.incrementalStrategy = options.enums === String ? $root.sqlanvil.ActionConfig.IncrementalStrategy[message.incrementalStrategy] === undefined ? message.incrementalStrategy : $root.sqlanvil.ActionConfig.IncrementalStrategy[message.incrementalStrategy] : message.incrementalStrategy;
|
|
8153
|
+
if (message.incrementalPredicates && message.incrementalPredicates.length) {
|
|
8154
|
+
object.incrementalPredicates = [];
|
|
8155
|
+
for (let j = 0; j < message.incrementalPredicates.length; ++j)
|
|
8156
|
+
object.incrementalPredicates[j] = message.incrementalPredicates[j];
|
|
8157
|
+
}
|
|
7998
8158
|
return object;
|
|
7999
8159
|
};
|
|
8000
8160
|
|
|
@@ -21165,6 +21325,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21165
21325
|
* @property {boolean|null} [includeTestsInCompiledGraph] ProjectConfig includeTestsInCompiledGraph
|
|
21166
21326
|
* @property {Object.<string,sqlanvil.IConnectionConfig>|null} [connections] ProjectConfig connections
|
|
21167
21327
|
* @property {string|null} [warehouseConnection] ProjectConfig warehouseConnection
|
|
21328
|
+
* @property {boolean|null} [preserveGovernanceControls] ProjectConfig preserveGovernanceControls
|
|
21168
21329
|
*/
|
|
21169
21330
|
|
|
21170
21331
|
/**
|
|
@@ -21320,6 +21481,14 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21320
21481
|
*/
|
|
21321
21482
|
ProjectConfig.prototype.warehouseConnection = "";
|
|
21322
21483
|
|
|
21484
|
+
/**
|
|
21485
|
+
* ProjectConfig preserveGovernanceControls.
|
|
21486
|
+
* @member {boolean} preserveGovernanceControls
|
|
21487
|
+
* @memberof sqlanvil.ProjectConfig
|
|
21488
|
+
* @instance
|
|
21489
|
+
*/
|
|
21490
|
+
ProjectConfig.prototype.preserveGovernanceControls = false;
|
|
21491
|
+
|
|
21323
21492
|
/**
|
|
21324
21493
|
* Creates a new ProjectConfig instance using the specified properties.
|
|
21325
21494
|
* @function create
|
|
@@ -21386,6 +21555,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21386
21555
|
}
|
|
21387
21556
|
if (message.warehouseConnection != null && Object.hasOwnProperty.call(message, "warehouseConnection"))
|
|
21388
21557
|
writer.uint32(/* id 24, wireType 2 =*/194).string(message.warehouseConnection);
|
|
21558
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
21559
|
+
writer.uint32(/* id 25, wireType 0 =*/200).bool(message.preserveGovernanceControls);
|
|
21389
21560
|
return writer;
|
|
21390
21561
|
};
|
|
21391
21562
|
|
|
@@ -21536,6 +21707,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21536
21707
|
message.warehouseConnection = reader.string();
|
|
21537
21708
|
break;
|
|
21538
21709
|
}
|
|
21710
|
+
case 25: {
|
|
21711
|
+
message.preserveGovernanceControls = reader.bool();
|
|
21712
|
+
break;
|
|
21713
|
+
}
|
|
21539
21714
|
default:
|
|
21540
21715
|
reader.skipType(tag & 7, long);
|
|
21541
21716
|
break;
|
|
@@ -21642,6 +21817,9 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21642
21817
|
if (message.warehouseConnection != null && Object.hasOwnProperty.call(message, "warehouseConnection"))
|
|
21643
21818
|
if (!$util.isString(message.warehouseConnection))
|
|
21644
21819
|
return "warehouseConnection: string expected";
|
|
21820
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
21821
|
+
if (typeof message.preserveGovernanceControls !== "boolean")
|
|
21822
|
+
return "preserveGovernanceControls: boolean expected";
|
|
21645
21823
|
return null;
|
|
21646
21824
|
};
|
|
21647
21825
|
|
|
@@ -21721,6 +21899,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21721
21899
|
}
|
|
21722
21900
|
if (object.warehouseConnection != null)
|
|
21723
21901
|
message.warehouseConnection = String(object.warehouseConnection);
|
|
21902
|
+
if (object.preserveGovernanceControls != null)
|
|
21903
|
+
message.preserveGovernanceControls = Boolean(object.preserveGovernanceControls);
|
|
21724
21904
|
return message;
|
|
21725
21905
|
};
|
|
21726
21906
|
|
|
@@ -21761,6 +21941,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21761
21941
|
object.defaultReservation = "";
|
|
21762
21942
|
object.includeTestsInCompiledGraph = false;
|
|
21763
21943
|
object.warehouseConnection = "";
|
|
21944
|
+
object.preserveGovernanceControls = false;
|
|
21764
21945
|
}
|
|
21765
21946
|
if (message.warehouse != null && Object.hasOwnProperty.call(message, "warehouse"))
|
|
21766
21947
|
object.warehouse = message.warehouse;
|
|
@@ -21809,6 +21990,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
21809
21990
|
}
|
|
21810
21991
|
if (message.warehouseConnection != null && Object.hasOwnProperty.call(message, "warehouseConnection"))
|
|
21811
21992
|
object.warehouseConnection = message.warehouseConnection;
|
|
21993
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
21994
|
+
object.preserveGovernanceControls = message.preserveGovernanceControls;
|
|
21812
21995
|
return object;
|
|
21813
21996
|
};
|
|
21814
21997
|
|
|
@@ -22553,6 +22736,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
22553
22736
|
* @property {sqlanvil.TableFormat|null} [tableFormat] BigQueryOptions tableFormat
|
|
22554
22737
|
* @property {sqlanvil.FileFormat|null} [fileFormat] BigQueryOptions fileFormat
|
|
22555
22738
|
* @property {string|null} [storageUri] BigQueryOptions storageUri
|
|
22739
|
+
* @property {Array.<string>|null} [incrementalPredicates] BigQueryOptions incrementalPredicates
|
|
22740
|
+
* @property {boolean|null} [preserveGovernanceControls] BigQueryOptions preserveGovernanceControls
|
|
22556
22741
|
*/
|
|
22557
22742
|
|
|
22558
22743
|
/**
|
|
@@ -22567,6 +22752,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
22567
22752
|
this.clusterBy = [];
|
|
22568
22753
|
this.labels = {};
|
|
22569
22754
|
this.additionalOptions = {};
|
|
22755
|
+
this.incrementalPredicates = [];
|
|
22570
22756
|
if (properties)
|
|
22571
22757
|
for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
|
|
22572
22758
|
if (properties[keys[i]] != null && keys[i] !== "__proto__")
|
|
@@ -22661,6 +22847,22 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
22661
22847
|
*/
|
|
22662
22848
|
BigQueryOptions.prototype.storageUri = "";
|
|
22663
22849
|
|
|
22850
|
+
/**
|
|
22851
|
+
* BigQueryOptions incrementalPredicates.
|
|
22852
|
+
* @member {Array.<string>} incrementalPredicates
|
|
22853
|
+
* @memberof sqlanvil.BigQueryOptions
|
|
22854
|
+
* @instance
|
|
22855
|
+
*/
|
|
22856
|
+
BigQueryOptions.prototype.incrementalPredicates = $util.emptyArray;
|
|
22857
|
+
|
|
22858
|
+
/**
|
|
22859
|
+
* BigQueryOptions preserveGovernanceControls.
|
|
22860
|
+
* @member {boolean} preserveGovernanceControls
|
|
22861
|
+
* @memberof sqlanvil.BigQueryOptions
|
|
22862
|
+
* @instance
|
|
22863
|
+
*/
|
|
22864
|
+
BigQueryOptions.prototype.preserveGovernanceControls = false;
|
|
22865
|
+
|
|
22664
22866
|
/**
|
|
22665
22867
|
* Creates a new BigQueryOptions instance using the specified properties.
|
|
22666
22868
|
* @function create
|
|
@@ -22714,6 +22916,11 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
22714
22916
|
writer.uint32(/* id 10, wireType 0 =*/80).int32(message.fileFormat);
|
|
22715
22917
|
if (message.storageUri != null && Object.hasOwnProperty.call(message, "storageUri"))
|
|
22716
22918
|
writer.uint32(/* id 11, wireType 2 =*/90).string(message.storageUri);
|
|
22919
|
+
if (message.incrementalPredicates != null && message.incrementalPredicates.length)
|
|
22920
|
+
for (let i = 0; i < message.incrementalPredicates.length; ++i)
|
|
22921
|
+
writer.uint32(/* id 12, wireType 2 =*/98).string(message.incrementalPredicates[i]);
|
|
22922
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
22923
|
+
writer.uint32(/* id 13, wireType 0 =*/104).bool(message.preserveGovernanceControls);
|
|
22717
22924
|
return writer;
|
|
22718
22925
|
};
|
|
22719
22926
|
|
|
@@ -22842,6 +23049,16 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
22842
23049
|
message.storageUri = reader.string();
|
|
22843
23050
|
break;
|
|
22844
23051
|
}
|
|
23052
|
+
case 12: {
|
|
23053
|
+
if (!(message.incrementalPredicates && message.incrementalPredicates.length))
|
|
23054
|
+
message.incrementalPredicates = [];
|
|
23055
|
+
message.incrementalPredicates.push(reader.string());
|
|
23056
|
+
break;
|
|
23057
|
+
}
|
|
23058
|
+
case 13: {
|
|
23059
|
+
message.preserveGovernanceControls = reader.bool();
|
|
23060
|
+
break;
|
|
23061
|
+
}
|
|
22845
23062
|
default:
|
|
22846
23063
|
reader.skipType(tag & 7, long);
|
|
22847
23064
|
break;
|
|
@@ -22938,6 +23155,16 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
22938
23155
|
if (message.storageUri != null && Object.hasOwnProperty.call(message, "storageUri"))
|
|
22939
23156
|
if (!$util.isString(message.storageUri))
|
|
22940
23157
|
return "storageUri: string expected";
|
|
23158
|
+
if (message.incrementalPredicates != null && Object.hasOwnProperty.call(message, "incrementalPredicates")) {
|
|
23159
|
+
if (!Array.isArray(message.incrementalPredicates))
|
|
23160
|
+
return "incrementalPredicates: array expected";
|
|
23161
|
+
for (let i = 0; i < message.incrementalPredicates.length; ++i)
|
|
23162
|
+
if (!$util.isString(message.incrementalPredicates[i]))
|
|
23163
|
+
return "incrementalPredicates: string[] expected";
|
|
23164
|
+
}
|
|
23165
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
23166
|
+
if (typeof message.preserveGovernanceControls !== "boolean")
|
|
23167
|
+
return "preserveGovernanceControls: boolean expected";
|
|
22941
23168
|
return null;
|
|
22942
23169
|
};
|
|
22943
23170
|
|
|
@@ -23030,6 +23257,15 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
23030
23257
|
}
|
|
23031
23258
|
if (object.storageUri != null)
|
|
23032
23259
|
message.storageUri = String(object.storageUri);
|
|
23260
|
+
if (object.incrementalPredicates) {
|
|
23261
|
+
if (!Array.isArray(object.incrementalPredicates))
|
|
23262
|
+
throw TypeError(".sqlanvil.BigQueryOptions.incrementalPredicates: array expected");
|
|
23263
|
+
message.incrementalPredicates = [];
|
|
23264
|
+
for (let i = 0; i < object.incrementalPredicates.length; ++i)
|
|
23265
|
+
message.incrementalPredicates[i] = String(object.incrementalPredicates[i]);
|
|
23266
|
+
}
|
|
23267
|
+
if (object.preserveGovernanceControls != null)
|
|
23268
|
+
message.preserveGovernanceControls = Boolean(object.preserveGovernanceControls);
|
|
23033
23269
|
return message;
|
|
23034
23270
|
};
|
|
23035
23271
|
|
|
@@ -23050,8 +23286,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
23050
23286
|
if (q > $util.recursionLimit)
|
|
23051
23287
|
throw Error("max depth exceeded");
|
|
23052
23288
|
let object = {};
|
|
23053
|
-
if (options.arrays || options.defaults)
|
|
23289
|
+
if (options.arrays || options.defaults) {
|
|
23054
23290
|
object.clusterBy = [];
|
|
23291
|
+
object.incrementalPredicates = [];
|
|
23292
|
+
}
|
|
23055
23293
|
if (options.objects || options.defaults) {
|
|
23056
23294
|
object.labels = {};
|
|
23057
23295
|
object.additionalOptions = {};
|
|
@@ -23065,6 +23303,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
23065
23303
|
object.tableFormat = options.enums === String ? "TABLE_FORMAT_UNSPECIFIED" : 0;
|
|
23066
23304
|
object.fileFormat = options.enums === String ? "FILE_FORMAT_UNSPECIFIED" : 0;
|
|
23067
23305
|
object.storageUri = "";
|
|
23306
|
+
object.preserveGovernanceControls = false;
|
|
23068
23307
|
}
|
|
23069
23308
|
if (message.partitionBy != null && Object.hasOwnProperty.call(message, "partitionBy"))
|
|
23070
23309
|
object.partitionBy = message.partitionBy;
|
|
@@ -23104,6 +23343,13 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
23104
23343
|
object.fileFormat = options.enums === String ? $root.sqlanvil.FileFormat[message.fileFormat] === undefined ? message.fileFormat : $root.sqlanvil.FileFormat[message.fileFormat] : message.fileFormat;
|
|
23105
23344
|
if (message.storageUri != null && Object.hasOwnProperty.call(message, "storageUri"))
|
|
23106
23345
|
object.storageUri = message.storageUri;
|
|
23346
|
+
if (message.incrementalPredicates && message.incrementalPredicates.length) {
|
|
23347
|
+
object.incrementalPredicates = [];
|
|
23348
|
+
for (let j = 0; j < message.incrementalPredicates.length; ++j)
|
|
23349
|
+
object.incrementalPredicates[j] = message.incrementalPredicates[j];
|
|
23350
|
+
}
|
|
23351
|
+
if (message.preserveGovernanceControls != null && Object.hasOwnProperty.call(message, "preserveGovernanceControls"))
|
|
23352
|
+
object.preserveGovernanceControls = message.preserveGovernanceControls;
|
|
23107
23353
|
return object;
|
|
23108
23354
|
};
|
|
23109
23355
|
|
|
@@ -24825,6 +25071,22 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
24825
25071
|
return values;
|
|
24826
25072
|
})();
|
|
24827
25073
|
|
|
25074
|
+
/**
|
|
25075
|
+
* IncrementalStrategy enum.
|
|
25076
|
+
* @name sqlanvil.IncrementalStrategy
|
|
25077
|
+
* @enum {number}
|
|
25078
|
+
* @property {number} INCREMENTAL_STRATEGY_UNSPECIFIED=0 INCREMENTAL_STRATEGY_UNSPECIFIED value
|
|
25079
|
+
* @property {number} MERGE=1 MERGE value
|
|
25080
|
+
* @property {number} INSERT_OVERWRITE=2 INSERT_OVERWRITE value
|
|
25081
|
+
*/
|
|
25082
|
+
sqlanvil.IncrementalStrategy = (function() {
|
|
25083
|
+
const valuesById = {}, values = Object.create(valuesById);
|
|
25084
|
+
values[valuesById[0] = "INCREMENTAL_STRATEGY_UNSPECIFIED"] = 0;
|
|
25085
|
+
values[valuesById[1] = "MERGE"] = 1;
|
|
25086
|
+
values[valuesById[2] = "INSERT_OVERWRITE"] = 2;
|
|
25087
|
+
return values;
|
|
25088
|
+
})();
|
|
25089
|
+
|
|
24828
25090
|
sqlanvil.Table = (function() {
|
|
24829
25091
|
|
|
24830
25092
|
/**
|
|
@@ -24852,6 +25114,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
24852
25114
|
* @property {Array.<string>|null} [incrementalPreOps] Table incrementalPreOps
|
|
24853
25115
|
* @property {Array.<string>|null} [incrementalPostOps] Table incrementalPostOps
|
|
24854
25116
|
* @property {string|null} [jitCode] Table jitCode
|
|
25117
|
+
* @property {sqlanvil.IncrementalStrategy|null} [incrementalStrategy] Table incrementalStrategy
|
|
24855
25118
|
* @property {sqlanvil.IBigQueryOptions|null} [bigquery] Table bigquery
|
|
24856
25119
|
* @property {sqlanvil.IPostgresOptions|null} [postgres] Table postgres
|
|
24857
25120
|
* @property {sqlanvil.ISupabaseOptions|null} [supabase] Table supabase
|
|
@@ -25049,6 +25312,14 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25049
25312
|
*/
|
|
25050
25313
|
Table.prototype.jitCode = "";
|
|
25051
25314
|
|
|
25315
|
+
/**
|
|
25316
|
+
* Table incrementalStrategy.
|
|
25317
|
+
* @member {sqlanvil.IncrementalStrategy} incrementalStrategy
|
|
25318
|
+
* @memberof sqlanvil.Table
|
|
25319
|
+
* @instance
|
|
25320
|
+
*/
|
|
25321
|
+
Table.prototype.incrementalStrategy = 0;
|
|
25322
|
+
|
|
25052
25323
|
/**
|
|
25053
25324
|
* Table bigquery.
|
|
25054
25325
|
* @member {sqlanvil.IBigQueryOptions|null|undefined} bigquery
|
|
@@ -25176,6 +25447,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25176
25447
|
$root.sqlanvil.SupabaseOptions.encode(message.supabase, writer.uint32(/* id 40, wireType 2 =*/322).fork(), q + 1).ldelim();
|
|
25177
25448
|
if (message.mysql != null && Object.hasOwnProperty.call(message, "mysql"))
|
|
25178
25449
|
$root.sqlanvil.MysqlOptions.encode(message.mysql, writer.uint32(/* id 41, wireType 2 =*/330).fork(), q + 1).ldelim();
|
|
25450
|
+
if (message.incrementalStrategy != null && Object.hasOwnProperty.call(message, "incrementalStrategy"))
|
|
25451
|
+
writer.uint32(/* id 42, wireType 0 =*/336).int32(message.incrementalStrategy);
|
|
25179
25452
|
return writer;
|
|
25180
25453
|
};
|
|
25181
25454
|
|
|
@@ -25314,6 +25587,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25314
25587
|
message.jitCode = reader.string();
|
|
25315
25588
|
break;
|
|
25316
25589
|
}
|
|
25590
|
+
case 42: {
|
|
25591
|
+
message.incrementalStrategy = reader.int32();
|
|
25592
|
+
break;
|
|
25593
|
+
}
|
|
25317
25594
|
case 22: {
|
|
25318
25595
|
message.bigquery = $root.sqlanvil.BigQueryOptions.decode(reader, reader.uint32(), undefined, long + 1);
|
|
25319
25596
|
break;
|
|
@@ -25492,6 +25769,15 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25492
25769
|
if (message.jitCode != null && Object.hasOwnProperty.call(message, "jitCode"))
|
|
25493
25770
|
if (!$util.isString(message.jitCode))
|
|
25494
25771
|
return "jitCode: string expected";
|
|
25772
|
+
if (message.incrementalStrategy != null && Object.hasOwnProperty.call(message, "incrementalStrategy"))
|
|
25773
|
+
switch (message.incrementalStrategy) {
|
|
25774
|
+
default:
|
|
25775
|
+
return "incrementalStrategy: enum value expected";
|
|
25776
|
+
case 0:
|
|
25777
|
+
case 1:
|
|
25778
|
+
case 2:
|
|
25779
|
+
break;
|
|
25780
|
+
}
|
|
25495
25781
|
if (message.bigquery != null && Object.hasOwnProperty.call(message, "bigquery")) {
|
|
25496
25782
|
let error = $root.sqlanvil.BigQueryOptions.verify(message.bigquery, long + 1);
|
|
25497
25783
|
if (error)
|
|
@@ -25687,6 +25973,26 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25687
25973
|
}
|
|
25688
25974
|
if (object.jitCode != null)
|
|
25689
25975
|
message.jitCode = String(object.jitCode);
|
|
25976
|
+
switch (object.incrementalStrategy) {
|
|
25977
|
+
default:
|
|
25978
|
+
if (typeof object.incrementalStrategy === "number") {
|
|
25979
|
+
message.incrementalStrategy = object.incrementalStrategy;
|
|
25980
|
+
break;
|
|
25981
|
+
}
|
|
25982
|
+
break;
|
|
25983
|
+
case "INCREMENTAL_STRATEGY_UNSPECIFIED":
|
|
25984
|
+
case 0:
|
|
25985
|
+
message.incrementalStrategy = 0;
|
|
25986
|
+
break;
|
|
25987
|
+
case "MERGE":
|
|
25988
|
+
case 1:
|
|
25989
|
+
message.incrementalStrategy = 1;
|
|
25990
|
+
break;
|
|
25991
|
+
case "INSERT_OVERWRITE":
|
|
25992
|
+
case 2:
|
|
25993
|
+
message.incrementalStrategy = 2;
|
|
25994
|
+
break;
|
|
25995
|
+
}
|
|
25690
25996
|
if (object.bigquery != null) {
|
|
25691
25997
|
if (!$util.isObject(object.bigquery))
|
|
25692
25998
|
throw TypeError(".sqlanvil.Table.bigquery: object expected");
|
|
@@ -25758,6 +26064,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25758
26064
|
object.postgres = null;
|
|
25759
26065
|
object.supabase = null;
|
|
25760
26066
|
object.mysql = null;
|
|
26067
|
+
object.incrementalStrategy = options.enums === String ? "INCREMENTAL_STRATEGY_UNSPECIFIED" : 0;
|
|
25761
26068
|
}
|
|
25762
26069
|
if (message.type != null && Object.hasOwnProperty.call(message, "type"))
|
|
25763
26070
|
object.type = message.type;
|
|
@@ -25832,6 +26139,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
25832
26139
|
object.supabase = $root.sqlanvil.SupabaseOptions.toObject(message.supabase, options, q + 1);
|
|
25833
26140
|
if (message.mysql != null && Object.hasOwnProperty.call(message, "mysql"))
|
|
25834
26141
|
object.mysql = $root.sqlanvil.MysqlOptions.toObject(message.mysql, options, q + 1);
|
|
26142
|
+
if (message.incrementalStrategy != null && Object.hasOwnProperty.call(message, "incrementalStrategy"))
|
|
26143
|
+
object.incrementalStrategy = options.enums === String ? $root.sqlanvil.IncrementalStrategy[message.incrementalStrategy] === undefined ? message.incrementalStrategy : $root.sqlanvil.IncrementalStrategy[message.incrementalStrategy] : message.incrementalStrategy;
|
|
25835
26144
|
return object;
|
|
25836
26145
|
};
|
|
25837
26146
|
|
|
@@ -37843,6 +38152,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
37843
38152
|
* @property {boolean|null} [includeDependents] RunConfig includeDependents
|
|
37844
38153
|
* @property {boolean|null} [fullRefresh] RunConfig fullRefresh
|
|
37845
38154
|
* @property {number|null} [timeoutMillis] RunConfig timeoutMillis
|
|
38155
|
+
* @property {number|null} [jitTimeoutMillis] RunConfig jitTimeoutMillis
|
|
37846
38156
|
* @property {boolean|null} [disableSetMetadata] RunConfig disableSetMetadata
|
|
37847
38157
|
* @property {boolean|null} [useRunCache] RunConfig useRunCache
|
|
37848
38158
|
*/
|
|
@@ -37912,6 +38222,14 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
37912
38222
|
*/
|
|
37913
38223
|
RunConfig.prototype.timeoutMillis = 0;
|
|
37914
38224
|
|
|
38225
|
+
/**
|
|
38226
|
+
* RunConfig jitTimeoutMillis.
|
|
38227
|
+
* @member {number} jitTimeoutMillis
|
|
38228
|
+
* @memberof sqlanvil.RunConfig
|
|
38229
|
+
* @instance
|
|
38230
|
+
*/
|
|
38231
|
+
RunConfig.prototype.jitTimeoutMillis = 0;
|
|
38232
|
+
|
|
37915
38233
|
/**
|
|
37916
38234
|
* RunConfig disableSetMetadata.
|
|
37917
38235
|
* @member {boolean} disableSetMetadata
|
|
@@ -37972,6 +38290,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
37972
38290
|
writer.uint32(/* id 8, wireType 0 =*/64).bool(message.useRunCache);
|
|
37973
38291
|
if (message.disableSetMetadata != null && Object.hasOwnProperty.call(message, "disableSetMetadata"))
|
|
37974
38292
|
writer.uint32(/* id 9, wireType 0 =*/72).bool(message.disableSetMetadata);
|
|
38293
|
+
if (message.jitTimeoutMillis != null && Object.hasOwnProperty.call(message, "jitTimeoutMillis"))
|
|
38294
|
+
writer.uint32(/* id 10, wireType 0 =*/80).int32(message.jitTimeoutMillis);
|
|
37975
38295
|
if (message.includeDependents != null && Object.hasOwnProperty.call(message, "includeDependents"))
|
|
37976
38296
|
writer.uint32(/* id 11, wireType 0 =*/88).bool(message.includeDependents);
|
|
37977
38297
|
return writer;
|
|
@@ -38042,6 +38362,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
38042
38362
|
message.timeoutMillis = reader.int32();
|
|
38043
38363
|
break;
|
|
38044
38364
|
}
|
|
38365
|
+
case 10: {
|
|
38366
|
+
message.jitTimeoutMillis = reader.int32();
|
|
38367
|
+
break;
|
|
38368
|
+
}
|
|
38045
38369
|
case 9: {
|
|
38046
38370
|
message.disableSetMetadata = reader.bool();
|
|
38047
38371
|
break;
|
|
@@ -38115,6 +38439,9 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
38115
38439
|
if (message.timeoutMillis != null && Object.hasOwnProperty.call(message, "timeoutMillis"))
|
|
38116
38440
|
if (!$util.isInteger(message.timeoutMillis))
|
|
38117
38441
|
return "timeoutMillis: integer expected";
|
|
38442
|
+
if (message.jitTimeoutMillis != null && Object.hasOwnProperty.call(message, "jitTimeoutMillis"))
|
|
38443
|
+
if (!$util.isInteger(message.jitTimeoutMillis))
|
|
38444
|
+
return "jitTimeoutMillis: integer expected";
|
|
38118
38445
|
if (message.disableSetMetadata != null && Object.hasOwnProperty.call(message, "disableSetMetadata"))
|
|
38119
38446
|
if (typeof message.disableSetMetadata !== "boolean")
|
|
38120
38447
|
return "disableSetMetadata: boolean expected";
|
|
@@ -38164,6 +38491,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
38164
38491
|
message.fullRefresh = Boolean(object.fullRefresh);
|
|
38165
38492
|
if (object.timeoutMillis != null)
|
|
38166
38493
|
message.timeoutMillis = object.timeoutMillis | 0;
|
|
38494
|
+
if (object.jitTimeoutMillis != null)
|
|
38495
|
+
message.jitTimeoutMillis = object.jitTimeoutMillis | 0;
|
|
38167
38496
|
if (object.disableSetMetadata != null)
|
|
38168
38497
|
message.disableSetMetadata = Boolean(object.disableSetMetadata);
|
|
38169
38498
|
if (object.useRunCache != null)
|
|
@@ -38198,6 +38527,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
38198
38527
|
object.timeoutMillis = 0;
|
|
38199
38528
|
object.useRunCache = false;
|
|
38200
38529
|
object.disableSetMetadata = false;
|
|
38530
|
+
object.jitTimeoutMillis = 0;
|
|
38201
38531
|
object.includeDependents = false;
|
|
38202
38532
|
}
|
|
38203
38533
|
if (message.actions && message.actions.length) {
|
|
@@ -38220,6 +38550,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
38220
38550
|
object.useRunCache = message.useRunCache;
|
|
38221
38551
|
if (message.disableSetMetadata != null && Object.hasOwnProperty.call(message, "disableSetMetadata"))
|
|
38222
38552
|
object.disableSetMetadata = message.disableSetMetadata;
|
|
38553
|
+
if (message.jitTimeoutMillis != null && Object.hasOwnProperty.call(message, "jitTimeoutMillis"))
|
|
38554
|
+
object.jitTimeoutMillis = message.jitTimeoutMillis;
|
|
38223
38555
|
if (message.includeDependents != null && Object.hasOwnProperty.call(message, "includeDependents"))
|
|
38224
38556
|
object.includeDependents = message.includeDependents;
|
|
38225
38557
|
return object;
|
|
@@ -42358,6 +42690,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42358
42690
|
* @property {string|null} [errorMessage] TaskResult errorMessage
|
|
42359
42691
|
* @property {sqlanvil.ITiming|null} [timing] TaskResult timing
|
|
42360
42692
|
* @property {sqlanvil.IExecutionMetadata|null} [metadata] TaskResult metadata
|
|
42693
|
+
* @property {string|null} [compiledSql] TaskResult compiledSql
|
|
42361
42694
|
*/
|
|
42362
42695
|
|
|
42363
42696
|
/**
|
|
@@ -42407,6 +42740,14 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42407
42740
|
*/
|
|
42408
42741
|
TaskResult.prototype.metadata = null;
|
|
42409
42742
|
|
|
42743
|
+
/**
|
|
42744
|
+
* TaskResult compiledSql.
|
|
42745
|
+
* @member {string} compiledSql
|
|
42746
|
+
* @memberof sqlanvil.TaskResult
|
|
42747
|
+
* @instance
|
|
42748
|
+
*/
|
|
42749
|
+
TaskResult.prototype.compiledSql = "";
|
|
42750
|
+
|
|
42410
42751
|
/**
|
|
42411
42752
|
* Creates a new TaskResult instance using the specified properties.
|
|
42412
42753
|
* @function create
|
|
@@ -42443,6 +42784,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42443
42784
|
$root.sqlanvil.Timing.encode(message.timing, writer.uint32(/* id 3, wireType 2 =*/26).fork(), q + 1).ldelim();
|
|
42444
42785
|
if (message.metadata != null && Object.hasOwnProperty.call(message, "metadata"))
|
|
42445
42786
|
$root.sqlanvil.ExecutionMetadata.encode(message.metadata, writer.uint32(/* id 4, wireType 2 =*/34).fork(), q + 1).ldelim();
|
|
42787
|
+
if (message.compiledSql != null && Object.hasOwnProperty.call(message, "compiledSql"))
|
|
42788
|
+
writer.uint32(/* id 5, wireType 2 =*/42).string(message.compiledSql);
|
|
42446
42789
|
return writer;
|
|
42447
42790
|
};
|
|
42448
42791
|
|
|
@@ -42499,6 +42842,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42499
42842
|
message.metadata = $root.sqlanvil.ExecutionMetadata.decode(reader, reader.uint32(), undefined, long + 1);
|
|
42500
42843
|
break;
|
|
42501
42844
|
}
|
|
42845
|
+
case 5: {
|
|
42846
|
+
message.compiledSql = reader.string();
|
|
42847
|
+
break;
|
|
42848
|
+
}
|
|
42502
42849
|
default:
|
|
42503
42850
|
reader.skipType(tag & 7, long);
|
|
42504
42851
|
break;
|
|
@@ -42563,6 +42910,9 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42563
42910
|
if (error)
|
|
42564
42911
|
return "metadata." + error;
|
|
42565
42912
|
}
|
|
42913
|
+
if (message.compiledSql != null && Object.hasOwnProperty.call(message, "compiledSql"))
|
|
42914
|
+
if (!$util.isString(message.compiledSql))
|
|
42915
|
+
return "compiledSql: string expected";
|
|
42566
42916
|
return null;
|
|
42567
42917
|
};
|
|
42568
42918
|
|
|
@@ -42628,6 +42978,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42628
42978
|
throw TypeError(".sqlanvil.TaskResult.metadata: object expected");
|
|
42629
42979
|
message.metadata = $root.sqlanvil.ExecutionMetadata.fromObject(object.metadata, long + 1);
|
|
42630
42980
|
}
|
|
42981
|
+
if (object.compiledSql != null)
|
|
42982
|
+
message.compiledSql = String(object.compiledSql);
|
|
42631
42983
|
return message;
|
|
42632
42984
|
};
|
|
42633
42985
|
|
|
@@ -42653,6 +43005,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42653
43005
|
object.errorMessage = "";
|
|
42654
43006
|
object.timing = null;
|
|
42655
43007
|
object.metadata = null;
|
|
43008
|
+
object.compiledSql = "";
|
|
42656
43009
|
}
|
|
42657
43010
|
if (message.status != null && Object.hasOwnProperty.call(message, "status"))
|
|
42658
43011
|
object.status = options.enums === String ? $root.sqlanvil.TaskResult.ExecutionStatus[message.status] === undefined ? message.status : $root.sqlanvil.TaskResult.ExecutionStatus[message.status] : message.status;
|
|
@@ -42662,6 +43015,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
42662
43015
|
object.timing = $root.sqlanvil.Timing.toObject(message.timing, options, q + 1);
|
|
42663
43016
|
if (message.metadata != null && Object.hasOwnProperty.call(message, "metadata"))
|
|
42664
43017
|
object.metadata = $root.sqlanvil.ExecutionMetadata.toObject(message.metadata, options, q + 1);
|
|
43018
|
+
if (message.compiledSql != null && Object.hasOwnProperty.call(message, "compiledSql"))
|
|
43019
|
+
object.compiledSql = message.compiledSql;
|
|
42665
43020
|
return object;
|
|
42666
43021
|
};
|
|
42667
43022
|
|
|
@@ -49519,15 +49874,41 @@ function resolveExportUri(spec, actionName, opts) {
|
|
|
49519
49874
|
return `${prefix}/${name}`;
|
|
49520
49875
|
}
|
|
49521
49876
|
|
|
49877
|
+
function formatStatement(statement, isLast, modifier) {
|
|
49878
|
+
const lastLineStart = statement.lastIndexOf("\n") + 1;
|
|
49879
|
+
const commentIndex = statement.indexOf("--", lastLineStart);
|
|
49880
|
+
let code;
|
|
49881
|
+
let comment;
|
|
49882
|
+
if (commentIndex === 0) {
|
|
49883
|
+
code = "";
|
|
49884
|
+
comment = statement;
|
|
49885
|
+
}
|
|
49886
|
+
else if (commentIndex > 0) {
|
|
49887
|
+
code = statement.substring(0, commentIndex).trimEnd();
|
|
49888
|
+
comment = statement.substring(commentIndex);
|
|
49889
|
+
}
|
|
49890
|
+
else {
|
|
49891
|
+
code = statement;
|
|
49892
|
+
comment = "";
|
|
49893
|
+
}
|
|
49894
|
+
if (modifier && code.length > 0) {
|
|
49895
|
+
code = modifier(code);
|
|
49896
|
+
}
|
|
49897
|
+
if (!isLast && code.length > 0 && !code.endsWith(";")) {
|
|
49898
|
+
code = code + ";";
|
|
49899
|
+
}
|
|
49900
|
+
if (code.length > 0 && comment.length > 0) {
|
|
49901
|
+
return code + " " + comment;
|
|
49902
|
+
}
|
|
49903
|
+
return code || comment;
|
|
49904
|
+
}
|
|
49522
49905
|
function concatenateQueries(statements, modifier) {
|
|
49523
|
-
|
|
49906
|
+
const processed = statements
|
|
49524
49907
|
.filter(statement => !!statement)
|
|
49525
49908
|
.map(statement => statement.trim())
|
|
49526
|
-
.
|
|
49527
|
-
|
|
49528
|
-
|
|
49529
|
-
.map(statement => (!!modifier ? modifier(statement) : statement))
|
|
49530
|
-
.join("\n;\n");
|
|
49909
|
+
.filter(statement => statement.length > 0);
|
|
49910
|
+
const formattedStatements = processed.map((statement, index) => formatStatement(statement, index === processed.length - 1, modifier));
|
|
49911
|
+
return formattedStatements.join("\n");
|
|
49531
49912
|
}
|
|
49532
49913
|
class Tasks {
|
|
49533
49914
|
constructor() {
|
|
@@ -49749,9 +50130,7 @@ from (${query}) as insertions`;
|
|
|
49749
50130
|
this.buildIncrementalSchemaChangeTasks(tasks, table);
|
|
49750
50131
|
case sqlanvil.OnSchemaChange.IGNORE:
|
|
49751
50132
|
default:
|
|
49752
|
-
tasks.add(Task.statement(table
|
|
49753
|
-
? this.mergeInto(table.target, tableMetadata === null || tableMetadata === void 0 ? void 0 : tableMetadata.fields.map(f => f.name), this.getIncrementalQuery(table), table.uniqueKey, table.bigquery && table.bigquery.updatePartitionFilter)
|
|
49754
|
-
: this.insertInto(table.target, tableMetadata === null || tableMetadata === void 0 ? void 0 : tableMetadata.fields.map(f => f.name).map(column => `\`${column}\``), this.getIncrementalQuery(table))));
|
|
50133
|
+
tasks.add(Task.statement(this.getIncrementalDmlStatement(table, (tableMetadata === null || tableMetadata === void 0 ? void 0 : tableMetadata.fields.map(f => f.name)) || [])));
|
|
49755
50134
|
break;
|
|
49756
50135
|
}
|
|
49757
50136
|
}
|
|
@@ -49988,14 +50367,68 @@ DROP TABLE IF EXISTS ${emptyTempTableName};
|
|
|
49988
50367
|
return `
|
|
49989
50368
|
create or replace view ${this.resolveTarget(target)} as ${query}`;
|
|
49990
50369
|
}
|
|
49991
|
-
|
|
50370
|
+
getIncrementalDmlStatement(table, columns) {
|
|
50371
|
+
const incrementalQuery = this.getIncrementalQuery(table);
|
|
50372
|
+
switch (table.incrementalStrategy) {
|
|
50373
|
+
case sqlanvil.IncrementalStrategy.INSERT_OVERWRITE:
|
|
50374
|
+
return this.insertOverwrite(table.target, columns, incrementalQuery, table.bigquery);
|
|
50375
|
+
case sqlanvil.IncrementalStrategy.MERGE:
|
|
50376
|
+
default:
|
|
50377
|
+
if (table.uniqueKey && table.uniqueKey.length > 0) {
|
|
50378
|
+
return this.mergeInto(table.target, columns, incrementalQuery, table.uniqueKey, table.bigquery);
|
|
50379
|
+
}
|
|
50380
|
+
return this.insertInto(table.target, columns.map(column => `\`${column}\``), incrementalQuery);
|
|
50381
|
+
}
|
|
50382
|
+
}
|
|
50383
|
+
buildIncrementalPredicatesString(incrementalPredicates) {
|
|
50384
|
+
var _a;
|
|
50385
|
+
const valid = (_a = incrementalPredicates === null || incrementalPredicates === void 0 ? void 0 : incrementalPredicates.filter(p => p.trim() !== "")) !== null && _a !== void 0 ? _a : [];
|
|
50386
|
+
if (valid.length === 0) {
|
|
50387
|
+
return "";
|
|
50388
|
+
}
|
|
50389
|
+
return `and ${valid.map(p => `(${p})`).join(" and ")}`;
|
|
50390
|
+
}
|
|
50391
|
+
insertOverwrite(target, columns, query, bigquery) {
|
|
50392
|
+
const partitionBy = bigquery === null || bigquery === void 0 ? void 0 : bigquery.partitionBy;
|
|
50393
|
+
const updatePartitionFilter = bigquery === null || bigquery === void 0 ? void 0 : bigquery.updatePartitionFilter;
|
|
50394
|
+
const incrementalPredicatesString = this.buildIncrementalPredicatesString(bigquery === null || bigquery === void 0 ? void 0 : bigquery.incrementalPredicates);
|
|
50395
|
+
const stagingTableUnqualified = `staging_table_temp_${this.uniqueIdGenerator()}`;
|
|
50396
|
+
const backtickedColumns = columns.map(column => `\`${column}\``);
|
|
50397
|
+
return `CREATE OR REPLACE TEMP TABLE \`${stagingTableUnqualified}\` AS (
|
|
50398
|
+
${query}
|
|
50399
|
+
);
|
|
50400
|
+
|
|
50401
|
+
BEGIN
|
|
50402
|
+
DECLARE partitions_for_replacement DEFAULT (
|
|
50403
|
+
ARRAY(
|
|
50404
|
+
SELECT DISTINCT ${partitionBy}
|
|
50405
|
+
FROM \`${stagingTableUnqualified}\`
|
|
50406
|
+
WHERE ${partitionBy} IS NOT NULL
|
|
50407
|
+
)
|
|
50408
|
+
);
|
|
50409
|
+
|
|
50410
|
+
MERGE ${this.resolveTarget(target)} T
|
|
50411
|
+
USING \`${stagingTableUnqualified}\` S
|
|
50412
|
+
ON FALSE
|
|
50413
|
+
WHEN NOT MATCHED BY SOURCE AND ${partitionBy} IN UNNEST(partitions_for_replacement)${updatePartitionFilter ? ` and T.${updatePartitionFilter}` : ""}${incrementalPredicatesString ? ` ${incrementalPredicatesString}` : ""}
|
|
50414
|
+
THEN
|
|
50415
|
+
DELETE
|
|
50416
|
+
WHEN NOT MATCHED BY TARGET THEN
|
|
50417
|
+
INSERT (${backtickedColumns.join(",")}) VALUES (${backtickedColumns.join(",")});
|
|
50418
|
+
END;
|
|
50419
|
+
|
|
50420
|
+
DROP TABLE IF EXISTS \`${stagingTableUnqualified}\`;`;
|
|
50421
|
+
}
|
|
50422
|
+
mergeInto(target, columns, query, uniqueKey, bigquery) {
|
|
50423
|
+
const updatePartitionFilter = bigquery === null || bigquery === void 0 ? void 0 : bigquery.updatePartitionFilter;
|
|
50424
|
+
const incrementalPredicatesString = this.buildIncrementalPredicatesString(bigquery === null || bigquery === void 0 ? void 0 : bigquery.incrementalPredicates);
|
|
49992
50425
|
const backtickedColumns = columns.map(column => `\`${column}\``);
|
|
49993
50426
|
return `
|
|
49994
50427
|
merge ${this.resolveTarget(target)} T
|
|
49995
50428
|
using (${query}
|
|
49996
50429
|
) S
|
|
49997
50430
|
on ${uniqueKey.map(uniqueKeyCol => `T.${uniqueKeyCol} = S.${uniqueKeyCol}`).join(` and `)}
|
|
49998
|
-
${updatePartitionFilter ? `and T.${updatePartitionFilter}` : ""}
|
|
50431
|
+
${updatePartitionFilter ? `and T.${updatePartitionFilter}` : ""}${incrementalPredicatesString ? `\n ${incrementalPredicatesString}` : ""}
|
|
49999
50432
|
when matched then
|
|
50000
50433
|
update set ${columns.map(column => `\`${column}\` = S.${column}`).join(",")}
|
|
50001
50434
|
when not matched then
|
|
@@ -50646,8 +51079,8 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
50646
51079
|
.filter(validationQuery => !!validationQuery.query);
|
|
50647
51080
|
}
|
|
50648
51081
|
|
|
50649
|
-
const version = "1.
|
|
50650
|
-
const dataformVersion = "3.0.
|
|
51082
|
+
const version = "1.29.0";
|
|
51083
|
+
const dataformVersion = "3.0.62";
|
|
50651
51084
|
|
|
50652
51085
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
50653
51086
|
const prunedGraph = prune(compiledGraph, runConfig);
|
|
@@ -51496,13 +51929,17 @@ function renderDeclarationSqlx(opts) {
|
|
|
51496
51929
|
lines.push(` schema: ${quote(opts.schema)},`);
|
|
51497
51930
|
}
|
|
51498
51931
|
lines.push(` name: ${quote(opts.name)},`);
|
|
51499
|
-
const typeLines = opts.columns.map(function (c) {
|
|
51932
|
+
const typeLines = opts.columns.map(function (c) {
|
|
51933
|
+
return ` ${keyToken(c.name.toLowerCase())}: ${quote(c.type)}`;
|
|
51934
|
+
});
|
|
51500
51935
|
const described = opts.columns.filter(function (c) { return !!c.description; });
|
|
51501
51936
|
lines.push(` columnTypes: {`);
|
|
51502
51937
|
lines.push(typeLines.join(",\n"));
|
|
51503
51938
|
lines.push(described.length > 0 ? ` },` : ` }`);
|
|
51504
51939
|
if (described.length > 0) {
|
|
51505
|
-
const descLines = described.map(function (c) {
|
|
51940
|
+
const descLines = described.map(function (c) {
|
|
51941
|
+
return ` ${keyToken(c.name.toLowerCase())}: ${quote(c.description)}`;
|
|
51942
|
+
});
|
|
51506
51943
|
lines.push(` columns: {`);
|
|
51507
51944
|
lines.push(descLines.join(",\n"));
|
|
51508
51945
|
lines.push(` }`);
|
|
@@ -52586,18 +53023,34 @@ class PostgresDbAdapter {
|
|
|
52586
53023
|
|
|
52587
53024
|
const MAX_PARAMS_PER_INSERT = 60000;
|
|
52588
53025
|
const quoteIdent = (id) => `"${String(id).replace(/"/g, '""')}"`;
|
|
53026
|
+
function foldColumns(columnTypes, target) {
|
|
53027
|
+
var _a;
|
|
53028
|
+
const cols = Object.keys(columnTypes).map(source => ({ source, target: source.toLowerCase() }));
|
|
53029
|
+
const byFolded = new Map();
|
|
53030
|
+
for (const c of cols) {
|
|
53031
|
+
byFolded.set(c.target, [...((_a = byFolded.get(c.target)) !== null && _a !== void 0 ? _a : []), c.source]);
|
|
53032
|
+
}
|
|
53033
|
+
for (const [folded, sources] of byFolded) {
|
|
53034
|
+
if (sources.length > 1) {
|
|
53035
|
+
throw new Error(`Source ${target.schema}.${target.name} has columns that differ only in case ` +
|
|
53036
|
+
`(${sources.join(", ")}); PostgreSQL folds them to the same identifier "${folded}". ` +
|
|
53037
|
+
`Rename one at the source, or select them with distinct aliases.`);
|
|
53038
|
+
}
|
|
53039
|
+
}
|
|
53040
|
+
return cols;
|
|
53041
|
+
}
|
|
52589
53042
|
async function createPgLoader(args) {
|
|
52590
53043
|
var _a;
|
|
52591
53044
|
const { target, columnTypes } = args;
|
|
52592
|
-
const cols = Object.keys(columnTypes);
|
|
52593
53045
|
const coerce = args.coerce || ((v) => (v === undefined ? null : v));
|
|
53046
|
+
const cols = foldColumns(columnTypes, target);
|
|
52594
53047
|
const pg = await PostgresDbAdapter.create(args.pg, {
|
|
52595
53048
|
concurrencyLimit: 1,
|
|
52596
53049
|
disableSslForTestsOnly: args.disableSslForTestsOnly
|
|
52597
53050
|
});
|
|
52598
53051
|
try {
|
|
52599
53052
|
const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
|
|
52600
|
-
const colDefs = cols.map(c => `${quoteIdent(c)} ${columnTypes[c]}`).join(", ");
|
|
53053
|
+
const colDefs = cols.map(c => `${quoteIdent(c.target)} ${columnTypes[c.source]}`).join(", ");
|
|
52601
53054
|
await pg.execute(`create schema if not exists ${quoteIdent(target.schema)}`);
|
|
52602
53055
|
const { rows: existing } = await pg.execute(`select c.relkind from pg_class c join pg_namespace n on n.oid = c.relnamespace ` +
|
|
52603
53056
|
`where n.nspname = $1 and c.relname = $2`, { params: [target.schema, target.name] });
|
|
@@ -52617,7 +53070,7 @@ async function createPgLoader(args) {
|
|
|
52617
53070
|
await pg.close();
|
|
52618
53071
|
throw e;
|
|
52619
53072
|
}
|
|
52620
|
-
const colIdents = cols.map(quoteIdent).join(", ");
|
|
53073
|
+
const colIdents = cols.map(c => quoteIdent(c.target)).join(", ");
|
|
52621
53074
|
const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
|
|
52622
53075
|
const batchRows = Math.max(1, Math.min(1000, Math.floor(MAX_PARAMS_PER_INSERT / cols.length)));
|
|
52623
53076
|
return {
|
|
@@ -52628,7 +53081,7 @@ async function createPgLoader(args) {
|
|
|
52628
53081
|
const params = [];
|
|
52629
53082
|
const tuples = batch.map((row, r) => {
|
|
52630
53083
|
const placeholders = cols.map((c, ci) => `$${r * cols.length + ci + 1}`);
|
|
52631
|
-
cols.forEach(c => params.push(coerce(row[c])));
|
|
53084
|
+
cols.forEach(c => params.push(coerce(row[c.source])));
|
|
52632
53085
|
return `(${placeholders.join(", ")})`;
|
|
52633
53086
|
});
|
|
52634
53087
|
await pg.execute(`insert into ${qualified} (${colIdents}) values ${tuples.join(", ")}`, {
|
|
@@ -53560,7 +54013,7 @@ const flags = {
|
|
|
53560
54013
|
};
|
|
53561
54014
|
const isSuccessfulAction = (actionResult) => actionResult.status === sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL ||
|
|
53562
54015
|
actionResult.status === sqlanvil.ActionResult.ExecutionStatus.DISABLED;
|
|
53563
|
-
function run(dbadapter, graph, executionOptions, partiallyExecutedRunResult = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
|
|
54016
|
+
function run(dbadapter, graph, executionOptions = {}, partiallyExecutedRunResult = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
|
|
53564
54017
|
return new Runner(dbadapter, graph, executionOptions, partiallyExecutedRunResult, runnerNotificationPeriodMillis).execute();
|
|
53565
54018
|
}
|
|
53566
54019
|
class Runner {
|
|
@@ -53568,17 +54021,18 @@ class Runner {
|
|
|
53568
54021
|
var _a;
|
|
53569
54022
|
this.dbadapter = dbadapter;
|
|
53570
54023
|
this.graph = graph;
|
|
53571
|
-
this.executionOptions = executionOptions;
|
|
53572
54024
|
this.runnerNotificationPeriodMillis = runnerNotificationPeriodMillis;
|
|
53573
54025
|
this.changeListeners = [];
|
|
53574
54026
|
this.lastNotificationTimestampMillis = 0;
|
|
53575
54027
|
this.stopped = false;
|
|
53576
54028
|
this.cancelled = false;
|
|
53577
54029
|
this.timedOut = false;
|
|
54030
|
+
this.skipReason = "";
|
|
53578
54031
|
this.runningExtracts = 0;
|
|
53579
54032
|
this.extractWaiters = [];
|
|
53580
|
-
this.
|
|
54033
|
+
this.executionOptions = Object.assign({ projectDir: "." }, executionOptions);
|
|
53581
54034
|
this.runResult = Object.assign({ actions: [] }, partiallyExecutedRunResult);
|
|
54035
|
+
this.allActionTargets = new Set(graph.actions.map(action => targetStringifier.stringify(action.target)));
|
|
53582
54036
|
this.warehouseStateByTarget = new Map();
|
|
53583
54037
|
(_a = graph.warehouseState.tables) === null || _a === void 0 ? void 0 : _a.forEach(tableMetadata => this.warehouseStateByTarget.set(targetStringifier.stringify(tableMetadata.target), tableMetadata));
|
|
53584
54038
|
this.executedActionTargets = new Set(this.runResult.actions
|
|
@@ -53591,6 +54045,12 @@ class Runner {
|
|
|
53591
54045
|
this.eEmitter = new EventEmitter__default["default"]();
|
|
53592
54046
|
this.eEmitter.setMaxListeners(0);
|
|
53593
54047
|
}
|
|
54048
|
+
static create(dbadapter, graph, options = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
|
|
54049
|
+
return new Runner(dbadapter, graph, options, {}, runnerNotificationPeriodMillis);
|
|
54050
|
+
}
|
|
54051
|
+
static resume(dbadapter, graph, runResult, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
|
|
54052
|
+
return new Runner(dbadapter, graph, {}, runResult, runnerNotificationPeriodMillis);
|
|
54053
|
+
}
|
|
53594
54054
|
async acquireExtractSlot() {
|
|
53595
54055
|
if (this.runningExtracts < Runner.MAX_CONCURRENT_EXTRACTS) {
|
|
53596
54056
|
this.runningExtracts++;
|
|
@@ -53623,7 +54083,7 @@ class Runner {
|
|
|
53623
54083
|
const timeoutMillis = this.graph.runConfig.timeoutMillis - elapsedTimeMillis;
|
|
53624
54084
|
this.timeout = setTimeout(() => {
|
|
53625
54085
|
this.timedOut = true;
|
|
53626
|
-
this.cancel();
|
|
54086
|
+
this.cancel("timeout");
|
|
53627
54087
|
}, timeoutMillis);
|
|
53628
54088
|
}
|
|
53629
54089
|
return this;
|
|
@@ -53631,7 +54091,22 @@ class Runner {
|
|
|
53631
54091
|
stop() {
|
|
53632
54092
|
this.stopped = true;
|
|
53633
54093
|
}
|
|
53634
|
-
cancel() {
|
|
54094
|
+
cancel(reason = "user") {
|
|
54095
|
+
var _a;
|
|
54096
|
+
if (!this.skipReason) {
|
|
54097
|
+
switch (reason) {
|
|
54098
|
+
case "timeout":
|
|
54099
|
+
this.skipReason = ((_a = this.graph.runConfig) === null || _a === void 0 ? void 0 : _a.timeoutMillis) ? `Run timed out after ${this.graph.runConfig.timeoutMillis / 1000} seconds`
|
|
54100
|
+
: "Run timed out";
|
|
54101
|
+
break;
|
|
54102
|
+
case "user":
|
|
54103
|
+
this.skipReason = "Run cancelled";
|
|
54104
|
+
break;
|
|
54105
|
+
case "cancellation":
|
|
54106
|
+
this.skipReason = "Run cancelled before action started";
|
|
54107
|
+
break;
|
|
54108
|
+
}
|
|
54109
|
+
}
|
|
53635
54110
|
this.cancelled = true;
|
|
53636
54111
|
this.eEmitter.emit(CANCEL_EVENT, undefined, undefined);
|
|
53637
54112
|
}
|
|
@@ -53703,13 +54178,26 @@ class Runner {
|
|
|
53703
54178
|
if (this.cancelled) {
|
|
53704
54179
|
const allPendingActions = this.pendingActions;
|
|
53705
54180
|
this.pendingActions = [];
|
|
53706
|
-
|
|
53707
|
-
|
|
53708
|
-
|
|
53709
|
-
tasks
|
|
53710
|
-
status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED
|
|
53711
|
-
}
|
|
53712
|
-
|
|
54181
|
+
const skipErrorMessage = this.skipReason || "Run cancelled before action started";
|
|
54182
|
+
allPendingActions.forEach(pendingAction => {
|
|
54183
|
+
let tasks;
|
|
54184
|
+
if (pendingAction.tasks && pendingAction.tasks.length > 0) {
|
|
54185
|
+
tasks = pendingAction.tasks.map((_, i) => (Object.assign({ status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED }, (i === 0 ? { errorMessage: skipErrorMessage } : {}))));
|
|
54186
|
+
}
|
|
54187
|
+
else {
|
|
54188
|
+
tasks = [
|
|
54189
|
+
{
|
|
54190
|
+
status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED,
|
|
54191
|
+
errorMessage: skipErrorMessage
|
|
54192
|
+
}
|
|
54193
|
+
];
|
|
54194
|
+
}
|
|
54195
|
+
this.runResult.actions.push({
|
|
54196
|
+
target: pendingAction.target,
|
|
54197
|
+
status: sqlanvil.ActionResult.ExecutionStatus.SKIPPED,
|
|
54198
|
+
tasks
|
|
54199
|
+
});
|
|
54200
|
+
});
|
|
53713
54201
|
this.notifyListeners();
|
|
53714
54202
|
return;
|
|
53715
54203
|
}
|
|
@@ -53805,7 +54293,8 @@ class Runner {
|
|
|
53805
54293
|
}
|
|
53806
54294
|
else {
|
|
53807
54295
|
actionResult.tasks.push({
|
|
53808
|
-
status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED
|
|
54296
|
+
status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED,
|
|
54297
|
+
errorMessage: this.skipReason || "Task skipped after cancellation"
|
|
53809
54298
|
});
|
|
53810
54299
|
}
|
|
53811
54300
|
}
|
|
@@ -54493,6 +54982,399 @@ ${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></t
|
|
|
54493
54982
|
`;
|
|
54494
54983
|
}
|
|
54495
54984
|
|
|
54985
|
+
function tokenize(sql) {
|
|
54986
|
+
const tokens = [];
|
|
54987
|
+
let i = 0;
|
|
54988
|
+
const push = (kind, start, end, value) => tokens.push({ kind, start, end, text: sql.slice(start, end), value });
|
|
54989
|
+
while (i < sql.length) {
|
|
54990
|
+
const ch = sql[i];
|
|
54991
|
+
if (/\s/.test(ch)) {
|
|
54992
|
+
const start = i;
|
|
54993
|
+
while (i < sql.length && /\s/.test(sql[i]))
|
|
54994
|
+
i++;
|
|
54995
|
+
push("whitespace", start, i);
|
|
54996
|
+
continue;
|
|
54997
|
+
}
|
|
54998
|
+
if (ch === "$" && sql[i + 1] === "{") {
|
|
54999
|
+
const start = i;
|
|
55000
|
+
let depth = 0;
|
|
55001
|
+
for (; i < sql.length; i++) {
|
|
55002
|
+
if (sql[i] === "{")
|
|
55003
|
+
depth++;
|
|
55004
|
+
else if (sql[i] === "}" && --depth === 0) {
|
|
55005
|
+
i++;
|
|
55006
|
+
break;
|
|
55007
|
+
}
|
|
55008
|
+
}
|
|
55009
|
+
push("template", start, i);
|
|
55010
|
+
continue;
|
|
55011
|
+
}
|
|
55012
|
+
if (ch === "-" && sql[i + 1] === "-") {
|
|
55013
|
+
const start = i;
|
|
55014
|
+
while (i < sql.length && sql[i] !== "\n")
|
|
55015
|
+
i++;
|
|
55016
|
+
push("comment", start, i);
|
|
55017
|
+
continue;
|
|
55018
|
+
}
|
|
55019
|
+
if (ch === "#") {
|
|
55020
|
+
const start = i;
|
|
55021
|
+
while (i < sql.length && sql[i] !== "\n")
|
|
55022
|
+
i++;
|
|
55023
|
+
push("comment", start, i);
|
|
55024
|
+
continue;
|
|
55025
|
+
}
|
|
55026
|
+
if (ch === "/" && sql[i + 1] === "*") {
|
|
55027
|
+
const start = i;
|
|
55028
|
+
i += 2;
|
|
55029
|
+
while (i < sql.length && !(sql[i] === "*" && sql[i + 1] === "/"))
|
|
55030
|
+
i++;
|
|
55031
|
+
i = Math.min(sql.length, i + 2);
|
|
55032
|
+
push("comment", start, i);
|
|
55033
|
+
continue;
|
|
55034
|
+
}
|
|
55035
|
+
const rawPrefix = (ch === "r" || ch === "R") && (sql[i + 1] === "'" || sql[i + 1] === '"');
|
|
55036
|
+
if (ch === "'" || ch === '"' || rawPrefix) {
|
|
55037
|
+
const start = i;
|
|
55038
|
+
if (rawPrefix)
|
|
55039
|
+
i++;
|
|
55040
|
+
const quote = sql[i];
|
|
55041
|
+
i++;
|
|
55042
|
+
for (; i < sql.length; i++) {
|
|
55043
|
+
if (sql[i] !== quote)
|
|
55044
|
+
continue;
|
|
55045
|
+
if (quote === "'" && sql[i + 1] === "'") {
|
|
55046
|
+
i++;
|
|
55047
|
+
continue;
|
|
55048
|
+
}
|
|
55049
|
+
i++;
|
|
55050
|
+
break;
|
|
55051
|
+
}
|
|
55052
|
+
const raw = sql.slice(start, i);
|
|
55053
|
+
const inner = raw.slice(rawPrefix ? 2 : 1, raw.length - (raw.endsWith(quote) ? 1 : 0));
|
|
55054
|
+
push(quote === '"' ? "quoted-ident" : "string", start, i, inner);
|
|
55055
|
+
continue;
|
|
55056
|
+
}
|
|
55057
|
+
if (ch === "`") {
|
|
55058
|
+
const start = i;
|
|
55059
|
+
i++;
|
|
55060
|
+
while (i < sql.length && sql[i] !== "`")
|
|
55061
|
+
i++;
|
|
55062
|
+
i = Math.min(sql.length, i + 1);
|
|
55063
|
+
push("backtick-ident", start, i, sql.slice(start + 1, i - 1));
|
|
55064
|
+
continue;
|
|
55065
|
+
}
|
|
55066
|
+
if (/[0-9]/.test(ch)) {
|
|
55067
|
+
const start = i;
|
|
55068
|
+
while (i < sql.length && /[0-9._eE]/.test(sql[i]))
|
|
55069
|
+
i++;
|
|
55070
|
+
push("number", start, i);
|
|
55071
|
+
continue;
|
|
55072
|
+
}
|
|
55073
|
+
if (/[A-Za-z_]/.test(ch)) {
|
|
55074
|
+
const start = i;
|
|
55075
|
+
while (i < sql.length && /[A-Za-z0-9_$]/.test(sql[i]))
|
|
55076
|
+
i++;
|
|
55077
|
+
push("word", start, i);
|
|
55078
|
+
continue;
|
|
55079
|
+
}
|
|
55080
|
+
push("punct", i, i + 1);
|
|
55081
|
+
i++;
|
|
55082
|
+
}
|
|
55083
|
+
return tokens;
|
|
55084
|
+
}
|
|
55085
|
+
function significant(tokens) {
|
|
55086
|
+
return tokens.filter(t => t.kind !== "whitespace" && t.kind !== "comment");
|
|
55087
|
+
}
|
|
55088
|
+
function isWord(t, ...words) {
|
|
55089
|
+
return (!!t && t.kind === "word" && words.some(w => t.text.toLowerCase() === w.toLowerCase()));
|
|
55090
|
+
}
|
|
55091
|
+
function matchBracket(toks, open) {
|
|
55092
|
+
var _a;
|
|
55093
|
+
const openCh = (_a = toks[open]) === null || _a === void 0 ? void 0 : _a.text;
|
|
55094
|
+
const closeCh = openCh === "(" ? ")" : openCh === "[" ? "]" : null;
|
|
55095
|
+
if (!closeCh)
|
|
55096
|
+
return -1;
|
|
55097
|
+
let depth = 0;
|
|
55098
|
+
for (let i = open; i < toks.length; i++) {
|
|
55099
|
+
if (toks[i].text === openCh)
|
|
55100
|
+
depth++;
|
|
55101
|
+
else if (toks[i].text === closeCh && --depth === 0)
|
|
55102
|
+
return i;
|
|
55103
|
+
}
|
|
55104
|
+
return -1;
|
|
55105
|
+
}
|
|
55106
|
+
const CLAUSE_ENDS = ["where", "group", "order", "having", "window", "limit", "qualify", "union", "intersect", "except"];
|
|
55107
|
+
function selectScopes(toks) {
|
|
55108
|
+
const scopes = [];
|
|
55109
|
+
for (let i = 0; i < toks.length; i++) {
|
|
55110
|
+
if (!isWord(toks[i], "select"))
|
|
55111
|
+
continue;
|
|
55112
|
+
let depth = 0;
|
|
55113
|
+
let from = -1;
|
|
55114
|
+
let listEnd = -1;
|
|
55115
|
+
let end = toks.length;
|
|
55116
|
+
for (let j = i + 1; j < toks.length; j++) {
|
|
55117
|
+
const t = toks[j];
|
|
55118
|
+
if (t.text === "(" || t.text === "[")
|
|
55119
|
+
depth++;
|
|
55120
|
+
else if (t.text === ")" || t.text === "]") {
|
|
55121
|
+
if (depth === 0) {
|
|
55122
|
+
end = j;
|
|
55123
|
+
break;
|
|
55124
|
+
}
|
|
55125
|
+
depth--;
|
|
55126
|
+
}
|
|
55127
|
+
else if (depth === 0 && t.kind === "word") {
|
|
55128
|
+
const w = t.text.toLowerCase();
|
|
55129
|
+
if (w === "from" && from === -1) {
|
|
55130
|
+
from = j;
|
|
55131
|
+
listEnd = j;
|
|
55132
|
+
}
|
|
55133
|
+
else if (CLAUSE_ENDS.includes(w)) {
|
|
55134
|
+
if (w === "except" && from === -1)
|
|
55135
|
+
continue;
|
|
55136
|
+
end = j;
|
|
55137
|
+
break;
|
|
55138
|
+
}
|
|
55139
|
+
}
|
|
55140
|
+
else if (depth === 0 && t.text === ";") {
|
|
55141
|
+
end = j;
|
|
55142
|
+
break;
|
|
55143
|
+
}
|
|
55144
|
+
}
|
|
55145
|
+
if (listEnd === -1)
|
|
55146
|
+
listEnd = end;
|
|
55147
|
+
scopes.push({
|
|
55148
|
+
select: i,
|
|
55149
|
+
listStart: i + 1,
|
|
55150
|
+
listEnd,
|
|
55151
|
+
from,
|
|
55152
|
+
fromStart: from === -1 ? -1 : from + 1,
|
|
55153
|
+
fromEnd: from === -1 ? -1 : end,
|
|
55154
|
+
});
|
|
55155
|
+
}
|
|
55156
|
+
return scopes;
|
|
55157
|
+
}
|
|
55158
|
+
function splitOnCommas(toks, start, end) {
|
|
55159
|
+
const out = [];
|
|
55160
|
+
let depth = 0;
|
|
55161
|
+
const stop = Math.min(end, toks.length);
|
|
55162
|
+
let from = Math.max(0, start);
|
|
55163
|
+
for (let i = from; i < stop; i++) {
|
|
55164
|
+
const t = toks[i].text;
|
|
55165
|
+
if (t === "(" || t === "[")
|
|
55166
|
+
depth++;
|
|
55167
|
+
else if (t === ")" || t === "]")
|
|
55168
|
+
depth--;
|
|
55169
|
+
else if (t === "," && depth === 0) {
|
|
55170
|
+
out.push([from, i]);
|
|
55171
|
+
from = i + 1;
|
|
55172
|
+
}
|
|
55173
|
+
}
|
|
55174
|
+
if (from < stop)
|
|
55175
|
+
out.push([from, stop]);
|
|
55176
|
+
return out;
|
|
55177
|
+
}
|
|
55178
|
+
const JOIN_NOISE = new Set([
|
|
55179
|
+
"on", "using", "left", "right", "inner", "full", "cross", "outer", "join", "lateral", "natural",
|
|
55180
|
+
]);
|
|
55181
|
+
function relationsIn(toks, start, end) {
|
|
55182
|
+
var _a, _b, _c;
|
|
55183
|
+
const rels = [];
|
|
55184
|
+
let i = start;
|
|
55185
|
+
while (i < end) {
|
|
55186
|
+
if (i > start && !isWord(toks[i - 1], "from", "join")) {
|
|
55187
|
+
i++;
|
|
55188
|
+
continue;
|
|
55189
|
+
}
|
|
55190
|
+
let j = i;
|
|
55191
|
+
let subquery = false;
|
|
55192
|
+
const parts = [];
|
|
55193
|
+
if (((_a = toks[j]) === null || _a === void 0 ? void 0 : _a.text) === "(") {
|
|
55194
|
+
const close = matchBracket(toks, j);
|
|
55195
|
+
if (close < 0)
|
|
55196
|
+
break;
|
|
55197
|
+
subquery = true;
|
|
55198
|
+
parts.push(...toks.slice(j, close + 1));
|
|
55199
|
+
j = close + 1;
|
|
55200
|
+
}
|
|
55201
|
+
else {
|
|
55202
|
+
while (j < end &&
|
|
55203
|
+
(toks[j].kind === "template" ||
|
|
55204
|
+
toks[j].kind === "word" ||
|
|
55205
|
+
toks[j].kind === "quoted-ident" ||
|
|
55206
|
+
toks[j].kind === "backtick-ident" ||
|
|
55207
|
+
toks[j].text === ".")) {
|
|
55208
|
+
if (isWord(toks[j], "as"))
|
|
55209
|
+
break;
|
|
55210
|
+
if (toks[j].kind === "word" && JOIN_NOISE.has(toks[j].text.toLowerCase()))
|
|
55211
|
+
break;
|
|
55212
|
+
parts.push(toks[j]);
|
|
55213
|
+
j++;
|
|
55214
|
+
}
|
|
55215
|
+
}
|
|
55216
|
+
if (!parts.length) {
|
|
55217
|
+
i++;
|
|
55218
|
+
continue;
|
|
55219
|
+
}
|
|
55220
|
+
let alias;
|
|
55221
|
+
let k = j;
|
|
55222
|
+
if (isWord(toks[k], "as"))
|
|
55223
|
+
k++;
|
|
55224
|
+
if (((_b = toks[k]) === null || _b === void 0 ? void 0 : _b.kind) === "word" && !JOIN_NOISE.has(toks[k].text.toLowerCase())) {
|
|
55225
|
+
alias = toks[k].text.toLowerCase();
|
|
55226
|
+
k++;
|
|
55227
|
+
}
|
|
55228
|
+
else {
|
|
55229
|
+
const last = [...parts].reverse().find(p => p.kind === "word" || p.kind === "quoted-ident");
|
|
55230
|
+
if (last && !subquery)
|
|
55231
|
+
alias = ((_c = last.value) !== null && _c !== void 0 ? _c : last.text).toLowerCase();
|
|
55232
|
+
}
|
|
55233
|
+
rels.push({ tokens: parts, alias, subquery });
|
|
55234
|
+
i = Math.max(k, j);
|
|
55235
|
+
}
|
|
55236
|
+
return rels;
|
|
55237
|
+
}
|
|
55238
|
+
|
|
55239
|
+
function findStructArrays(file, source) {
|
|
55240
|
+
var _a, _b, _c, _d, _e, _f;
|
|
55241
|
+
const toks = significant(tokenize(source));
|
|
55242
|
+
const found = [];
|
|
55243
|
+
for (let i = 0; i < toks.length; i++) {
|
|
55244
|
+
if (!isWord(toks[i], "array") || ((_a = toks[i + 1]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
55245
|
+
continue;
|
|
55246
|
+
const close = matchBracket(toks, i + 1);
|
|
55247
|
+
if (close < 0)
|
|
55248
|
+
continue;
|
|
55249
|
+
if (!isWord(toks[i + 2], "select") || !isWord(toks[i + 3], "as") || !isWord(toks[i + 4], "struct")) {
|
|
55250
|
+
continue;
|
|
55251
|
+
}
|
|
55252
|
+
const inner = toks.slice(i + 5, close);
|
|
55253
|
+
const innerSql = inner.map(t => t.text).join(" ");
|
|
55254
|
+
const innerToks = significant(tokenize(`select ${innerSql}`));
|
|
55255
|
+
const [scope] = selectScopes(innerToks);
|
|
55256
|
+
const fields = [];
|
|
55257
|
+
if (scope) {
|
|
55258
|
+
for (const [s, e] of splitOnCommas(innerToks, scope.listStart, scope.listEnd)) {
|
|
55259
|
+
const item = innerToks.slice(s, e);
|
|
55260
|
+
const last = item[item.length - 1];
|
|
55261
|
+
fields.push(((_c = (_b = last === null || last === void 0 ? void 0 : last.value) !== null && _b !== void 0 ? _b : last === null || last === void 0 ? void 0 : last.text) !== null && _c !== void 0 ? _c : "").replace(/^"|"$/g, ""));
|
|
55262
|
+
}
|
|
55263
|
+
}
|
|
55264
|
+
const rels = scope && scope.from !== -1 ? relationsIn(innerToks, scope.fromStart, scope.fromEnd) : [];
|
|
55265
|
+
const source_ = (_e = (_d = rels[0]) === null || _d === void 0 ? void 0 : _d.tokens.map(t => t.text).join("")) !== null && _e !== void 0 ? _e : "";
|
|
55266
|
+
let correlation;
|
|
55267
|
+
const whereIdx = innerToks.findIndex(t => isWord(t, "where"));
|
|
55268
|
+
if (whereIdx !== -1) {
|
|
55269
|
+
const eq = innerToks.findIndex((t, k) => k > whereIdx && t.text === "=");
|
|
55270
|
+
const left = innerToks.slice(Math.max(whereIdx + 1, eq - 3), eq);
|
|
55271
|
+
const right = innerToks.slice(eq + 1, eq + 4);
|
|
55272
|
+
const tail = (parts) => {
|
|
55273
|
+
var _a;
|
|
55274
|
+
const words = parts.filter(t => t.kind === "word" || t.kind === "quoted-ident");
|
|
55275
|
+
return words.length ? ((_a = words[words.length - 1].value) !== null && _a !== void 0 ? _a : words[words.length - 1].text) : "";
|
|
55276
|
+
};
|
|
55277
|
+
if (eq !== -1)
|
|
55278
|
+
correlation = { sourceColumn: tail(left), parentColumn: tail(right) };
|
|
55279
|
+
}
|
|
55280
|
+
let end = toks[close].end;
|
|
55281
|
+
let column = "";
|
|
55282
|
+
if (isWord(toks[close + 1], "as") && toks[close + 2]) {
|
|
55283
|
+
column = ((_f = toks[close + 2].value) !== null && _f !== void 0 ? _f : toks[close + 2].text).replace(/^"|"$/g, "");
|
|
55284
|
+
end = toks[close + 2].end;
|
|
55285
|
+
}
|
|
55286
|
+
found.push({
|
|
55287
|
+
file,
|
|
55288
|
+
line: source.slice(0, toks[i].start).split("\n").length,
|
|
55289
|
+
column,
|
|
55290
|
+
fields,
|
|
55291
|
+
source: source_,
|
|
55292
|
+
correlation,
|
|
55293
|
+
start: toks[i].start,
|
|
55294
|
+
end,
|
|
55295
|
+
});
|
|
55296
|
+
}
|
|
55297
|
+
return found;
|
|
55298
|
+
}
|
|
55299
|
+
function suggestedSql(site) {
|
|
55300
|
+
var _a, _b, _c, _d, _e, _f;
|
|
55301
|
+
const fields = site.fields.length ? site.fields : ["…"];
|
|
55302
|
+
const key = (_b = (_a = site.correlation) === null || _a === void 0 ? void 0 : _a.sourceColumn) !== null && _b !== void 0 ? _b : "<parent key>";
|
|
55303
|
+
const parentKey = (_d = (_c = site.correlation) === null || _c === void 0 ? void 0 : _c.parentColumn) !== null && _d !== void 0 ? _d : "<parent key>";
|
|
55304
|
+
const src = site.source || "<source relation>";
|
|
55305
|
+
if (site.strategy === "collapse" && site.consumers.length) {
|
|
55306
|
+
const [k, v] = [(_e = fields[0]) !== null && _e !== void 0 ? _e : "key", (_f = fields[1]) !== null && _f !== void 0 ? _f : "value"];
|
|
55307
|
+
return [
|
|
55308
|
+
`-- Drop the \`${site.column}\` column from this action entirely, and have each reader`,
|
|
55309
|
+
`-- pivot ${src} directly. A consumer that today reads:`,
|
|
55310
|
+
`--`,
|
|
55311
|
+
`-- (select any_value(${v}) from unnest(${site.column}) where ${k} = '<name>')`,
|
|
55312
|
+
`--`,
|
|
55313
|
+
`-- becomes, selecting from ${src} grouped by ${parentKey}:`,
|
|
55314
|
+
``,
|
|
55315
|
+
`max(${v}) filter (where ${k} = '<name>') as <name>`,
|
|
55316
|
+
].join("\n");
|
|
55317
|
+
}
|
|
55318
|
+
if (site.strategy === "collapse") {
|
|
55319
|
+
return [
|
|
55320
|
+
`-- Nothing reads \`${site.column}\`. Confirm that, then delete the column — porting an`,
|
|
55321
|
+
`-- unused array costs a model change for no consumer.`,
|
|
55322
|
+
].join("\n");
|
|
55323
|
+
}
|
|
55324
|
+
if (site.strategy === "child-table") {
|
|
55325
|
+
const cols = fields.map(f => ` ${f}`).join(",\n");
|
|
55326
|
+
return [
|
|
55327
|
+
`-- A new action holding the rows, keyed by the parent — this IS what the array was:`,
|
|
55328
|
+
``,
|
|
55329
|
+
`config { type: "view" }`,
|
|
55330
|
+
``,
|
|
55331
|
+
`select`,
|
|
55332
|
+
` ${key} as ${parentKey},`,
|
|
55333
|
+
cols,
|
|
55334
|
+
`from \${ref("${src.replace(/^\$\{ref\(["']|["']\)\}$/g, "")}")}`,
|
|
55335
|
+
``,
|
|
55336
|
+
`-- Then drop \`${site.column}\` from the parent and have readers join instead:`,
|
|
55337
|
+
`-- join \${ref("<this new action>")} c on c.${parentKey} = p.${parentKey}`,
|
|
55338
|
+
].join("\n");
|
|
55339
|
+
}
|
|
55340
|
+
const pairs = fields.map(f => `'${f}', ${f}`).join(", ");
|
|
55341
|
+
return [
|
|
55342
|
+
`-- Fallback. Keeps the shape but pushes a document model into a relational warehouse:`,
|
|
55343
|
+
``,
|
|
55344
|
+
`(select jsonb_agg(jsonb_build_object(${pairs})) from ${src} …) as ${site.column}`,
|
|
55345
|
+
``,
|
|
55346
|
+
`-- Readers then need jsonb_array_elements(...) and ->> instead of UNNEST.`,
|
|
55347
|
+
].join("\n");
|
|
55348
|
+
}
|
|
55349
|
+
function chooseStrategies(sites, files) {
|
|
55350
|
+
return sites.map(site => {
|
|
55351
|
+
const consumers = [];
|
|
55352
|
+
for (const f of files) {
|
|
55353
|
+
if (f.file === site.file || !site.column)
|
|
55354
|
+
continue;
|
|
55355
|
+
const toks = significant(tokenize(f.source));
|
|
55356
|
+
const uses = toks
|
|
55357
|
+
.map((t, i) => ({ t, i }))
|
|
55358
|
+
.filter(({ t }) => t.kind === "word" && t.text.toLowerCase() === site.column.toLowerCase());
|
|
55359
|
+
if (!uses.length)
|
|
55360
|
+
continue;
|
|
55361
|
+
const unnestOnly = uses.every(({ i }) => { var _a; return isWord(toks[i - 2], "unnest") && ((_a = toks[i - 1]) === null || _a === void 0 ? void 0 : _a.text) === "("; });
|
|
55362
|
+
consumers.push({ file: f.file, unnestOnly });
|
|
55363
|
+
}
|
|
55364
|
+
const readers = consumers.length;
|
|
55365
|
+
if (readers && consumers.every(c => c.unnestOnly)) {
|
|
55366
|
+
return Object.assign(Object.assign({}, site), { consumers, strategy: "collapse", rationale: `every read of \`${site.column}\` is an UNNEST (${readers} file(s)), so the array is a ` +
|
|
55367
|
+
`round trip — read ${site.source || "the source"} directly and drop the array` });
|
|
55368
|
+
}
|
|
55369
|
+
if (!readers) {
|
|
55370
|
+
return Object.assign(Object.assign({}, site), { consumers, strategy: "collapse", rationale: `nothing in the project reads \`${site.column}\` — confirm it is unused, then drop it ` +
|
|
55371
|
+
`rather than porting it` });
|
|
55372
|
+
}
|
|
55373
|
+
return Object.assign(Object.assign({}, site), { consumers, strategy: "child-table", rationale: `\`${site.column}\` is read in ${readers} file(s) and not only via UNNEST, so it needs to ` +
|
|
55374
|
+
`exist as a relation: a child table keyed by the parent, joined back` });
|
|
55375
|
+
});
|
|
55376
|
+
}
|
|
55377
|
+
|
|
54496
55378
|
function containsPath(parent, child) {
|
|
54497
55379
|
const rel = path__namespace.relative(parent, child);
|
|
54498
55380
|
return rel === "" || (!rel.startsWith("..") && !path__namespace.isAbsolute(rel));
|
|
@@ -54623,6 +55505,192 @@ function parseSqlxConfig(body) {
|
|
|
54623
55505
|
description: str("description", body),
|
|
54624
55506
|
};
|
|
54625
55507
|
}
|
|
55508
|
+
const CLASS_META = {
|
|
55509
|
+
qualify: {
|
|
55510
|
+
title: "QUALIFY clause",
|
|
55511
|
+
severity: "blocks-compile",
|
|
55512
|
+
owner: "mechanical",
|
|
55513
|
+
postgres: "wrap the query and filter on the window function in an outer WHERE",
|
|
55514
|
+
why: "PostgreSQL has no QUALIFY; the window result must be computed in a subquery first.",
|
|
55515
|
+
},
|
|
55516
|
+
struct: {
|
|
55517
|
+
title: "STRUCT / ARRAY of STRUCT",
|
|
55518
|
+
severity: "blocks-compile",
|
|
55519
|
+
owner: "needs-decision",
|
|
55520
|
+
postgres: "a child table joined back, jsonb, or eliminate the array entirely",
|
|
55521
|
+
why: "An array of structs IS a one-to-many relationship. A child table is usually the better " +
|
|
55522
|
+
"model, and where the array is only ever UNNESTed straight back into a pivot it can be " +
|
|
55523
|
+
"dropped altogether — but which of the three is right depends on what reads it.",
|
|
55524
|
+
},
|
|
55525
|
+
unnest: {
|
|
55526
|
+
title: "UNNEST",
|
|
55527
|
+
severity: "blocks-compile",
|
|
55528
|
+
owner: "needs-decision",
|
|
55529
|
+
postgres: "cross join unnest(array) — but only for arrays of scalars",
|
|
55530
|
+
why: "PostgreSQL's unnest takes arrays only, so an UNNEST over structs depends on how the STRUCT above is resolved.",
|
|
55531
|
+
},
|
|
55532
|
+
"array-type": {
|
|
55533
|
+
title: "ARRAY<T> type syntax",
|
|
55534
|
+
severity: "blocks-compile",
|
|
55535
|
+
owner: "mechanical",
|
|
55536
|
+
postgres: "T[]",
|
|
55537
|
+
},
|
|
55538
|
+
"parse-date": {
|
|
55539
|
+
title: "PARSE_DATE / PARSE_TIMESTAMP",
|
|
55540
|
+
severity: "fails-at-run",
|
|
55541
|
+
owner: "mechanical",
|
|
55542
|
+
postgres: "to_date / to_timestamp, with the format tokens translated",
|
|
55543
|
+
why: "The format tokens differ (%B %Y → 'Month YYYY'). A wrong token does not fail the " +
|
|
55544
|
+
"statement — it fails on the rows that do not match, so verify against real data.",
|
|
55545
|
+
},
|
|
55546
|
+
"format-date": {
|
|
55547
|
+
title: "FORMAT_DATE / FORMAT_TIMESTAMP",
|
|
55548
|
+
severity: "fails-at-run",
|
|
55549
|
+
owner: "mechanical",
|
|
55550
|
+
postgres: "to_char, with the format tokens translated",
|
|
55551
|
+
},
|
|
55552
|
+
"date-diff-unit": {
|
|
55553
|
+
title: "DATE_DIFF with an unhandled unit",
|
|
55554
|
+
severity: "blocks-compile",
|
|
55555
|
+
owner: "mechanical",
|
|
55556
|
+
postgres: "date subtraction, or EXTRACT(EPOCH FROM …) scaled to the unit",
|
|
55557
|
+
why: "DAY and HOUR are converted automatically; other units need the right arithmetic per unit.",
|
|
55558
|
+
},
|
|
55559
|
+
"pseudo-column": {
|
|
55560
|
+
title: "BigQuery pseudo-column",
|
|
55561
|
+
severity: "blocks-compile",
|
|
55562
|
+
owner: "needs-decision",
|
|
55563
|
+
why: "_TABLE_SUFFIX / _PARTITIONTIME have no PostgreSQL analogue; the model usually needs restructuring.",
|
|
55564
|
+
},
|
|
55565
|
+
"export-data": {
|
|
55566
|
+
title: "EXPORT DATA statement",
|
|
55567
|
+
severity: "blocks-compile",
|
|
55568
|
+
owner: "mechanical",
|
|
55569
|
+
postgres: 'a sqlanvil `type: "export"` action',
|
|
55570
|
+
},
|
|
55571
|
+
scripting: {
|
|
55572
|
+
title: "BigQuery scripting (EXECUTE IMMEDIATE)",
|
|
55573
|
+
severity: "blocks-compile",
|
|
55574
|
+
owner: "needs-decision",
|
|
55575
|
+
postgres: "a PL/pgSQL DO block, or an operations action",
|
|
55576
|
+
why: "Dynamic SQL rarely ports mechanically — the shape of the rewrite depends on what it is generating.",
|
|
55577
|
+
},
|
|
55578
|
+
"generate-uuid": {
|
|
55579
|
+
title: "GENERATE_UUID()",
|
|
55580
|
+
severity: "blocks-compile",
|
|
55581
|
+
owner: "mechanical",
|
|
55582
|
+
postgres: "gen_random_uuid()",
|
|
55583
|
+
},
|
|
55584
|
+
"group-by-all": {
|
|
55585
|
+
title: "GROUP BY ALL",
|
|
55586
|
+
severity: "blocks-compile",
|
|
55587
|
+
owner: "mechanical",
|
|
55588
|
+
postgres: "`sqlanvil migrate-fix` rewrites it as ordinals (group by 1, 2, 5)",
|
|
55589
|
+
why: "Window functions must be excluded as well as aggregates — PostgreSQL rejects them in " +
|
|
55590
|
+
"GROUP BY, and an aggregate followed by OVER is a window function, not an aggregate.",
|
|
55591
|
+
},
|
|
55592
|
+
"star-except": {
|
|
55593
|
+
title: "SELECT * EXCEPT (…)",
|
|
55594
|
+
severity: "blocks-compile",
|
|
55595
|
+
owner: "mechanical",
|
|
55596
|
+
postgres: "`sqlanvil migrate-fix` expands it, once introspect has scaffolded the columns",
|
|
55597
|
+
why: "The column list comes from the source relation — a declaration's columnTypes, or another " +
|
|
55598
|
+
"action's select list resolved recursively. Enumerating also means a column added upstream " +
|
|
55599
|
+
"will no longer appear automatically.",
|
|
55600
|
+
},
|
|
55601
|
+
"not-enforced": {
|
|
55602
|
+
title: "NOT ENFORCED constraint (translated, left commented)",
|
|
55603
|
+
severity: "changes-meaning",
|
|
55604
|
+
owner: "needs-decision",
|
|
55605
|
+
postgres: "the constraint is written out above, commented",
|
|
55606
|
+
why: "BigQuery never checked these. Enforcing them may fail on data that has never been " +
|
|
55607
|
+
"validated; dropping them loses the declared model. Enable per table, deliberately.",
|
|
55608
|
+
},
|
|
55609
|
+
};
|
|
55610
|
+
function analyseStructArrays(report, outDir) {
|
|
55611
|
+
const files = report.files
|
|
55612
|
+
.filter(f => f.file.endsWith(".sqlx"))
|
|
55613
|
+
.map(f => {
|
|
55614
|
+
try {
|
|
55615
|
+
return { file: f.file, source: fs__namespace$1.readFileSync(path__namespace.join(outDir, f.file), "utf8") };
|
|
55616
|
+
}
|
|
55617
|
+
catch (_a) {
|
|
55618
|
+
return null;
|
|
55619
|
+
}
|
|
55620
|
+
})
|
|
55621
|
+
.filter((f) => f !== null);
|
|
55622
|
+
const raw = files.flatMap(f => findStructArrays(f.file, f.source));
|
|
55623
|
+
if (!raw.length)
|
|
55624
|
+
return;
|
|
55625
|
+
const sites = chooseStrategies(raw, files);
|
|
55626
|
+
report.todo = report.todo.filter(t => t.id !== "struct");
|
|
55627
|
+
const meta = CLASS_META.struct;
|
|
55628
|
+
report.todo.push({
|
|
55629
|
+
id: "struct",
|
|
55630
|
+
title: meta.title,
|
|
55631
|
+
count: sites.length,
|
|
55632
|
+
severity: meta.severity,
|
|
55633
|
+
owner: meta.owner,
|
|
55634
|
+
postgres: "per site below — collapse, a child table, or jsonb",
|
|
55635
|
+
why: meta.why,
|
|
55636
|
+
sql: sites.map(s => ({
|
|
55637
|
+
file: s.file,
|
|
55638
|
+
line: s.line,
|
|
55639
|
+
note: `${s.strategy} — ${s.rationale}`,
|
|
55640
|
+
sql: suggestedSql(s),
|
|
55641
|
+
})),
|
|
55642
|
+
locations: sites.map(s => ({ file: s.file, lines: [s.line] })),
|
|
55643
|
+
});
|
|
55644
|
+
}
|
|
55645
|
+
function buildTriage(report) {
|
|
55646
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
55647
|
+
const todo = new Map();
|
|
55648
|
+
const applied = new Map();
|
|
55649
|
+
for (const file of report.files) {
|
|
55650
|
+
for (const f of (_a = file.findings) !== null && _a !== void 0 ? _a : []) {
|
|
55651
|
+
const id = (_b = f.class) !== null && _b !== void 0 ? _b : classIdFor(f.note);
|
|
55652
|
+
const meta = CLASS_META[id];
|
|
55653
|
+
if (f.kind === "rewrite") {
|
|
55654
|
+
const seen = (_c = applied.get(id)) !== null && _c !== void 0 ? _c : { id, title: (_d = meta === null || meta === void 0 ? void 0 : meta.title) !== null && _d !== void 0 ? _d : f.note, count: 0 };
|
|
55655
|
+
seen.count++;
|
|
55656
|
+
applied.set(id, seen);
|
|
55657
|
+
if (!meta || meta.owner !== "needs-decision")
|
|
55658
|
+
continue;
|
|
55659
|
+
}
|
|
55660
|
+
const entry = (_e = todo.get(id)) !== null && _e !== void 0 ? _e : {
|
|
55661
|
+
id,
|
|
55662
|
+
title: (_f = meta === null || meta === void 0 ? void 0 : meta.title) !== null && _f !== void 0 ? _f : f.note,
|
|
55663
|
+
count: 0,
|
|
55664
|
+
severity: (_g = meta === null || meta === void 0 ? void 0 : meta.severity) !== null && _g !== void 0 ? _g : "blocks-compile",
|
|
55665
|
+
owner: (_h = meta === null || meta === void 0 ? void 0 : meta.owner) !== null && _h !== void 0 ? _h : "mechanical",
|
|
55666
|
+
postgres: meta === null || meta === void 0 ? void 0 : meta.postgres,
|
|
55667
|
+
why: meta === null || meta === void 0 ? void 0 : meta.why,
|
|
55668
|
+
locations: [],
|
|
55669
|
+
};
|
|
55670
|
+
entry.count++;
|
|
55671
|
+
const loc = entry.locations.find(l => l.file === file.file);
|
|
55672
|
+
if (loc)
|
|
55673
|
+
loc.lines.push(f.line);
|
|
55674
|
+
else
|
|
55675
|
+
entry.locations.push({ file: file.file, lines: [f.line] });
|
|
55676
|
+
todo.set(id, entry);
|
|
55677
|
+
}
|
|
55678
|
+
}
|
|
55679
|
+
const rank = {
|
|
55680
|
+
"changes-meaning": 0,
|
|
55681
|
+
"fails-at-run": 1,
|
|
55682
|
+
"blocks-compile": 2,
|
|
55683
|
+
};
|
|
55684
|
+
report.todo = [...todo.values()].sort((a, b) => rank[a.severity] - rank[b.severity] || b.count - a.count);
|
|
55685
|
+
report.applied = [...applied.values()].sort((a, b) => b.count - a.count);
|
|
55686
|
+
}
|
|
55687
|
+
function classIdFor(note) {
|
|
55688
|
+
return note
|
|
55689
|
+
.slice(0, 40)
|
|
55690
|
+
.toLowerCase()
|
|
55691
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
55692
|
+
.replace(/^-|-$/g, "");
|
|
55693
|
+
}
|
|
54626
55694
|
const GLOBAL_RULES = [
|
|
54627
55695
|
{
|
|
54628
55696
|
pattern: /\bdataform\.projectConfig\b/g,
|
|
@@ -54632,12 +55700,6 @@ const GLOBAL_RULES = [
|
|
|
54632
55700
|
},
|
|
54633
55701
|
];
|
|
54634
55702
|
const SQL_RULES = [
|
|
54635
|
-
{
|
|
54636
|
-
pattern: /\bSAFE_CAST\s*\(/g,
|
|
54637
|
-
replacement: "CAST(",
|
|
54638
|
-
kind: "rewrite",
|
|
54639
|
-
note: "SAFE_CAST → CAST — Postgres CAST raises on bad input; verify the data or guard it",
|
|
54640
|
-
},
|
|
54641
55703
|
{
|
|
54642
55704
|
pattern: /\bCURRENT_DATE\s*\(\s*\)/g,
|
|
54643
55705
|
replacement: "CURRENT_DATE",
|
|
@@ -54650,32 +55712,50 @@ const SQL_RULES = [
|
|
|
54650
55712
|
note: "CURRENT_DATE('<tz>') — Postgres: (CURRENT_TIMESTAMP AT TIME ZONE '<tz>')::date",
|
|
54651
55713
|
},
|
|
54652
55714
|
{
|
|
54653
|
-
pattern:
|
|
55715
|
+
pattern: /\*\s*except\s*\(/gi,
|
|
54654
55716
|
kind: "flag",
|
|
54655
|
-
note: "
|
|
55717
|
+
note: "SELECT * EXCEPT — run `sqlanvil migrate-fix` after introspect to expand it",
|
|
55718
|
+
id: "star-except",
|
|
54656
55719
|
},
|
|
54657
55720
|
{
|
|
54658
|
-
pattern: /\
|
|
55721
|
+
pattern: /\bGROUP\s+BY\s+ALL\b/gi,
|
|
54659
55722
|
kind: "flag",
|
|
54660
|
-
note: "
|
|
55723
|
+
note: "GROUP BY ALL — run `sqlanvil migrate-fix` after introspect to rewrite it",
|
|
55724
|
+
id: "group-by-all",
|
|
54661
55725
|
},
|
|
54662
|
-
{ pattern: /\bQUALIFY\b/gi, kind: "flag", note: "QUALIFY — rewrite as a subquery/CTE filter on the window function"
|
|
54663
|
-
|
|
54664
|
-
{ pattern: /\
|
|
54665
|
-
|
|
54666
|
-
{ pattern: /\
|
|
54667
|
-
|
|
54668
|
-
{ pattern: /\
|
|
55726
|
+
{ pattern: /\bQUALIFY\b/gi, kind: "flag", note: "QUALIFY — rewrite as a subquery/CTE filter on the window function",
|
|
55727
|
+
id: "qualify", },
|
|
55728
|
+
{ pattern: /\bUNNEST\s*\(/g, kind: "flag", note: "UNNEST — Postgres unnest() works on arrays only (no STRUCT arrays); verify semantics",
|
|
55729
|
+
id: "unnest", },
|
|
55730
|
+
{ pattern: /\bSTRUCT\s*[(<]/g, kind: "flag", note: "STRUCT — no Postgres equivalent; consider jsonb or a composite type",
|
|
55731
|
+
id: "struct", },
|
|
55732
|
+
{ pattern: /\bARRAY\s*</g, kind: "flag", note: "ARRAY<T> type syntax — Postgres uses T[]",
|
|
55733
|
+
id: "array-type", },
|
|
55734
|
+
{ pattern: /\bFORMAT_DATE\s*\(/g, kind: "flag", note: "FORMAT_DATE → TO_CHAR (different format tokens)",
|
|
55735
|
+
id: "format-date", },
|
|
55736
|
+
{ pattern: /\bPARSE_DATE\s*\(/g, kind: "flag", note: "PARSE_DATE → TO_DATE (different format tokens)",
|
|
55737
|
+
id: "parse-date", },
|
|
55738
|
+
{ pattern: /\bFORMAT_TIMESTAMP\s*\(/g, kind: "flag", note: "FORMAT_TIMESTAMP → TO_CHAR (different format tokens)",
|
|
55739
|
+
id: "format-date", },
|
|
54669
55740
|
{
|
|
54670
55741
|
pattern: /\b(DATE_ADD|DATE_SUB|TIMESTAMP_ADD|TIMESTAMP_SUB|DATETIME_ADD|DATETIME_SUB)\s*\(/g,
|
|
54671
55742
|
kind: "flag",
|
|
54672
55743
|
note: "BigQuery date arithmetic — Postgres uses interval arithmetic (x + INTERVAL '1 day')",
|
|
54673
55744
|
},
|
|
54674
|
-
{ pattern: /\bDATE_DIFF\s*\(/g, kind: "flag", note: "DATE_DIFF — Postgres: subtraction / EXTRACT(EPOCH FROM ...) depending on the part"
|
|
54675
|
-
|
|
54676
|
-
{ pattern:
|
|
54677
|
-
|
|
54678
|
-
{
|
|
55745
|
+
{ pattern: /\bDATE_DIFF\s*\(/g, kind: "flag", note: "DATE_DIFF — Postgres: subtraction / EXTRACT(EPOCH FROM ...) depending on the part",
|
|
55746
|
+
id: "date-diff-unit", },
|
|
55747
|
+
{ pattern: /_PARTITIONTIME|_PARTITIONDATE|_TABLE_SUFFIX/g, kind: "flag", note: "BigQuery pseudo-column has no Postgres equivalent",
|
|
55748
|
+
id: "pseudo-column", },
|
|
55749
|
+
{
|
|
55750
|
+
pattern: /\bEXPORT\s+DATA\b/gi,
|
|
55751
|
+
kind: "flag",
|
|
55752
|
+
note: 'EXPORT DATA — use a sqlanvil `type: "export"` action instead',
|
|
55753
|
+
id: "export-data",
|
|
55754
|
+
},
|
|
55755
|
+
{ pattern: /\bEXECUTE\s+IMMEDIATE\b/gi, kind: "flag", note: "BigQuery scripting (EXECUTE IMMEDIATE) — rewrite as a PL/pgSQL DO block or operations",
|
|
55756
|
+
id: "scripting", },
|
|
55757
|
+
{ pattern: /\bGENERATE_UUID\s*\(\s*\)/g, kind: "flag", note: "GENERATE_UUID() → gen_random_uuid()",
|
|
55758
|
+
id: "generate-uuid", },
|
|
54679
55759
|
{
|
|
54680
55760
|
pattern: /`[A-Za-z0-9_-]+\.[A-Za-z0-9_$]+\.[A-Za-z0-9_$]+`/g,
|
|
54681
55761
|
kind: "flag",
|
|
@@ -54712,10 +55792,394 @@ ${description} columnTypes: {
|
|
|
54712
55792
|
],
|
|
54713
55793
|
};
|
|
54714
55794
|
}
|
|
55795
|
+
function maskSql(text, keepStrings = false) {
|
|
55796
|
+
let out = text.replace(/--[^\n]*/g, m => " ".repeat(m.length));
|
|
55797
|
+
out = out.replace(/\/\*[\s\S]*?\*\//g, m => " ".repeat(m.length));
|
|
55798
|
+
if (!keepStrings) {
|
|
55799
|
+
out = out.replace(/'(?:[^']|'')*'/g, m => `'${" ".repeat(Math.max(0, m.length - 2))}'`);
|
|
55800
|
+
}
|
|
55801
|
+
return out;
|
|
55802
|
+
}
|
|
55803
|
+
function matchParen(text, open) {
|
|
55804
|
+
let depth = 0;
|
|
55805
|
+
for (let i = open; i < text.length; i++) {
|
|
55806
|
+
if (text[i] === "(")
|
|
55807
|
+
depth++;
|
|
55808
|
+
else if (text[i] === ")" && --depth === 0)
|
|
55809
|
+
return i;
|
|
55810
|
+
}
|
|
55811
|
+
return -1;
|
|
55812
|
+
}
|
|
55813
|
+
function splitArgs(src, mask, from, to) {
|
|
55814
|
+
const parts = [];
|
|
55815
|
+
let depth = 0;
|
|
55816
|
+
let start = from;
|
|
55817
|
+
for (let i = from; i < to; i++) {
|
|
55818
|
+
const ch = mask[i];
|
|
55819
|
+
if (ch === "(")
|
|
55820
|
+
depth++;
|
|
55821
|
+
else if (ch === ")")
|
|
55822
|
+
depth--;
|
|
55823
|
+
else if (ch === "," && depth === 0) {
|
|
55824
|
+
parts.push(src.slice(start, i).trim());
|
|
55825
|
+
start = i + 1;
|
|
55826
|
+
}
|
|
55827
|
+
}
|
|
55828
|
+
parts.push(src.slice(start, to).trim());
|
|
55829
|
+
return parts;
|
|
55830
|
+
}
|
|
55831
|
+
const INTERVAL_ARG = /^interval\s+(\d+)\s+([a-z]+)$/i;
|
|
55832
|
+
const TYPE_NAMES = {
|
|
55833
|
+
string: "text",
|
|
55834
|
+
int64: "bigint",
|
|
55835
|
+
float64: "double precision",
|
|
55836
|
+
numeric: "numeric",
|
|
55837
|
+
bignumeric: "numeric",
|
|
55838
|
+
bool: "boolean",
|
|
55839
|
+
bytes: "bytea",
|
|
55840
|
+
datetime: "timestamp",
|
|
55841
|
+
timestamp: "timestamptz",
|
|
55842
|
+
};
|
|
55843
|
+
function pgType(bqType) {
|
|
55844
|
+
var _a;
|
|
55845
|
+
const t = bqType.trim().toLowerCase();
|
|
55846
|
+
return (_a = TYPE_NAMES[t]) !== null && _a !== void 0 ? _a : t;
|
|
55847
|
+
}
|
|
55848
|
+
const CALL_RULES = [
|
|
55849
|
+
{
|
|
55850
|
+
name: "cast",
|
|
55851
|
+
note: "CAST(x AS <BigQuery type>) → the PostgreSQL type name",
|
|
55852
|
+
rewrite(args) {
|
|
55853
|
+
if (args.length !== 1)
|
|
55854
|
+
return null;
|
|
55855
|
+
const m = /^([\s\S]*?)\s+as\s+([A-Za-z][\w ]*)$/i.exec(args[0]);
|
|
55856
|
+
if (!m)
|
|
55857
|
+
return null;
|
|
55858
|
+
const t = pgType(m[2]);
|
|
55859
|
+
return t === m[2].trim().toLowerCase() ? null : `cast(${m[1]} as ${t})`;
|
|
55860
|
+
},
|
|
55861
|
+
},
|
|
55862
|
+
{
|
|
55863
|
+
name: "safe_cast",
|
|
55864
|
+
note: "SAFE_CAST → guarded cast via pg_input_is_valid (NULL on bad input, as in BigQuery)",
|
|
55865
|
+
rewrite(args) {
|
|
55866
|
+
if (args.length !== 1)
|
|
55867
|
+
return null;
|
|
55868
|
+
const m = /^([\s\S]*?)\s+as\s+([A-Za-z][\w ]*)$/i.exec(args[0]);
|
|
55869
|
+
if (!m)
|
|
55870
|
+
return null;
|
|
55871
|
+
const [, expr, type] = m;
|
|
55872
|
+
const t = pgType(type);
|
|
55873
|
+
if (t === "text")
|
|
55874
|
+
return `(${expr})::text`;
|
|
55875
|
+
return `(case when pg_input_is_valid((${expr})::text, '${t}') then (${expr})::${t} end)`;
|
|
55876
|
+
},
|
|
55877
|
+
},
|
|
55878
|
+
{
|
|
55879
|
+
name: "safe_divide",
|
|
55880
|
+
note: "SAFE_DIVIDE(a, b) → a / nullif(b, 0)",
|
|
55881
|
+
rewrite: args => (args.length === 2 ? `(${args[0]} / nullif(${args[1]}, 0))` : null),
|
|
55882
|
+
},
|
|
55883
|
+
{
|
|
55884
|
+
name: "collate",
|
|
55885
|
+
note: "COLLATE(x, '') → x (collation stripping is a no-op in PostgreSQL)",
|
|
55886
|
+
rewrite: args => args.length === 2 && (args[1] === "''" || args[1] === '""') ? args[0] : null,
|
|
55887
|
+
},
|
|
55888
|
+
{
|
|
55889
|
+
name: "regexp_extract",
|
|
55890
|
+
note: "REGEXP_EXTRACT(x, pattern) → substring(x from pattern)",
|
|
55891
|
+
rewrite: args => (args.length === 2 ? `substring(${args[0]} from ${args[1]})` : null),
|
|
55892
|
+
},
|
|
55893
|
+
{
|
|
55894
|
+
name: "split",
|
|
55895
|
+
note: "SPLIT(x, d) → string_to_array(x, d); an empty delimiter becomes NULL (per character)",
|
|
55896
|
+
rewrite(args) {
|
|
55897
|
+
if (args.length !== 2)
|
|
55898
|
+
return null;
|
|
55899
|
+
const d = args[1] === "''" || args[1] === '""' ? "NULL" : args[1];
|
|
55900
|
+
return `string_to_array(${args[0]}, ${d})`;
|
|
55901
|
+
},
|
|
55902
|
+
},
|
|
55903
|
+
{
|
|
55904
|
+
name: "date_sub",
|
|
55905
|
+
note: "DATE_SUB(d, INTERVAL n unit) → (d - interval 'n unit')",
|
|
55906
|
+
rewrite(args) {
|
|
55907
|
+
if (args.length !== 2)
|
|
55908
|
+
return null;
|
|
55909
|
+
const m = INTERVAL_ARG.exec(args[1]);
|
|
55910
|
+
return m ? `(${args[0]} - interval '${m[1]} ${m[2].toLowerCase()}')` : null;
|
|
55911
|
+
},
|
|
55912
|
+
},
|
|
55913
|
+
{
|
|
55914
|
+
name: "date_add",
|
|
55915
|
+
note: "DATE_ADD(d, INTERVAL n unit) → (d + interval 'n unit')",
|
|
55916
|
+
rewrite(args) {
|
|
55917
|
+
if (args.length !== 2)
|
|
55918
|
+
return null;
|
|
55919
|
+
const m = INTERVAL_ARG.exec(args[1]);
|
|
55920
|
+
return m ? `(${args[0]} + interval '${m[1]} ${m[2].toLowerCase()}')` : null;
|
|
55921
|
+
},
|
|
55922
|
+
},
|
|
55923
|
+
{
|
|
55924
|
+
name: "date_diff",
|
|
55925
|
+
note: "DATE_DIFF(a, b, DAY|HOUR) → PostgreSQL date/epoch arithmetic",
|
|
55926
|
+
rewrite(args) {
|
|
55927
|
+
if (args.length !== 3)
|
|
55928
|
+
return null;
|
|
55929
|
+
const unit = args[2].trim().toLowerCase();
|
|
55930
|
+
if (unit === "day")
|
|
55931
|
+
return `(${args[0]}::date - ${args[1]}::date)`;
|
|
55932
|
+
if (unit === "hour") {
|
|
55933
|
+
return (`(extract(epoch from date_trunc('hour', ${args[0]})` +
|
|
55934
|
+
` - date_trunc('hour', ${args[1]})) / 3600)::bigint`);
|
|
55935
|
+
}
|
|
55936
|
+
return null;
|
|
55937
|
+
},
|
|
55938
|
+
},
|
|
55939
|
+
{
|
|
55940
|
+
name: "unix_seconds",
|
|
55941
|
+
note: "UNIX_SECONDS(ts) → extract(epoch from ts)::bigint",
|
|
55942
|
+
rewrite: args => (args.length === 1 ? `extract(epoch from ${args[0]})::bigint` : null),
|
|
55943
|
+
},
|
|
55944
|
+
];
|
|
55945
|
+
function applyCallRules(text, isRewritableOffset) {
|
|
55946
|
+
const applied = [];
|
|
55947
|
+
let result = text;
|
|
55948
|
+
for (let pass = 0; pass < 10; pass++) {
|
|
55949
|
+
const mask = maskSql(result);
|
|
55950
|
+
const sites = [];
|
|
55951
|
+
for (const rule of CALL_RULES) {
|
|
55952
|
+
const finder = new RegExp(`\\b${rule.name}\\s*\\(`, "gi");
|
|
55953
|
+
let m;
|
|
55954
|
+
while ((m = finder.exec(mask)) !== null) {
|
|
55955
|
+
const open = m.index + m[0].length - 1;
|
|
55956
|
+
const close = matchParen(mask, open);
|
|
55957
|
+
if (close < 0 || !isRewritableOffset(m.index))
|
|
55958
|
+
continue;
|
|
55959
|
+
const out = rule.rewrite(splitArgs(result, mask, open + 1, close));
|
|
55960
|
+
if (out === null)
|
|
55961
|
+
continue;
|
|
55962
|
+
sites.push({ start: m.index, end: close + 1, out, note: rule.note });
|
|
55963
|
+
}
|
|
55964
|
+
}
|
|
55965
|
+
if (!sites.length)
|
|
55966
|
+
break;
|
|
55967
|
+
sites.sort((a, b) => b.start - a.start);
|
|
55968
|
+
const seen = [];
|
|
55969
|
+
for (const s of sites) {
|
|
55970
|
+
if (seen.some(p => s.start < p.end && p.start < s.end))
|
|
55971
|
+
continue;
|
|
55972
|
+
seen.push(s);
|
|
55973
|
+
applied.push({ line: result.slice(0, s.start).split("\n").length, note: s.note });
|
|
55974
|
+
result = result.slice(0, s.start) + s.out + result.slice(s.end);
|
|
55975
|
+
}
|
|
55976
|
+
}
|
|
55977
|
+
const before = result;
|
|
55978
|
+
result = result.replace(/\[\s*(?:SAFE_)?OFFSET\s*\(\s*([^()]*?)\s*\)\s*\]/gi, (_all, idx) => (/^\d+$/.test(idx) ? `[${Number(idx) + 1}]` : `[(${idx}) + 1]`));
|
|
55979
|
+
if (result !== before) {
|
|
55980
|
+
applied.push({ line: 1, note: "array[OFFSET(n)] → array[n + 1] (PostgreSQL is 1-based)" });
|
|
55981
|
+
}
|
|
55982
|
+
result = parenthesiseSubscriptedCalls(result);
|
|
55983
|
+
return { text: result, applied };
|
|
55984
|
+
}
|
|
55985
|
+
function jsBlockSpans(source) {
|
|
55986
|
+
const spans = [];
|
|
55987
|
+
const re = /(^|\n)\s*js\s*\{/g;
|
|
55988
|
+
let m;
|
|
55989
|
+
while ((m = re.exec(source))) {
|
|
55990
|
+
const open = source.indexOf("{", m.index + m[0].length - 1);
|
|
55991
|
+
const close = matchParenLike(source, open, "{", "}");
|
|
55992
|
+
spans.push({ start: m.index, end: close < 0 ? source.length : close + 1 });
|
|
55993
|
+
}
|
|
55994
|
+
return spans;
|
|
55995
|
+
}
|
|
55996
|
+
function matchParenLike(text, open, o, c) {
|
|
55997
|
+
let depth = 0;
|
|
55998
|
+
for (let i = open; i < text.length; i++) {
|
|
55999
|
+
if (text[i] === o)
|
|
56000
|
+
depth++;
|
|
56001
|
+
else if (text[i] === c && --depth === 0)
|
|
56002
|
+
return i;
|
|
56003
|
+
}
|
|
56004
|
+
return -1;
|
|
56005
|
+
}
|
|
56006
|
+
function applyTextRules(text, isSqlOffset, applied) {
|
|
56007
|
+
const note = (offset, n) => applied.push({ line: text.slice(0, offset).split("\n").length, note: n });
|
|
56008
|
+
const rewrite = (src, mask, pattern, replace, ruleNote) => {
|
|
56009
|
+
const edits = [];
|
|
56010
|
+
let m;
|
|
56011
|
+
pattern.lastIndex = 0;
|
|
56012
|
+
while ((m = pattern.exec(mask)) !== null) {
|
|
56013
|
+
if (!isSqlOffset(m.index))
|
|
56014
|
+
continue;
|
|
56015
|
+
const sub = replace(m);
|
|
56016
|
+
if (sub !== null)
|
|
56017
|
+
edits.push({ start: m.index, end: m.index + m[0].length, out: sub });
|
|
56018
|
+
}
|
|
56019
|
+
let out = src;
|
|
56020
|
+
for (const e of edits.reverse()) {
|
|
56021
|
+
note(e.start, ruleNote);
|
|
56022
|
+
out = out.slice(0, e.start) + e.out + out.slice(e.end);
|
|
56023
|
+
}
|
|
56024
|
+
return out;
|
|
56025
|
+
};
|
|
56026
|
+
let result = text;
|
|
56027
|
+
result = rewrite(result, maskSql(result, true), /(?<![\w"'])#[^\n]*/g, m => `--${m[0].slice(1)}`, "# line comment → -- (PostgreSQL has no # comment)");
|
|
56028
|
+
result = rewrite(result, maskSql(result, true), /(?<![A-Za-z0-9_])[rR](?=['"])/g, () => "", "r'…' raw string → '…' (no raw-string prefix in PostgreSQL)");
|
|
56029
|
+
result = rewrite(result, maskSql(result, true), /(\bas\s+)?"([^"\n]*)"/gi, m => {
|
|
56030
|
+
if (m[1])
|
|
56031
|
+
return null;
|
|
56032
|
+
return m[2].includes("${") || m[2].includes("'") ? null : `'${m[2]}'`;
|
|
56033
|
+
}, 'BigQuery "string" → \'string\' (double quotes are identifiers in PostgreSQL)');
|
|
56034
|
+
result = rewrite(result, maskSql(result), /`([^`\n]+)`/g, m => (m[1].includes(".") || m[1].includes("${") ? null : `"${m[1]}"`), "`identifier` → \"identifier\"");
|
|
56035
|
+
result = rewrite(result, maskSql(result, true), /\s+collate\s+(''|"")/gi, () => "", "x COLLATE '' → x (collation stripping is a no-op in PostgreSQL)");
|
|
56036
|
+
for (;;) {
|
|
56037
|
+
const mask = maskSql(result);
|
|
56038
|
+
const m = /\bextract\s*\(\s*dayofweek\s+from\s+/i.exec(mask);
|
|
56039
|
+
if (!m || !isSqlOffset(m.index))
|
|
56040
|
+
break;
|
|
56041
|
+
const close = matchParen(mask, mask.indexOf("(", m.index));
|
|
56042
|
+
if (close < 0)
|
|
56043
|
+
break;
|
|
56044
|
+
const inner = result.slice(m.index + m[0].length, close).trim();
|
|
56045
|
+
note(m.index, "EXTRACT(DAYOFWEEK FROM x) → extract(dow from x) + 1 (BigQuery is 1-based)");
|
|
56046
|
+
result =
|
|
56047
|
+
result.slice(0, m.index) + `(extract(dow from ${inner}) + 1)` + result.slice(close + 1);
|
|
56048
|
+
}
|
|
56049
|
+
return rewriteUnenforcedConstraints(result, isSqlOffset, applied);
|
|
56050
|
+
}
|
|
56051
|
+
const UNENFORCED_HEADER = [
|
|
56052
|
+
"BigQuery declared the constraint below NOT ENFORCED — advisory metadata for BI tools,",
|
|
56053
|
+
"never checked by the warehouse. PostgreSQL has no unenforced constraint, so the real",
|
|
56054
|
+
"equivalent is written out here and left commented: uncommenting enforces it, and it will",
|
|
56055
|
+
"fail if the data violates it. A foreign key also needs the referenced table's own key to",
|
|
56056
|
+
"exist, so enable those upstream first.",
|
|
56057
|
+
];
|
|
56058
|
+
function rewriteUnenforcedConstraints(text, isSqlOffset, applied) {
|
|
56059
|
+
const mask = maskSql(text);
|
|
56060
|
+
const sites = [];
|
|
56061
|
+
const finder = /\bALTER\s+TABLE\b/gi;
|
|
56062
|
+
let m;
|
|
56063
|
+
while ((m = finder.exec(mask)) !== null) {
|
|
56064
|
+
let depth = 0;
|
|
56065
|
+
let end = mask.length;
|
|
56066
|
+
for (let i = m.index; i < mask.length; i++) {
|
|
56067
|
+
if (mask[i] === "(")
|
|
56068
|
+
depth++;
|
|
56069
|
+
else if (mask[i] === ")")
|
|
56070
|
+
depth--;
|
|
56071
|
+
else if (mask[i] === ";" && depth === 0) {
|
|
56072
|
+
end = i + 1;
|
|
56073
|
+
break;
|
|
56074
|
+
}
|
|
56075
|
+
}
|
|
56076
|
+
finder.lastIndex = end;
|
|
56077
|
+
const stmt = text.slice(m.index, end);
|
|
56078
|
+
if (!/NOT\s+ENFORCED/i.test(stmt) || !isSqlOffset(m.index))
|
|
56079
|
+
continue;
|
|
56080
|
+
const body = stmt.replace(/;\s*$/, "").replace(/^ALTER\s+TABLE\s*/i, "");
|
|
56081
|
+
const firstAction = /\b(ADD|DROP|ALTER)\b/i.exec(body);
|
|
56082
|
+
if (!firstAction)
|
|
56083
|
+
continue;
|
|
56084
|
+
const target = body.slice(0, firstAction.index).trim();
|
|
56085
|
+
const actions = splitTopLevel(body.slice(firstAction.index))
|
|
56086
|
+
.map(a => a.trim())
|
|
56087
|
+
.filter(a => /^ADD\b/i.test(a))
|
|
56088
|
+
.map(a => a.replace(/\s*\bNOT\s+ENFORCED\b/gi, "").trim());
|
|
56089
|
+
if (!actions.length)
|
|
56090
|
+
continue;
|
|
56091
|
+
const lineStart = text.lastIndexOf("\n", m.index) + 1;
|
|
56092
|
+
const indent = /^[ \t]*/.exec(text.slice(lineStart))[0];
|
|
56093
|
+
const commented = [
|
|
56094
|
+
...UNENFORCED_HEADER.map(l => `-- ${l}`),
|
|
56095
|
+
...(actions.length === 1
|
|
56096
|
+
? [`-- ALTER TABLE ${target} ${actions[0]};`]
|
|
56097
|
+
: [
|
|
56098
|
+
`-- ALTER TABLE ${target}`,
|
|
56099
|
+
...actions.map((a, i) => `-- ${a}${i === actions.length - 1 ? ";" : ","}`),
|
|
56100
|
+
]),
|
|
56101
|
+
].join(`\n${indent}`);
|
|
56102
|
+
sites.push({ start: m.index, end, out: commented });
|
|
56103
|
+
}
|
|
56104
|
+
let result = text;
|
|
56105
|
+
for (const s of sites.reverse()) {
|
|
56106
|
+
applied.push({
|
|
56107
|
+
line: text.slice(0, s.start).split("\n").length,
|
|
56108
|
+
note: "NOT ENFORCED constraint → PostgreSQL equivalent, commented out " +
|
|
56109
|
+
"(enforcing it may fail on data that has never been checked)",
|
|
56110
|
+
});
|
|
56111
|
+
result = result.slice(0, s.start) + s.out + result.slice(s.end);
|
|
56112
|
+
}
|
|
56113
|
+
return result;
|
|
56114
|
+
}
|
|
56115
|
+
function splitTopLevel(s) {
|
|
56116
|
+
const parts = [];
|
|
56117
|
+
let depth = 0;
|
|
56118
|
+
let cur = "";
|
|
56119
|
+
for (const ch of s) {
|
|
56120
|
+
if (ch === "(")
|
|
56121
|
+
depth++;
|
|
56122
|
+
else if (ch === ")")
|
|
56123
|
+
depth--;
|
|
56124
|
+
if (ch === "," && depth === 0) {
|
|
56125
|
+
parts.push(cur);
|
|
56126
|
+
cur = "";
|
|
56127
|
+
}
|
|
56128
|
+
else
|
|
56129
|
+
cur += ch;
|
|
56130
|
+
}
|
|
56131
|
+
parts.push(cur);
|
|
56132
|
+
return parts;
|
|
56133
|
+
}
|
|
56134
|
+
function parenthesiseSubscriptedCalls(text) {
|
|
56135
|
+
const finder = /\b[A-Za-z_]\w*\s*\(/g;
|
|
56136
|
+
const spans = [];
|
|
56137
|
+
let m;
|
|
56138
|
+
while ((m = finder.exec(text)) !== null) {
|
|
56139
|
+
const close = matchParen(text, m.index + m[0].length - 1);
|
|
56140
|
+
if (close < 0)
|
|
56141
|
+
continue;
|
|
56142
|
+
if (/^\s*\[/.test(text.slice(close + 1)))
|
|
56143
|
+
spans.push({ start: m.index, end: close + 1 });
|
|
56144
|
+
}
|
|
56145
|
+
let out = text;
|
|
56146
|
+
for (const s of spans.reverse()) {
|
|
56147
|
+
out = `${out.slice(0, s.start)}(${out.slice(s.start, s.end)})${out.slice(s.end)}`;
|
|
56148
|
+
}
|
|
56149
|
+
return out;
|
|
56150
|
+
}
|
|
54715
56151
|
function convertTarget(source, jsMode, dialect = "postgres") {
|
|
56152
|
+
var _a;
|
|
54716
56153
|
const dialectPass = dialect === "postgres";
|
|
54717
|
-
|
|
56154
|
+
let span = jsMode ? null : findConfigBlock(source);
|
|
54718
56155
|
const findings = [];
|
|
56156
|
+
if (dialectPass) {
|
|
56157
|
+
const outsideConfig = (off) => {
|
|
56158
|
+
const s = jsMode ? null : findConfigBlock(source);
|
|
56159
|
+
return !s || off < s.start || off >= s.end;
|
|
56160
|
+
};
|
|
56161
|
+
const called = applyCallRules(source, outsideConfig);
|
|
56162
|
+
if (called.text !== source)
|
|
56163
|
+
source = called.text;
|
|
56164
|
+
for (const a of called.applied) {
|
|
56165
|
+
findings.push({ line: a.line, kind: "rewrite", note: a.note, class: classIdFor(a.note) });
|
|
56166
|
+
}
|
|
56167
|
+
const textApplied = [];
|
|
56168
|
+
const jsSpans = jsMode ? [{ start: 0, end: source.length }] : jsBlockSpans(source);
|
|
56169
|
+
const templates = tokenize(source).filter(t => t.kind === "template");
|
|
56170
|
+
source = applyTextRules(source, off => outsideConfig(off) &&
|
|
56171
|
+
!jsSpans.some(s => off >= s.start && off < s.end) &&
|
|
56172
|
+
!templates.some(t => off >= t.start && off < t.end), textApplied);
|
|
56173
|
+
for (const a of textApplied) {
|
|
56174
|
+
findings.push({
|
|
56175
|
+
line: a.line,
|
|
56176
|
+
kind: "rewrite",
|
|
56177
|
+
note: a.note,
|
|
56178
|
+
class: a.note.startsWith("NOT ENFORCED") ? "not-enforced" : classIdFor(a.note),
|
|
56179
|
+
});
|
|
56180
|
+
}
|
|
56181
|
+
span = jsMode ? null : findConfigBlock(source);
|
|
56182
|
+
}
|
|
54719
56183
|
const lines = source.split("\n");
|
|
54720
56184
|
const jsLines = jsMode ? new Set() : jsBlockLines(source);
|
|
54721
56185
|
let configStartLine = -1;
|
|
@@ -54780,15 +56244,16 @@ function convertTarget(source, jsMode, dialect = "postgres") {
|
|
|
54780
56244
|
rule.pattern.lastIndex = 0;
|
|
54781
56245
|
if (!rule.pattern.test(line))
|
|
54782
56246
|
continue;
|
|
56247
|
+
const cls = (_a = rule.id) !== null && _a !== void 0 ? _a : classIdFor(rule.note);
|
|
54783
56248
|
if (rule.replacement !== undefined) {
|
|
54784
56249
|
rule.pattern.lastIndex = 0;
|
|
54785
56250
|
line = line.replace(rule.pattern, rule.replacement);
|
|
54786
|
-
findings.push({ line: lineNo, kind: "rewrite", note: rule.note });
|
|
56251
|
+
findings.push({ line: lineNo, kind: "rewrite", note: rule.note, class: cls });
|
|
54787
56252
|
if (markersSafe)
|
|
54788
56253
|
notesForLine.push(rule.note);
|
|
54789
56254
|
}
|
|
54790
56255
|
else {
|
|
54791
|
-
findings.push({ line: lineNo, kind: "flag", note: rule.note });
|
|
56256
|
+
findings.push({ line: lineNo, kind: "flag", note: rule.note, class: cls });
|
|
54792
56257
|
if (markersSafe)
|
|
54793
56258
|
notesForLine.push(rule.note);
|
|
54794
56259
|
}
|
|
@@ -54864,6 +56329,8 @@ async function migrateDataform(opts) {
|
|
|
54864
56329
|
files: [],
|
|
54865
56330
|
skippedForSafety: [],
|
|
54866
56331
|
warnings: [],
|
|
56332
|
+
todo: [],
|
|
56333
|
+
applied: [],
|
|
54867
56334
|
};
|
|
54868
56335
|
const sqlxFiles = [];
|
|
54869
56336
|
for (const rel of files) {
|
|
@@ -55095,6 +56562,8 @@ async function migrateDataform(opts) {
|
|
|
55095
56562
|
if (skippedForSafety.size > 0) {
|
|
55096
56563
|
report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
|
|
55097
56564
|
}
|
|
56565
|
+
buildTriage(report);
|
|
56566
|
+
analyseStructArrays(report, outDir);
|
|
55098
56567
|
writer.write("migration-report.json", JSON.stringify(report, null, 2));
|
|
55099
56568
|
writer.write("migration-report.md", renderReportMd(report));
|
|
55100
56569
|
return report;
|
|
@@ -55156,6 +56625,71 @@ function renderReportMd(r) {
|
|
|
55156
56625
|
`of BigQuery connections.`);
|
|
55157
56626
|
}
|
|
55158
56627
|
}
|
|
56628
|
+
if (r.todo.length) {
|
|
56629
|
+
const needing = r.todo.filter(t => t.owner === "needs-decision");
|
|
56630
|
+
const mechanical = r.todo.filter(t => t.owner === "mechanical");
|
|
56631
|
+
lines.push("");
|
|
56632
|
+
lines.push(`## What is left to do`);
|
|
56633
|
+
lines.push("");
|
|
56634
|
+
lines.push(`${r.todo.reduce((n, t) => n + t.count, 0)} site(s) across ${r.todo.length} construct(s). ` +
|
|
56635
|
+
`Work them largest-first within each section — the ordering below already does that.`);
|
|
56636
|
+
lines.push("");
|
|
56637
|
+
lines.push(`**${mechanical.reduce((n, t) => n + t.count, 0)} site(s) are mechanical** — one correct ` +
|
|
56638
|
+
`answer, no judgement needed. **${needing.reduce((n, t) => n + t.count, 0)} site(s) need a ` +
|
|
56639
|
+
`decision** that cannot be read off the SQL: whether a feature is still wanted, what a ` +
|
|
56640
|
+
`structure should become, whether a behaviour was deliberate.`);
|
|
56641
|
+
const severityLabel = {
|
|
56642
|
+
"changes-meaning": "⚠ runs, but may return DIFFERENT RESULTS",
|
|
56643
|
+
"fails-at-run": "fails when the action runs",
|
|
56644
|
+
"blocks-compile": "fails to compile",
|
|
56645
|
+
};
|
|
56646
|
+
const section = (title, items, preamble) => {
|
|
56647
|
+
var _a;
|
|
56648
|
+
if (!items.length)
|
|
56649
|
+
return;
|
|
56650
|
+
lines.push("");
|
|
56651
|
+
lines.push(`### ${title}`);
|
|
56652
|
+
lines.push("");
|
|
56653
|
+
lines.push(preamble);
|
|
56654
|
+
for (const t of items) {
|
|
56655
|
+
lines.push("");
|
|
56656
|
+
lines.push(`- [ ] **${t.title}** — ${t.count} site(s) · ${severityLabel[t.severity]}`);
|
|
56657
|
+
if (t.postgres)
|
|
56658
|
+
lines.push(` - PostgreSQL: ${t.postgres}`);
|
|
56659
|
+
if (t.why)
|
|
56660
|
+
lines.push(` - ${t.why}`);
|
|
56661
|
+
const shown = t.locations.slice(0, 8);
|
|
56662
|
+
for (const l of shown) {
|
|
56663
|
+
lines.push(` - \`${l.file}\` L${l.lines.slice(0, 6).join(", L")}`);
|
|
56664
|
+
}
|
|
56665
|
+
if (t.locations.length > shown.length) {
|
|
56666
|
+
lines.push(` - …and ${t.locations.length - shown.length} more file(s)`);
|
|
56667
|
+
}
|
|
56668
|
+
for (const s of (_a = t.sql) !== null && _a !== void 0 ? _a : []) {
|
|
56669
|
+
lines.push("");
|
|
56670
|
+
lines.push(` \`${s.file}\` L${s.line} — ${s.note}`);
|
|
56671
|
+
lines.push("");
|
|
56672
|
+
lines.push(" ```sql");
|
|
56673
|
+
for (const l of s.sql.split("\n"))
|
|
56674
|
+
lines.push(` ${l}`);
|
|
56675
|
+
lines.push(" ```");
|
|
56676
|
+
}
|
|
56677
|
+
}
|
|
56678
|
+
};
|
|
56679
|
+
section("Needs a decision", needing, "Start here — these gate the rest, and each is a question for whoever owns the data, not " +
|
|
56680
|
+
"something to work out from the code.");
|
|
56681
|
+
section("Mechanical", mechanical, "These have one correct answer and can be worked through without further input.");
|
|
56682
|
+
}
|
|
56683
|
+
if (r.applied.length) {
|
|
56684
|
+
lines.push("");
|
|
56685
|
+
lines.push(`## Rewrites applied automatically`);
|
|
56686
|
+
lines.push("");
|
|
56687
|
+
lines.push(`Listed so they can be reviewed rather than trusted — a converted project is worth reading ` +
|
|
56688
|
+
`before it is run.`);
|
|
56689
|
+
lines.push("");
|
|
56690
|
+
for (const a of r.applied)
|
|
56691
|
+
lines.push(`- ${a.title} — ${a.count}`);
|
|
56692
|
+
}
|
|
55159
56693
|
lines.push("");
|
|
55160
56694
|
lines.push(`## Target files (${targets.length})`);
|
|
55161
56695
|
lines.push("");
|
|
@@ -55217,6 +56751,454 @@ function renderReportMd(r) {
|
|
|
55217
56751
|
return lines.join("\n");
|
|
55218
56752
|
}
|
|
55219
56753
|
|
|
56754
|
+
function parseColumnTypes(config) {
|
|
56755
|
+
const m = /\bcolumnTypes\s*:\s*\{/.exec(config);
|
|
56756
|
+
if (!m)
|
|
56757
|
+
return [];
|
|
56758
|
+
let depth = 0;
|
|
56759
|
+
let end = config.length;
|
|
56760
|
+
for (let i = m.index + m[0].length - 1; i < config.length; i++) {
|
|
56761
|
+
if (config[i] === "{")
|
|
56762
|
+
depth++;
|
|
56763
|
+
else if (config[i] === "}" && --depth === 0) {
|
|
56764
|
+
end = i;
|
|
56765
|
+
break;
|
|
56766
|
+
}
|
|
56767
|
+
}
|
|
56768
|
+
const body = config.slice(m.index + m[0].length, end);
|
|
56769
|
+
const withoutComments = body.replace(/\/\/[^\n]*/g, "");
|
|
56770
|
+
return [...withoutComments.matchAll(/(^|,)\s*([A-Za-z_]\w*)\s*:/g)].map(x => x[2]);
|
|
56771
|
+
}
|
|
56772
|
+
function loadActions(projectDir) {
|
|
56773
|
+
const actions = [];
|
|
56774
|
+
const walk = (dir) => {
|
|
56775
|
+
var _a;
|
|
56776
|
+
if (!fs__namespace.existsSync(dir))
|
|
56777
|
+
return;
|
|
56778
|
+
for (const entry of fs__namespace.readdirSync(dir, { withFileTypes: true })) {
|
|
56779
|
+
const full = path__namespace.join(dir, entry.name);
|
|
56780
|
+
if (entry.isDirectory())
|
|
56781
|
+
walk(full);
|
|
56782
|
+
else if (entry.name.endsWith(".sqlx")) {
|
|
56783
|
+
const source = fs__namespace.readFileSync(full, "utf8");
|
|
56784
|
+
const span = findConfigBlock(source);
|
|
56785
|
+
const config = span ? source.slice(span.start, span.end) : "";
|
|
56786
|
+
const parsed = parseSqlxConfig(config);
|
|
56787
|
+
actions.push({
|
|
56788
|
+
file: path__namespace.relative(projectDir, full),
|
|
56789
|
+
type: parsed.type,
|
|
56790
|
+
name: (_a = parsed.name) !== null && _a !== void 0 ? _a : path__namespace.basename(entry.name, ".sqlx"),
|
|
56791
|
+
schema: parsed.schema,
|
|
56792
|
+
columns: parseColumnTypes(config),
|
|
56793
|
+
body: span ? source.slice(span.end) : source,
|
|
56794
|
+
bodyOffset: span ? span.end : 0,
|
|
56795
|
+
source,
|
|
56796
|
+
});
|
|
56797
|
+
}
|
|
56798
|
+
}
|
|
56799
|
+
};
|
|
56800
|
+
walk(path__namespace.join(projectDir, "definitions"));
|
|
56801
|
+
return actions;
|
|
56802
|
+
}
|
|
56803
|
+
function outputName(toks, start, end) {
|
|
56804
|
+
var _a, _b;
|
|
56805
|
+
const items = toks.slice(start, end);
|
|
56806
|
+
if (!items.length)
|
|
56807
|
+
return null;
|
|
56808
|
+
if (items.some(t => t.text === "*"))
|
|
56809
|
+
return null;
|
|
56810
|
+
const last = items[items.length - 1];
|
|
56811
|
+
if (isWord(items[items.length - 2], "as")) {
|
|
56812
|
+
return ((_a = last.value) !== null && _a !== void 0 ? _a : last.text).replace(/^"|"$/g, "");
|
|
56813
|
+
}
|
|
56814
|
+
const bare = items.filter(t => t.text !== ".");
|
|
56815
|
+
if (bare.every(t => t.kind === "word" || t.kind === "quoted-ident")) {
|
|
56816
|
+
return ((_b = last.value) !== null && _b !== void 0 ? _b : last.text).replace(/^"|"$/g, "");
|
|
56817
|
+
}
|
|
56818
|
+
return null;
|
|
56819
|
+
}
|
|
56820
|
+
class Resolver {
|
|
56821
|
+
constructor(actions) {
|
|
56822
|
+
var _a;
|
|
56823
|
+
this.actions = actions;
|
|
56824
|
+
this.byName = new Map();
|
|
56825
|
+
this.byPair = new Map();
|
|
56826
|
+
this.memo = new Map();
|
|
56827
|
+
for (const a of actions) {
|
|
56828
|
+
this.byName.set(a.name, [...((_a = this.byName.get(a.name)) !== null && _a !== void 0 ? _a : []), a]);
|
|
56829
|
+
if (a.schema)
|
|
56830
|
+
this.byPair.set(`${a.schema}.${a.name}`, a);
|
|
56831
|
+
}
|
|
56832
|
+
}
|
|
56833
|
+
resolveRef(tok) {
|
|
56834
|
+
var _a;
|
|
56835
|
+
const args = [...tok.text.matchAll(/["']([^"']+)["']/g)].map(m => m[1]);
|
|
56836
|
+
if (!args.length)
|
|
56837
|
+
return { reason: "ref with no literal arguments" };
|
|
56838
|
+
const name = args[args.length - 1];
|
|
56839
|
+
if (args.length >= 2) {
|
|
56840
|
+
const hit = this.byPair.get(`${args[args.length - 2]}.${name}`);
|
|
56841
|
+
return hit ? { action: hit } : { reason: `no action ${args[args.length - 2]}.${name}` };
|
|
56842
|
+
}
|
|
56843
|
+
const candidates = (_a = this.byName.get(name)) !== null && _a !== void 0 ? _a : [];
|
|
56844
|
+
if (!candidates.length)
|
|
56845
|
+
return { reason: `unknown action ${name}` };
|
|
56846
|
+
if (candidates.length > 1)
|
|
56847
|
+
return { reason: `ambiguous action ${name}` };
|
|
56848
|
+
return { action: candidates[0] };
|
|
56849
|
+
}
|
|
56850
|
+
columnsOf(action, seen = new Set()) {
|
|
56851
|
+
if (action.columns.length)
|
|
56852
|
+
return action.columns;
|
|
56853
|
+
if (action.type === "declaration")
|
|
56854
|
+
return null;
|
|
56855
|
+
if (seen.has(action.file))
|
|
56856
|
+
return null;
|
|
56857
|
+
const memo = this.memo.get(action.file);
|
|
56858
|
+
if (memo !== undefined && !seen.size)
|
|
56859
|
+
return memo;
|
|
56860
|
+
seen.add(action.file);
|
|
56861
|
+
const result = this.selectColumns(action.body, new Set(seen));
|
|
56862
|
+
if (!seen.size)
|
|
56863
|
+
this.memo.set(action.file, result);
|
|
56864
|
+
return result;
|
|
56865
|
+
}
|
|
56866
|
+
selectColumns(sql, seen) {
|
|
56867
|
+
var _a;
|
|
56868
|
+
const toks = significant(tokenize(sql));
|
|
56869
|
+
const scopes = selectScopes(toks);
|
|
56870
|
+
if (!scopes.length)
|
|
56871
|
+
return null;
|
|
56872
|
+
const scope = scopes.reduce((best, s) => (s.select > best.select ? s : best), scopes[0]);
|
|
56873
|
+
const rels = scope.from === -1 ? [] : relationsIn(toks, scope.fromStart, scope.fromEnd);
|
|
56874
|
+
const names = [];
|
|
56875
|
+
for (const [s, e] of splitOnCommas(toks, scope.listStart, scope.listEnd)) {
|
|
56876
|
+
const item = toks.slice(s, e);
|
|
56877
|
+
const star = item.findIndex(t => t.text === "*");
|
|
56878
|
+
if (star === -1) {
|
|
56879
|
+
const n = outputName(toks, s, e);
|
|
56880
|
+
if (n === null)
|
|
56881
|
+
return null;
|
|
56882
|
+
names.push(n);
|
|
56883
|
+
continue;
|
|
56884
|
+
}
|
|
56885
|
+
const qualifier = star >= 2 && ((_a = item[star - 1]) === null || _a === void 0 ? void 0 : _a.text) === "." ? item[star - 2].text.toLowerCase() : null;
|
|
56886
|
+
const rel = qualifier ? rels.find(r => r.alias === qualifier) : rels[0];
|
|
56887
|
+
if (!rel)
|
|
56888
|
+
return null;
|
|
56889
|
+
const cols = this.relationColumns(rel.tokens, rel.subquery, seen);
|
|
56890
|
+
if (!cols)
|
|
56891
|
+
return null;
|
|
56892
|
+
const excluded = starExclusions(item, star);
|
|
56893
|
+
names.push(...cols.filter(c => !excluded.has(c.toLowerCase())));
|
|
56894
|
+
}
|
|
56895
|
+
return names.length ? names : null;
|
|
56896
|
+
}
|
|
56897
|
+
relationColumns(tokens, subquery, seen) {
|
|
56898
|
+
if (subquery) {
|
|
56899
|
+
const inner = tokens
|
|
56900
|
+
.slice(1, -1)
|
|
56901
|
+
.map(t => t.text)
|
|
56902
|
+
.join(" ");
|
|
56903
|
+
return this.selectColumns(inner, seen);
|
|
56904
|
+
}
|
|
56905
|
+
const tmpl = tokens.find(t => t.kind === "template");
|
|
56906
|
+
if (tmpl) {
|
|
56907
|
+
const { action } = this.resolveRef(tmpl);
|
|
56908
|
+
return action ? this.columnsOf(action, seen) : null;
|
|
56909
|
+
}
|
|
56910
|
+
return null;
|
|
56911
|
+
}
|
|
56912
|
+
}
|
|
56913
|
+
function starExclusions(item, star) {
|
|
56914
|
+
var _a, _b;
|
|
56915
|
+
const out = new Set();
|
|
56916
|
+
if (!isWord(item[star + 1], "except"))
|
|
56917
|
+
return out;
|
|
56918
|
+
const open = star + 2;
|
|
56919
|
+
if (((_a = item[open]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
56920
|
+
return out;
|
|
56921
|
+
const close = matchBracket(item, open);
|
|
56922
|
+
if (close < 0)
|
|
56923
|
+
return out;
|
|
56924
|
+
for (const [s, e] of splitOnCommas(item, open + 1, close)) {
|
|
56925
|
+
const t = item[s];
|
|
56926
|
+
if (t)
|
|
56927
|
+
out.add(((_b = t.value) !== null && _b !== void 0 ? _b : t.text).replace(/^"|"$/g, "").toLowerCase());
|
|
56928
|
+
}
|
|
56929
|
+
return out;
|
|
56930
|
+
}
|
|
56931
|
+
function quoteIfNeeded(col) {
|
|
56932
|
+
return /^[a-z_][a-z0-9_]*$/.test(col) ? col : `"${col}"`;
|
|
56933
|
+
}
|
|
56934
|
+
const MARKER = "-- sqlanvil:star-except";
|
|
56935
|
+
async function migrateFix(opts) {
|
|
56936
|
+
var _a;
|
|
56937
|
+
const result = { expanded: 0, groupByAll: 0, files: [], unresolved: [] };
|
|
56938
|
+
const touched = new Set();
|
|
56939
|
+
for (let pass = 0; pass < 12; pass++) {
|
|
56940
|
+
const actions = loadActions(opts.projectDir);
|
|
56941
|
+
const resolver = new Resolver(actions);
|
|
56942
|
+
let changedThisPass = false;
|
|
56943
|
+
result.unresolved = [];
|
|
56944
|
+
for (const action of actions) {
|
|
56945
|
+
const source = action.source;
|
|
56946
|
+
const toks = significant(tokenize(source));
|
|
56947
|
+
const scopes = selectScopes(toks);
|
|
56948
|
+
const edits = [];
|
|
56949
|
+
for (const scope of scopes) {
|
|
56950
|
+
const rels = scope.from === -1 ? [] : relationsIn(toks, scope.fromStart, scope.fromEnd);
|
|
56951
|
+
for (const [s, e] of splitOnCommas(toks, scope.listStart, scope.listEnd)) {
|
|
56952
|
+
const item = toks.slice(s, e);
|
|
56953
|
+
const star = item.findIndex(t => t.text === "*");
|
|
56954
|
+
if (star === -1 || !isWord(item[star + 1], "except"))
|
|
56955
|
+
continue;
|
|
56956
|
+
const line = source.slice(0, item[star].start).split("\n").length;
|
|
56957
|
+
const qualifier = star >= 2 && ((_a = item[star - 1]) === null || _a === void 0 ? void 0 : _a.text) === "." ? item[star - 2].text.toLowerCase() : null;
|
|
56958
|
+
const rel = qualifier ? rels.find(r => r.alias === qualifier) : rels[0];
|
|
56959
|
+
if (!rel) {
|
|
56960
|
+
result.unresolved.push({
|
|
56961
|
+
file: action.file,
|
|
56962
|
+
line,
|
|
56963
|
+
reason: qualifier
|
|
56964
|
+
? `no relation aliased "${qualifier}" in the FROM/JOIN chain`
|
|
56965
|
+
: "star has no FROM relation",
|
|
56966
|
+
});
|
|
56967
|
+
continue;
|
|
56968
|
+
}
|
|
56969
|
+
let cols = resolver.relationColumns(rel.tokens, rel.subquery, new Set());
|
|
56970
|
+
if (!cols) {
|
|
56971
|
+
const bare = rel.tokens.map(t => t.text).join("");
|
|
56972
|
+
const cte = cteColumns(source, bare, resolver);
|
|
56973
|
+
cols = cte;
|
|
56974
|
+
}
|
|
56975
|
+
if (!cols) {
|
|
56976
|
+
const shown = rel.tokens.map(t => t.text).join("");
|
|
56977
|
+
result.unresolved.push({
|
|
56978
|
+
file: action.file,
|
|
56979
|
+
line,
|
|
56980
|
+
reason: rel.tokens.some(t => t.kind === "template")
|
|
56981
|
+
? `columns of ${shown} are unknown — has introspect run for it?`
|
|
56982
|
+
: `cannot determine the columns of ${shown}`,
|
|
56983
|
+
});
|
|
56984
|
+
continue;
|
|
56985
|
+
}
|
|
56986
|
+
const excluded = starExclusions(item, star);
|
|
56987
|
+
const missing = [...excluded].filter(x => !cols.some(c => c.toLowerCase() === x));
|
|
56988
|
+
if (missing.length) {
|
|
56989
|
+
result.unresolved.push({
|
|
56990
|
+
file: action.file,
|
|
56991
|
+
line,
|
|
56992
|
+
reason: `EXCEPT names ${missing.join(", ")}, which the source does not have`,
|
|
56993
|
+
});
|
|
56994
|
+
continue;
|
|
56995
|
+
}
|
|
56996
|
+
const keep = cols.filter(c => !excluded.has(c.toLowerCase()));
|
|
56997
|
+
if (!keep.length) {
|
|
56998
|
+
result.unresolved.push({
|
|
56999
|
+
file: action.file,
|
|
57000
|
+
line,
|
|
57001
|
+
reason: "the EXCEPT list removes every column",
|
|
57002
|
+
});
|
|
57003
|
+
continue;
|
|
57004
|
+
}
|
|
57005
|
+
const close = matchBracket(item, star + 2);
|
|
57006
|
+
const lineStart = source.lastIndexOf("\n", item[star].start) + 1;
|
|
57007
|
+
const indent = /^[ \t]*/.exec(source.slice(lineStart))[0] + " ";
|
|
57008
|
+
const prefix = qualifier ? `${qualifier}.` : "";
|
|
57009
|
+
const body = keep.map(c => `${indent}${prefix}${quoteIfNeeded(c)}`).join(",\n");
|
|
57010
|
+
const marker = `${indent}${MARKER} ${prefix}* from ` +
|
|
57011
|
+
`${rel.tokens.map(t => t.text).join("")} minus (${[...excluded].join(", ")})`;
|
|
57012
|
+
edits.push({
|
|
57013
|
+
start: qualifier ? item[star - 2].start : item[star].start,
|
|
57014
|
+
end: item[close].end,
|
|
57015
|
+
out: `\n${marker}\n${body}`,
|
|
57016
|
+
});
|
|
57017
|
+
}
|
|
57018
|
+
}
|
|
57019
|
+
if (!edits.length)
|
|
57020
|
+
continue;
|
|
57021
|
+
let out = source;
|
|
57022
|
+
for (const edit of edits.sort((a, b) => b.start - a.start)) {
|
|
57023
|
+
const head = out.slice(0, edit.start).replace(/[ \t]+$/, "");
|
|
57024
|
+
out = head + edit.out + out.slice(edit.end);
|
|
57025
|
+
result.expanded++;
|
|
57026
|
+
}
|
|
57027
|
+
changedThisPass = true;
|
|
57028
|
+
touched.add(action.file);
|
|
57029
|
+
if (opts.write) {
|
|
57030
|
+
fs__namespace.writeFileSync(path__namespace.join(opts.projectDir, action.file), out);
|
|
57031
|
+
}
|
|
57032
|
+
else {
|
|
57033
|
+
action.source = out;
|
|
57034
|
+
}
|
|
57035
|
+
}
|
|
57036
|
+
if (!changedThisPass || !opts.write)
|
|
57037
|
+
break;
|
|
57038
|
+
}
|
|
57039
|
+
for (const action of loadActions(opts.projectDir)) {
|
|
57040
|
+
const { text, expanded, unresolved } = expandGroupByAll(action.source);
|
|
57041
|
+
for (const u of unresolved) {
|
|
57042
|
+
result.unresolved.push({ file: action.file, line: u.line, reason: u.reason });
|
|
57043
|
+
}
|
|
57044
|
+
if (!expanded)
|
|
57045
|
+
continue;
|
|
57046
|
+
result.groupByAll += expanded;
|
|
57047
|
+
touched.add(action.file);
|
|
57048
|
+
if (opts.write)
|
|
57049
|
+
fs__namespace.writeFileSync(path__namespace.join(opts.projectDir, action.file), text);
|
|
57050
|
+
}
|
|
57051
|
+
result.files = [...touched].sort();
|
|
57052
|
+
if (opts.write)
|
|
57053
|
+
updateMigrationReport(opts.projectDir, result);
|
|
57054
|
+
return result;
|
|
57055
|
+
}
|
|
57056
|
+
function updateMigrationReport(projectDir, result) {
|
|
57057
|
+
var _a, _b, _c;
|
|
57058
|
+
const jsonPath = path__namespace.join(projectDir, "migration-report.json");
|
|
57059
|
+
if (!fs__namespace.existsSync(jsonPath))
|
|
57060
|
+
return;
|
|
57061
|
+
let report;
|
|
57062
|
+
try {
|
|
57063
|
+
report = JSON.parse(fs__namespace.readFileSync(jsonPath, "utf8"));
|
|
57064
|
+
}
|
|
57065
|
+
catch (_d) {
|
|
57066
|
+
return;
|
|
57067
|
+
}
|
|
57068
|
+
if (!Array.isArray(report.todo))
|
|
57069
|
+
return;
|
|
57070
|
+
const OWNED = new Set(["star-except", "group-by-all"]);
|
|
57071
|
+
const remaining = new Map();
|
|
57072
|
+
for (const site of result.unresolved) {
|
|
57073
|
+
const id = /GROUP BY ALL/i.test(site.reason) || /star expands/i.test(site.reason)
|
|
57074
|
+
? "group-by-all"
|
|
57075
|
+
: "star-except";
|
|
57076
|
+
const prior = report.todo.find(t => t.id === id);
|
|
57077
|
+
const entry = (_a = remaining.get(id)) !== null && _a !== void 0 ? _a : {
|
|
57078
|
+
id,
|
|
57079
|
+
title: (_b = prior === null || prior === void 0 ? void 0 : prior.title) !== null && _b !== void 0 ? _b : id,
|
|
57080
|
+
count: 0,
|
|
57081
|
+
severity: (_c = prior === null || prior === void 0 ? void 0 : prior.severity) !== null && _c !== void 0 ? _c : "blocks-compile",
|
|
57082
|
+
owner: "needs-decision",
|
|
57083
|
+
postgres: prior === null || prior === void 0 ? void 0 : prior.postgres,
|
|
57084
|
+
why: site.reason,
|
|
57085
|
+
locations: [],
|
|
57086
|
+
};
|
|
57087
|
+
entry.count++;
|
|
57088
|
+
const loc = entry.locations.find(l => l.file === site.file);
|
|
57089
|
+
if (loc)
|
|
57090
|
+
loc.lines.push(site.line);
|
|
57091
|
+
else
|
|
57092
|
+
entry.locations.push({ file: site.file, lines: [site.line] });
|
|
57093
|
+
remaining.set(id, entry);
|
|
57094
|
+
}
|
|
57095
|
+
report.todo = [...report.todo.filter(t => !OWNED.has(t.id)), ...remaining.values()];
|
|
57096
|
+
const applied = (n, id, title) => {
|
|
57097
|
+
if (!n)
|
|
57098
|
+
return;
|
|
57099
|
+
const seen = report.applied.find(a => a.id === id);
|
|
57100
|
+
if (seen)
|
|
57101
|
+
seen.count = n;
|
|
57102
|
+
else
|
|
57103
|
+
report.applied.push({ id, title, count: n });
|
|
57104
|
+
};
|
|
57105
|
+
applied(result.expanded, "star-except", "SELECT * EXCEPT expanded (migrate-fix)");
|
|
57106
|
+
applied(result.groupByAll, "group-by-all", "GROUP BY ALL rewritten as ordinals (migrate-fix)");
|
|
57107
|
+
fs__namespace.writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
|
57108
|
+
fs__namespace.writeFileSync(path__namespace.join(projectDir, "migration-report.md"), renderReportMd(report));
|
|
57109
|
+
}
|
|
57110
|
+
const AGGREGATES = new Set([
|
|
57111
|
+
"count", "sum", "avg", "min", "max", "array_agg", "string_agg", "bool_and", "bool_or", "every",
|
|
57112
|
+
"stddev", "stddev_pop", "stddev_samp", "variance", "var_pop", "var_samp", "jsonb_agg",
|
|
57113
|
+
"json_agg", "jsonb_object_agg", "json_object_agg", "corr", "covar_pop", "covar_samp",
|
|
57114
|
+
"percentile_cont", "percentile_disc", "bit_and", "bit_or", "logical_and", "logical_or",
|
|
57115
|
+
"any_value", "countif",
|
|
57116
|
+
]);
|
|
57117
|
+
function isGroupable(item) {
|
|
57118
|
+
var _a, _b;
|
|
57119
|
+
for (let i = 0; i < item.length; i++) {
|
|
57120
|
+
if (item[i].kind !== "word" || !AGGREGATES.has(item[i].text.toLowerCase()))
|
|
57121
|
+
continue;
|
|
57122
|
+
if (((_a = item[i + 1]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
57123
|
+
continue;
|
|
57124
|
+
const close = matchBracket(item, i + 1);
|
|
57125
|
+
if (close < 0)
|
|
57126
|
+
return false;
|
|
57127
|
+
let after = close + 1;
|
|
57128
|
+
if (isWord(item[after], "filter") && ((_b = item[after + 1]) === null || _b === void 0 ? void 0 : _b.text) === "(") {
|
|
57129
|
+
const filterClose = matchBracket(item, after + 1);
|
|
57130
|
+
if (filterClose > 0)
|
|
57131
|
+
after = filterClose + 1;
|
|
57132
|
+
}
|
|
57133
|
+
if (!isWord(item[after], "over"))
|
|
57134
|
+
return false;
|
|
57135
|
+
}
|
|
57136
|
+
return !item.some((t, i) => isWord(t, "over") && i > 0);
|
|
57137
|
+
}
|
|
57138
|
+
function isStarItem(item) {
|
|
57139
|
+
if (!item.length)
|
|
57140
|
+
return false;
|
|
57141
|
+
if (item[0].text === "*")
|
|
57142
|
+
return true;
|
|
57143
|
+
return item.length >= 3 && item[1].text === "." && item[2].text === "*";
|
|
57144
|
+
}
|
|
57145
|
+
function expandGroupByAll(source) {
|
|
57146
|
+
const toks = significant(tokenize(source));
|
|
57147
|
+
const scopes = selectScopes(toks);
|
|
57148
|
+
const edits = [];
|
|
57149
|
+
const unresolved = [];
|
|
57150
|
+
for (let i = 0; i + 2 < toks.length; i++) {
|
|
57151
|
+
if (!isWord(toks[i], "group") || !isWord(toks[i + 1], "by") || !isWord(toks[i + 2], "all")) {
|
|
57152
|
+
continue;
|
|
57153
|
+
}
|
|
57154
|
+
const line = source.slice(0, toks[i].start).split("\n").length;
|
|
57155
|
+
const scope = scopes.filter(s => s.select < i).pop();
|
|
57156
|
+
if (!scope)
|
|
57157
|
+
continue;
|
|
57158
|
+
const items = splitOnCommas(toks, scope.listStart, scope.listEnd);
|
|
57159
|
+
if (items.some(([s, e]) => isStarItem(toks.slice(s, e)))) {
|
|
57160
|
+
unresolved.push({
|
|
57161
|
+
line,
|
|
57162
|
+
reason: "GROUP BY ALL over a select list containing `*` — the ordinals depend on how many " +
|
|
57163
|
+
"columns the star expands to",
|
|
57164
|
+
});
|
|
57165
|
+
continue;
|
|
57166
|
+
}
|
|
57167
|
+
const ordinals = items
|
|
57168
|
+
.map(([s, e], idx) => (isGroupable(toks.slice(s, e)) ? idx + 1 : null))
|
|
57169
|
+
.filter((n) => n !== null);
|
|
57170
|
+
if (!ordinals.length) {
|
|
57171
|
+
unresolved.push({ line, reason: "every select-list item is an aggregate" });
|
|
57172
|
+
continue;
|
|
57173
|
+
}
|
|
57174
|
+
edits.push({ start: toks[i + 2].start, end: toks[i + 2].end, out: ordinals.join(", ") });
|
|
57175
|
+
}
|
|
57176
|
+
let text = source;
|
|
57177
|
+
for (const e of edits.reverse()) {
|
|
57178
|
+
text = text.slice(0, e.start) + e.out + text.slice(e.end);
|
|
57179
|
+
}
|
|
57180
|
+
return { text, expanded: edits.length, unresolved };
|
|
57181
|
+
}
|
|
57182
|
+
function cteColumns(source, name, resolver) {
|
|
57183
|
+
var _a;
|
|
57184
|
+
const toks = significant(tokenize(source));
|
|
57185
|
+
for (let i = 0; i < toks.length; i++) {
|
|
57186
|
+
if (toks[i].kind !== "word" || toks[i].text.toLowerCase() !== name.toLowerCase())
|
|
57187
|
+
continue;
|
|
57188
|
+
if (!isWord(toks[i + 1], "as") || ((_a = toks[i + 2]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
57189
|
+
continue;
|
|
57190
|
+
const close = matchBracket(toks, i + 2);
|
|
57191
|
+
if (close < 0)
|
|
57192
|
+
return null;
|
|
57193
|
+
const inner = toks
|
|
57194
|
+
.slice(i + 3, close)
|
|
57195
|
+
.map(t => t.text)
|
|
57196
|
+
.join(" ");
|
|
57197
|
+
return resolver.selectColumns(inner, new Set());
|
|
57198
|
+
}
|
|
57199
|
+
return null;
|
|
57200
|
+
}
|
|
57201
|
+
|
|
55220
57202
|
function actuallyResolve(...filePaths) {
|
|
55221
57203
|
return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
|
|
55222
57204
|
}
|
|
@@ -58354,6 +60336,39 @@ function runCli() {
|
|
|
58354
60336
|
print(`Next: sqlanvil compile ${argv["out-dir"]}`);
|
|
58355
60337
|
return 0;
|
|
58356
60338
|
}
|
|
60339
|
+
},
|
|
60340
|
+
{
|
|
60341
|
+
format: "migrate-fix [project-dir]",
|
|
60342
|
+
description: "Finish a converted project's dialect work, AFTER `scripts/introspect_all.sh` has run. " +
|
|
60343
|
+
"Expands `SELECT * EXCEPT (...)` into explicit column lists and rewrites `GROUP BY ALL` " +
|
|
60344
|
+
"as positional ordinals — both need the source columns that only exist once " +
|
|
60345
|
+
"declarations have been introspected, so this is the " +
|
|
60346
|
+
"third phase of a migration (convert, introspect, fix), not part of the conversion. " +
|
|
60347
|
+
"Re-runnable; reports anything it could not resolve instead of guessing.",
|
|
60348
|
+
positionalOptions: [projectDirMustExistOption],
|
|
60349
|
+
options: [
|
|
60350
|
+
option("dry-run", {
|
|
60351
|
+
describe: "Report what would change without writing.",
|
|
60352
|
+
type: "boolean",
|
|
60353
|
+
default: false
|
|
60354
|
+
})
|
|
60355
|
+
],
|
|
60356
|
+
processFn: async (argv) => {
|
|
60357
|
+
const result = await migrateFix({
|
|
60358
|
+
projectDir: argv["project-dir"],
|
|
60359
|
+
write: !argv["dry-run"]
|
|
60360
|
+
});
|
|
60361
|
+
const verb = argv["dry-run"] ? "would rewrite" : "rewrote";
|
|
60362
|
+
print(`${verb} ${result.expanded} star-except site(s) and ${result.groupByAll} ` +
|
|
60363
|
+
`GROUP BY ALL clause(s) across ${result.files.length} file(s).`);
|
|
60364
|
+
if (result.unresolved.length) {
|
|
60365
|
+
printError(`${result.unresolved.length} site(s) need a look:`);
|
|
60366
|
+
for (const u of result.unresolved) {
|
|
60367
|
+
printError(` ${u.file}:${u.line} — ${u.reason}`);
|
|
60368
|
+
}
|
|
60369
|
+
}
|
|
60370
|
+
return result.unresolved.length ? 1 : 0;
|
|
60371
|
+
}
|
|
58357
60372
|
}
|
|
58358
60373
|
]
|
|
58359
60374
|
})
|