@sqlanvil/cli 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +1237 -50
- package/package.json +1 -1
- package/worker_bundle.js +625 -0
package/bundle.js
CHANGED
|
@@ -29,6 +29,7 @@ var childProcess = require('child_process');
|
|
|
29
29
|
var fs = require('fs-extra');
|
|
30
30
|
var tmp = require('tmp');
|
|
31
31
|
var util = require('util');
|
|
32
|
+
var mysql = require('mysql2/promise');
|
|
32
33
|
var pg = require('pg');
|
|
33
34
|
var bigquery = require('@google-cloud/bigquery');
|
|
34
35
|
var EventEmitter = require('events');
|
|
@@ -36,7 +37,6 @@ var Long = require('long');
|
|
|
36
37
|
var os = require('os');
|
|
37
38
|
var promisePoolExecutor = require('promise-pool-executor');
|
|
38
39
|
var sizeof = require('object-sizeof');
|
|
39
|
-
var mysql = require('mysql2/promise');
|
|
40
40
|
var QueryStream = require('pg-query-stream');
|
|
41
41
|
var readlineSync = require('readline-sync');
|
|
42
42
|
var untildify = require('untildify');
|
|
@@ -76,12 +76,12 @@ var semver__namespace = /*#__PURE__*/_interopNamespace(semver);
|
|
|
76
76
|
var childProcess__namespace = /*#__PURE__*/_interopNamespace(childProcess);
|
|
77
77
|
var fs__namespace = /*#__PURE__*/_interopNamespace(fs);
|
|
78
78
|
var tmp__namespace = /*#__PURE__*/_interopNamespace(tmp);
|
|
79
|
+
var mysql__namespace = /*#__PURE__*/_interopNamespace(mysql);
|
|
79
80
|
var pg__namespace = /*#__PURE__*/_interopNamespace(pg);
|
|
80
81
|
var EventEmitter__default = /*#__PURE__*/_interopDefaultLegacy(EventEmitter);
|
|
81
82
|
var Long__default = /*#__PURE__*/_interopDefaultLegacy(Long);
|
|
82
83
|
var os__namespace = /*#__PURE__*/_interopNamespace(os);
|
|
83
84
|
var sizeof__default = /*#__PURE__*/_interopDefaultLegacy(sizeof);
|
|
84
|
-
var mysql__namespace = /*#__PURE__*/_interopNamespace(mysql);
|
|
85
85
|
var QueryStream__default = /*#__PURE__*/_interopDefaultLegacy(QueryStream);
|
|
86
86
|
var readlineSync__namespace = /*#__PURE__*/_interopNamespace(readlineSync);
|
|
87
87
|
var untildify__default = /*#__PURE__*/_interopDefaultLegacy(untildify);
|
|
@@ -15249,6 +15249,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15249
15249
|
* @property {string|null} [charset] MysqlOptions charset
|
|
15250
15250
|
* @property {string|null} [collation] MysqlOptions collation
|
|
15251
15251
|
* @property {Array.<sqlanvil.MysqlOptions.IIndex>|null} [indexes] MysqlOptions indexes
|
|
15252
|
+
* @property {sqlanvil.MysqlOptions.IPartition|null} [partition] MysqlOptions partition
|
|
15252
15253
|
*/
|
|
15253
15254
|
|
|
15254
15255
|
/**
|
|
@@ -15299,6 +15300,14 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15299
15300
|
*/
|
|
15300
15301
|
MysqlOptions.prototype.indexes = $util.emptyArray;
|
|
15301
15302
|
|
|
15303
|
+
/**
|
|
15304
|
+
* MysqlOptions partition.
|
|
15305
|
+
* @member {sqlanvil.MysqlOptions.IPartition|null|undefined} partition
|
|
15306
|
+
* @memberof sqlanvil.MysqlOptions
|
|
15307
|
+
* @instance
|
|
15308
|
+
*/
|
|
15309
|
+
MysqlOptions.prototype.partition = null;
|
|
15310
|
+
|
|
15302
15311
|
/**
|
|
15303
15312
|
* Creates a new MysqlOptions instance using the specified properties.
|
|
15304
15313
|
* @function create
|
|
@@ -15332,6 +15341,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15332
15341
|
if (message.indexes != null && message.indexes.length)
|
|
15333
15342
|
for (let i = 0; i < message.indexes.length; ++i)
|
|
15334
15343
|
$root.sqlanvil.MysqlOptions.Index.encode(message.indexes[i], writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim();
|
|
15344
|
+
if (message.partition != null && Object.hasOwnProperty.call(message, "partition"))
|
|
15345
|
+
$root.sqlanvil.MysqlOptions.Partition.encode(message.partition, writer.uint32(/* id 5, wireType 2 =*/42).fork()).ldelim();
|
|
15335
15346
|
return writer;
|
|
15336
15347
|
};
|
|
15337
15348
|
|
|
@@ -15390,6 +15401,10 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15390
15401
|
message.indexes.push($root.sqlanvil.MysqlOptions.Index.decode(reader, reader.uint32(), undefined, long + 1));
|
|
15391
15402
|
break;
|
|
15392
15403
|
}
|
|
15404
|
+
case 5: {
|
|
15405
|
+
message.partition = $root.sqlanvil.MysqlOptions.Partition.decode(reader, reader.uint32(), undefined, long + 1);
|
|
15406
|
+
break;
|
|
15407
|
+
}
|
|
15393
15408
|
default:
|
|
15394
15409
|
reader.skipType(tag & 7, long);
|
|
15395
15410
|
break;
|
|
@@ -15447,6 +15462,11 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15447
15462
|
return "indexes." + error;
|
|
15448
15463
|
}
|
|
15449
15464
|
}
|
|
15465
|
+
if (message.partition != null && message.hasOwnProperty("partition")) {
|
|
15466
|
+
let error = $root.sqlanvil.MysqlOptions.Partition.verify(message.partition, long + 1);
|
|
15467
|
+
if (error)
|
|
15468
|
+
return "partition." + error;
|
|
15469
|
+
}
|
|
15450
15470
|
return null;
|
|
15451
15471
|
};
|
|
15452
15472
|
|
|
@@ -15482,6 +15502,11 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15482
15502
|
message.indexes[i] = $root.sqlanvil.MysqlOptions.Index.fromObject(object.indexes[i], long + 1);
|
|
15483
15503
|
}
|
|
15484
15504
|
}
|
|
15505
|
+
if (object.partition != null) {
|
|
15506
|
+
if (typeof object.partition !== "object")
|
|
15507
|
+
throw TypeError(".sqlanvil.MysqlOptions.partition: object expected");
|
|
15508
|
+
message.partition = $root.sqlanvil.MysqlOptions.Partition.fromObject(object.partition, long + 1);
|
|
15509
|
+
}
|
|
15485
15510
|
return message;
|
|
15486
15511
|
};
|
|
15487
15512
|
|
|
@@ -15504,6 +15529,7 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15504
15529
|
object.engine = "";
|
|
15505
15530
|
object.charset = "";
|
|
15506
15531
|
object.collation = "";
|
|
15532
|
+
object.partition = null;
|
|
15507
15533
|
}
|
|
15508
15534
|
if (message.engine != null && message.hasOwnProperty("engine"))
|
|
15509
15535
|
object.engine = message.engine;
|
|
@@ -15516,6 +15542,8 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15516
15542
|
for (let j = 0; j < message.indexes.length; ++j)
|
|
15517
15543
|
object.indexes[j] = $root.sqlanvil.MysqlOptions.Index.toObject(message.indexes[j], options);
|
|
15518
15544
|
}
|
|
15545
|
+
if (message.partition != null && message.hasOwnProperty("partition"))
|
|
15546
|
+
object.partition = $root.sqlanvil.MysqlOptions.Partition.toObject(message.partition, options);
|
|
15519
15547
|
return object;
|
|
15520
15548
|
};
|
|
15521
15549
|
|
|
@@ -15826,6 +15854,603 @@ const sqlanvil = $root.sqlanvil = (() => {
|
|
|
15826
15854
|
return Index;
|
|
15827
15855
|
})();
|
|
15828
15856
|
|
|
15857
|
+
MysqlOptions.Partition = (function() {
|
|
15858
|
+
|
|
15859
|
+
/**
|
|
15860
|
+
* Properties of a Partition.
|
|
15861
|
+
* @memberof sqlanvil.MysqlOptions
|
|
15862
|
+
* @interface IPartition
|
|
15863
|
+
* @property {sqlanvil.MysqlOptions.Partition.Kind|null} [kind] Partition kind
|
|
15864
|
+
* @property {string|null} [expression] Partition expression
|
|
15865
|
+
* @property {Array.<sqlanvil.MysqlOptions.Partition.IBound>|null} [partitions] Partition partitions
|
|
15866
|
+
* @property {number|null} [count] Partition count
|
|
15867
|
+
*/
|
|
15868
|
+
|
|
15869
|
+
/**
|
|
15870
|
+
* Constructs a new Partition.
|
|
15871
|
+
* @memberof sqlanvil.MysqlOptions
|
|
15872
|
+
* @classdesc Represents a Partition.
|
|
15873
|
+
* @implements IPartition
|
|
15874
|
+
* @constructor
|
|
15875
|
+
* @param {sqlanvil.MysqlOptions.IPartition=} [properties] Properties to set
|
|
15876
|
+
*/
|
|
15877
|
+
function Partition(properties) {
|
|
15878
|
+
this.partitions = [];
|
|
15879
|
+
if (properties)
|
|
15880
|
+
for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
|
|
15881
|
+
if (properties[keys[i]] != null && keys[i] !== "__proto__")
|
|
15882
|
+
this[keys[i]] = properties[keys[i]];
|
|
15883
|
+
}
|
|
15884
|
+
|
|
15885
|
+
/**
|
|
15886
|
+
* Partition kind.
|
|
15887
|
+
* @member {sqlanvil.MysqlOptions.Partition.Kind} kind
|
|
15888
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15889
|
+
* @instance
|
|
15890
|
+
*/
|
|
15891
|
+
Partition.prototype.kind = 0;
|
|
15892
|
+
|
|
15893
|
+
/**
|
|
15894
|
+
* Partition expression.
|
|
15895
|
+
* @member {string} expression
|
|
15896
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15897
|
+
* @instance
|
|
15898
|
+
*/
|
|
15899
|
+
Partition.prototype.expression = "";
|
|
15900
|
+
|
|
15901
|
+
/**
|
|
15902
|
+
* Partition partitions.
|
|
15903
|
+
* @member {Array.<sqlanvil.MysqlOptions.Partition.IBound>} partitions
|
|
15904
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15905
|
+
* @instance
|
|
15906
|
+
*/
|
|
15907
|
+
Partition.prototype.partitions = $util.emptyArray;
|
|
15908
|
+
|
|
15909
|
+
/**
|
|
15910
|
+
* Partition count.
|
|
15911
|
+
* @member {number} count
|
|
15912
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15913
|
+
* @instance
|
|
15914
|
+
*/
|
|
15915
|
+
Partition.prototype.count = 0;
|
|
15916
|
+
|
|
15917
|
+
/**
|
|
15918
|
+
* Creates a new Partition instance using the specified properties.
|
|
15919
|
+
* @function create
|
|
15920
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15921
|
+
* @static
|
|
15922
|
+
* @param {sqlanvil.MysqlOptions.IPartition=} [properties] Properties to set
|
|
15923
|
+
* @returns {sqlanvil.MysqlOptions.Partition} Partition instance
|
|
15924
|
+
*/
|
|
15925
|
+
Partition.create = function create(properties) {
|
|
15926
|
+
return new Partition(properties);
|
|
15927
|
+
};
|
|
15928
|
+
|
|
15929
|
+
/**
|
|
15930
|
+
* Encodes the specified Partition message. Does not implicitly {@link sqlanvil.MysqlOptions.Partition.verify|verify} messages.
|
|
15931
|
+
* @function encode
|
|
15932
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15933
|
+
* @static
|
|
15934
|
+
* @param {sqlanvil.MysqlOptions.IPartition} message Partition message or plain object to encode
|
|
15935
|
+
* @param {$protobuf.Writer} [writer] Writer to encode to
|
|
15936
|
+
* @returns {$protobuf.Writer} Writer
|
|
15937
|
+
*/
|
|
15938
|
+
Partition.encode = function encode(message, writer) {
|
|
15939
|
+
if (!writer)
|
|
15940
|
+
writer = $Writer.create();
|
|
15941
|
+
if (message.kind != null && Object.hasOwnProperty.call(message, "kind"))
|
|
15942
|
+
writer.uint32(/* id 1, wireType 0 =*/8).int32(message.kind);
|
|
15943
|
+
if (message.expression != null && Object.hasOwnProperty.call(message, "expression"))
|
|
15944
|
+
writer.uint32(/* id 2, wireType 2 =*/18).string(message.expression);
|
|
15945
|
+
if (message.partitions != null && message.partitions.length)
|
|
15946
|
+
for (let i = 0; i < message.partitions.length; ++i)
|
|
15947
|
+
$root.sqlanvil.MysqlOptions.Partition.Bound.encode(message.partitions[i], writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim();
|
|
15948
|
+
if (message.count != null && Object.hasOwnProperty.call(message, "count"))
|
|
15949
|
+
writer.uint32(/* id 4, wireType 0 =*/32).uint32(message.count);
|
|
15950
|
+
return writer;
|
|
15951
|
+
};
|
|
15952
|
+
|
|
15953
|
+
/**
|
|
15954
|
+
* Encodes the specified Partition message, length delimited. Does not implicitly {@link sqlanvil.MysqlOptions.Partition.verify|verify} messages.
|
|
15955
|
+
* @function encodeDelimited
|
|
15956
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15957
|
+
* @static
|
|
15958
|
+
* @param {sqlanvil.MysqlOptions.IPartition} message Partition message or plain object to encode
|
|
15959
|
+
* @param {$protobuf.Writer} [writer] Writer to encode to
|
|
15960
|
+
* @returns {$protobuf.Writer} Writer
|
|
15961
|
+
*/
|
|
15962
|
+
Partition.encodeDelimited = function encodeDelimited(message, writer) {
|
|
15963
|
+
return this.encode(message, writer).ldelim();
|
|
15964
|
+
};
|
|
15965
|
+
|
|
15966
|
+
/**
|
|
15967
|
+
* Decodes a Partition message from the specified reader or buffer.
|
|
15968
|
+
* @function decode
|
|
15969
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
15970
|
+
* @static
|
|
15971
|
+
* @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
|
|
15972
|
+
* @param {number} [length] Message length if known beforehand
|
|
15973
|
+
* @returns {sqlanvil.MysqlOptions.Partition} Partition
|
|
15974
|
+
* @throws {Error} If the payload is not a reader or valid buffer
|
|
15975
|
+
* @throws {$protobuf.util.ProtocolError} If required fields are missing
|
|
15976
|
+
*/
|
|
15977
|
+
Partition.decode = function decode(reader, length, error, long) {
|
|
15978
|
+
if (!(reader instanceof $Reader))
|
|
15979
|
+
reader = $Reader.create(reader);
|
|
15980
|
+
if (long === undefined)
|
|
15981
|
+
long = 0;
|
|
15982
|
+
if (long > $Reader.recursionLimit)
|
|
15983
|
+
throw Error("maximum nesting depth exceeded");
|
|
15984
|
+
let end = length === undefined ? reader.len : reader.pos + length, message = new $root.sqlanvil.MysqlOptions.Partition();
|
|
15985
|
+
while (reader.pos < end) {
|
|
15986
|
+
let tag = reader.uint32();
|
|
15987
|
+
if (tag === error)
|
|
15988
|
+
break;
|
|
15989
|
+
switch (tag >>> 3) {
|
|
15990
|
+
case 1: {
|
|
15991
|
+
message.kind = reader.int32();
|
|
15992
|
+
break;
|
|
15993
|
+
}
|
|
15994
|
+
case 2: {
|
|
15995
|
+
message.expression = reader.string();
|
|
15996
|
+
break;
|
|
15997
|
+
}
|
|
15998
|
+
case 3: {
|
|
15999
|
+
if (!(message.partitions && message.partitions.length))
|
|
16000
|
+
message.partitions = [];
|
|
16001
|
+
message.partitions.push($root.sqlanvil.MysqlOptions.Partition.Bound.decode(reader, reader.uint32(), undefined, long + 1));
|
|
16002
|
+
break;
|
|
16003
|
+
}
|
|
16004
|
+
case 4: {
|
|
16005
|
+
message.count = reader.uint32();
|
|
16006
|
+
break;
|
|
16007
|
+
}
|
|
16008
|
+
default:
|
|
16009
|
+
reader.skipType(tag & 7, long);
|
|
16010
|
+
break;
|
|
16011
|
+
}
|
|
16012
|
+
}
|
|
16013
|
+
return message;
|
|
16014
|
+
};
|
|
16015
|
+
|
|
16016
|
+
/**
|
|
16017
|
+
* Decodes a Partition message from the specified reader or buffer, length delimited.
|
|
16018
|
+
* @function decodeDelimited
|
|
16019
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16020
|
+
* @static
|
|
16021
|
+
* @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
|
|
16022
|
+
* @returns {sqlanvil.MysqlOptions.Partition} Partition
|
|
16023
|
+
* @throws {Error} If the payload is not a reader or valid buffer
|
|
16024
|
+
* @throws {$protobuf.util.ProtocolError} If required fields are missing
|
|
16025
|
+
*/
|
|
16026
|
+
Partition.decodeDelimited = function decodeDelimited(reader) {
|
|
16027
|
+
if (!(reader instanceof $Reader))
|
|
16028
|
+
reader = new $Reader(reader);
|
|
16029
|
+
return this.decode(reader, reader.uint32());
|
|
16030
|
+
};
|
|
16031
|
+
|
|
16032
|
+
/**
|
|
16033
|
+
* Verifies a Partition message.
|
|
16034
|
+
* @function verify
|
|
16035
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16036
|
+
* @static
|
|
16037
|
+
* @param {Object.<string,*>} message Plain object to verify
|
|
16038
|
+
* @returns {string|null} `null` if valid, otherwise the reason why it is not
|
|
16039
|
+
*/
|
|
16040
|
+
Partition.verify = function verify(message, long) {
|
|
16041
|
+
if (typeof message !== "object" || message === null)
|
|
16042
|
+
return "object expected";
|
|
16043
|
+
if (long === undefined)
|
|
16044
|
+
long = 0;
|
|
16045
|
+
if (long > $util.recursionLimit)
|
|
16046
|
+
return "maximum nesting depth exceeded";
|
|
16047
|
+
if (message.kind != null && message.hasOwnProperty("kind"))
|
|
16048
|
+
switch (message.kind) {
|
|
16049
|
+
default:
|
|
16050
|
+
return "kind: enum value expected";
|
|
16051
|
+
case 0:
|
|
16052
|
+
case 1:
|
|
16053
|
+
case 2:
|
|
16054
|
+
case 3:
|
|
16055
|
+
break;
|
|
16056
|
+
}
|
|
16057
|
+
if (message.expression != null && message.hasOwnProperty("expression"))
|
|
16058
|
+
if (!$util.isString(message.expression))
|
|
16059
|
+
return "expression: string expected";
|
|
16060
|
+
if (message.partitions != null && message.hasOwnProperty("partitions")) {
|
|
16061
|
+
if (!Array.isArray(message.partitions))
|
|
16062
|
+
return "partitions: array expected";
|
|
16063
|
+
for (let i = 0; i < message.partitions.length; ++i) {
|
|
16064
|
+
let error = $root.sqlanvil.MysqlOptions.Partition.Bound.verify(message.partitions[i], long + 1);
|
|
16065
|
+
if (error)
|
|
16066
|
+
return "partitions." + error;
|
|
16067
|
+
}
|
|
16068
|
+
}
|
|
16069
|
+
if (message.count != null && message.hasOwnProperty("count"))
|
|
16070
|
+
if (!$util.isInteger(message.count))
|
|
16071
|
+
return "count: integer expected";
|
|
16072
|
+
return null;
|
|
16073
|
+
};
|
|
16074
|
+
|
|
16075
|
+
/**
|
|
16076
|
+
* Creates a Partition message from a plain object. Also converts values to their respective internal types.
|
|
16077
|
+
* @function fromObject
|
|
16078
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16079
|
+
* @static
|
|
16080
|
+
* @param {Object.<string,*>} object Plain object
|
|
16081
|
+
* @returns {sqlanvil.MysqlOptions.Partition} Partition
|
|
16082
|
+
*/
|
|
16083
|
+
Partition.fromObject = function fromObject(object, long) {
|
|
16084
|
+
if (object instanceof $root.sqlanvil.MysqlOptions.Partition)
|
|
16085
|
+
return object;
|
|
16086
|
+
if (long === undefined)
|
|
16087
|
+
long = 0;
|
|
16088
|
+
if (long > $util.recursionLimit)
|
|
16089
|
+
throw Error("maximum nesting depth exceeded");
|
|
16090
|
+
let message = new $root.sqlanvil.MysqlOptions.Partition();
|
|
16091
|
+
switch (object.kind) {
|
|
16092
|
+
default:
|
|
16093
|
+
if (typeof object.kind === "number") {
|
|
16094
|
+
message.kind = object.kind;
|
|
16095
|
+
break;
|
|
16096
|
+
}
|
|
16097
|
+
break;
|
|
16098
|
+
case "RANGE":
|
|
16099
|
+
case 0:
|
|
16100
|
+
message.kind = 0;
|
|
16101
|
+
break;
|
|
16102
|
+
case "LIST":
|
|
16103
|
+
case 1:
|
|
16104
|
+
message.kind = 1;
|
|
16105
|
+
break;
|
|
16106
|
+
case "HASH":
|
|
16107
|
+
case 2:
|
|
16108
|
+
message.kind = 2;
|
|
16109
|
+
break;
|
|
16110
|
+
case "KEY":
|
|
16111
|
+
case 3:
|
|
16112
|
+
message.kind = 3;
|
|
16113
|
+
break;
|
|
16114
|
+
}
|
|
16115
|
+
if (object.expression != null)
|
|
16116
|
+
message.expression = String(object.expression);
|
|
16117
|
+
if (object.partitions) {
|
|
16118
|
+
if (!Array.isArray(object.partitions))
|
|
16119
|
+
throw TypeError(".sqlanvil.MysqlOptions.Partition.partitions: array expected");
|
|
16120
|
+
message.partitions = [];
|
|
16121
|
+
for (let i = 0; i < object.partitions.length; ++i) {
|
|
16122
|
+
if (typeof object.partitions[i] !== "object")
|
|
16123
|
+
throw TypeError(".sqlanvil.MysqlOptions.Partition.partitions: object expected");
|
|
16124
|
+
message.partitions[i] = $root.sqlanvil.MysqlOptions.Partition.Bound.fromObject(object.partitions[i], long + 1);
|
|
16125
|
+
}
|
|
16126
|
+
}
|
|
16127
|
+
if (object.count != null)
|
|
16128
|
+
message.count = object.count >>> 0;
|
|
16129
|
+
return message;
|
|
16130
|
+
};
|
|
16131
|
+
|
|
16132
|
+
/**
|
|
16133
|
+
* Creates a plain object from a Partition message. Also converts values to other types if specified.
|
|
16134
|
+
* @function toObject
|
|
16135
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16136
|
+
* @static
|
|
16137
|
+
* @param {sqlanvil.MysqlOptions.Partition} message Partition
|
|
16138
|
+
* @param {$protobuf.IConversionOptions} [options] Conversion options
|
|
16139
|
+
* @returns {Object.<string,*>} Plain object
|
|
16140
|
+
*/
|
|
16141
|
+
Partition.toObject = function toObject(message, options) {
|
|
16142
|
+
if (!options)
|
|
16143
|
+
options = {};
|
|
16144
|
+
let object = {};
|
|
16145
|
+
if (options.arrays || options.defaults)
|
|
16146
|
+
object.partitions = [];
|
|
16147
|
+
if (options.defaults) {
|
|
16148
|
+
object.kind = options.enums === String ? "RANGE" : 0;
|
|
16149
|
+
object.expression = "";
|
|
16150
|
+
object.count = 0;
|
|
16151
|
+
}
|
|
16152
|
+
if (message.kind != null && message.hasOwnProperty("kind"))
|
|
16153
|
+
object.kind = options.enums === String ? $root.sqlanvil.MysqlOptions.Partition.Kind[message.kind] === undefined ? message.kind : $root.sqlanvil.MysqlOptions.Partition.Kind[message.kind] : message.kind;
|
|
16154
|
+
if (message.expression != null && message.hasOwnProperty("expression"))
|
|
16155
|
+
object.expression = message.expression;
|
|
16156
|
+
if (message.partitions && message.partitions.length) {
|
|
16157
|
+
object.partitions = [];
|
|
16158
|
+
for (let j = 0; j < message.partitions.length; ++j)
|
|
16159
|
+
object.partitions[j] = $root.sqlanvil.MysqlOptions.Partition.Bound.toObject(message.partitions[j], options);
|
|
16160
|
+
}
|
|
16161
|
+
if (message.count != null && message.hasOwnProperty("count"))
|
|
16162
|
+
object.count = message.count;
|
|
16163
|
+
return object;
|
|
16164
|
+
};
|
|
16165
|
+
|
|
16166
|
+
/**
|
|
16167
|
+
* Converts this Partition to JSON.
|
|
16168
|
+
* @function toJSON
|
|
16169
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16170
|
+
* @instance
|
|
16171
|
+
* @returns {Object.<string,*>} JSON object
|
|
16172
|
+
*/
|
|
16173
|
+
Partition.prototype.toJSON = function toJSON() {
|
|
16174
|
+
return this.constructor.toObject(this, $protobuf__namespace.util.toJSONOptions);
|
|
16175
|
+
};
|
|
16176
|
+
|
|
16177
|
+
/**
|
|
16178
|
+
* Gets the default type url for Partition
|
|
16179
|
+
* @function getTypeUrl
|
|
16180
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16181
|
+
* @static
|
|
16182
|
+
* @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com")
|
|
16183
|
+
* @returns {string} The default type url
|
|
16184
|
+
*/
|
|
16185
|
+
Partition.getTypeUrl = function getTypeUrl(typeUrlPrefix) {
|
|
16186
|
+
if (typeUrlPrefix === undefined) {
|
|
16187
|
+
typeUrlPrefix = "type.googleapis.com";
|
|
16188
|
+
}
|
|
16189
|
+
return typeUrlPrefix + "/sqlanvil.MysqlOptions.Partition";
|
|
16190
|
+
};
|
|
16191
|
+
|
|
16192
|
+
/**
|
|
16193
|
+
* Kind enum.
|
|
16194
|
+
* @name sqlanvil.MysqlOptions.Partition.Kind
|
|
16195
|
+
* @enum {number}
|
|
16196
|
+
* @property {number} RANGE=0 RANGE value
|
|
16197
|
+
* @property {number} LIST=1 LIST value
|
|
16198
|
+
* @property {number} HASH=2 HASH value
|
|
16199
|
+
* @property {number} KEY=3 KEY value
|
|
16200
|
+
*/
|
|
16201
|
+
Partition.Kind = (function() {
|
|
16202
|
+
const valuesById = {}, values = Object.create(valuesById);
|
|
16203
|
+
values[valuesById[0] = "RANGE"] = 0;
|
|
16204
|
+
values[valuesById[1] = "LIST"] = 1;
|
|
16205
|
+
values[valuesById[2] = "HASH"] = 2;
|
|
16206
|
+
values[valuesById[3] = "KEY"] = 3;
|
|
16207
|
+
return values;
|
|
16208
|
+
})();
|
|
16209
|
+
|
|
16210
|
+
Partition.Bound = (function() {
|
|
16211
|
+
|
|
16212
|
+
/**
|
|
16213
|
+
* Properties of a Bound.
|
|
16214
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16215
|
+
* @interface IBound
|
|
16216
|
+
* @property {string|null} [name] Bound name
|
|
16217
|
+
* @property {string|null} [values] Bound values
|
|
16218
|
+
*/
|
|
16219
|
+
|
|
16220
|
+
/**
|
|
16221
|
+
* Constructs a new Bound.
|
|
16222
|
+
* @memberof sqlanvil.MysqlOptions.Partition
|
|
16223
|
+
* @classdesc Represents a Bound.
|
|
16224
|
+
* @implements IBound
|
|
16225
|
+
* @constructor
|
|
16226
|
+
* @param {sqlanvil.MysqlOptions.Partition.IBound=} [properties] Properties to set
|
|
16227
|
+
*/
|
|
16228
|
+
function Bound(properties) {
|
|
16229
|
+
if (properties)
|
|
16230
|
+
for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
|
|
16231
|
+
if (properties[keys[i]] != null && keys[i] !== "__proto__")
|
|
16232
|
+
this[keys[i]] = properties[keys[i]];
|
|
16233
|
+
}
|
|
16234
|
+
|
|
16235
|
+
/**
|
|
16236
|
+
* Bound name.
|
|
16237
|
+
* @member {string} name
|
|
16238
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16239
|
+
* @instance
|
|
16240
|
+
*/
|
|
16241
|
+
Bound.prototype.name = "";
|
|
16242
|
+
|
|
16243
|
+
/**
|
|
16244
|
+
* Bound values.
|
|
16245
|
+
* @member {string} values
|
|
16246
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16247
|
+
* @instance
|
|
16248
|
+
*/
|
|
16249
|
+
Bound.prototype.values = "";
|
|
16250
|
+
|
|
16251
|
+
/**
|
|
16252
|
+
* Creates a new Bound instance using the specified properties.
|
|
16253
|
+
* @function create
|
|
16254
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16255
|
+
* @static
|
|
16256
|
+
* @param {sqlanvil.MysqlOptions.Partition.IBound=} [properties] Properties to set
|
|
16257
|
+
* @returns {sqlanvil.MysqlOptions.Partition.Bound} Bound instance
|
|
16258
|
+
*/
|
|
16259
|
+
Bound.create = function create(properties) {
|
|
16260
|
+
return new Bound(properties);
|
|
16261
|
+
};
|
|
16262
|
+
|
|
16263
|
+
/**
|
|
16264
|
+
* Encodes the specified Bound message. Does not implicitly {@link sqlanvil.MysqlOptions.Partition.Bound.verify|verify} messages.
|
|
16265
|
+
* @function encode
|
|
16266
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16267
|
+
* @static
|
|
16268
|
+
* @param {sqlanvil.MysqlOptions.Partition.IBound} message Bound message or plain object to encode
|
|
16269
|
+
* @param {$protobuf.Writer} [writer] Writer to encode to
|
|
16270
|
+
* @returns {$protobuf.Writer} Writer
|
|
16271
|
+
*/
|
|
16272
|
+
Bound.encode = function encode(message, writer) {
|
|
16273
|
+
if (!writer)
|
|
16274
|
+
writer = $Writer.create();
|
|
16275
|
+
if (message.name != null && Object.hasOwnProperty.call(message, "name"))
|
|
16276
|
+
writer.uint32(/* id 1, wireType 2 =*/10).string(message.name);
|
|
16277
|
+
if (message.values != null && Object.hasOwnProperty.call(message, "values"))
|
|
16278
|
+
writer.uint32(/* id 2, wireType 2 =*/18).string(message.values);
|
|
16279
|
+
return writer;
|
|
16280
|
+
};
|
|
16281
|
+
|
|
16282
|
+
/**
|
|
16283
|
+
* Encodes the specified Bound message, length delimited. Does not implicitly {@link sqlanvil.MysqlOptions.Partition.Bound.verify|verify} messages.
|
|
16284
|
+
* @function encodeDelimited
|
|
16285
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16286
|
+
* @static
|
|
16287
|
+
* @param {sqlanvil.MysqlOptions.Partition.IBound} message Bound message or plain object to encode
|
|
16288
|
+
* @param {$protobuf.Writer} [writer] Writer to encode to
|
|
16289
|
+
* @returns {$protobuf.Writer} Writer
|
|
16290
|
+
*/
|
|
16291
|
+
Bound.encodeDelimited = function encodeDelimited(message, writer) {
|
|
16292
|
+
return this.encode(message, writer).ldelim();
|
|
16293
|
+
};
|
|
16294
|
+
|
|
16295
|
+
/**
|
|
16296
|
+
* Decodes a Bound message from the specified reader or buffer.
|
|
16297
|
+
* @function decode
|
|
16298
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16299
|
+
* @static
|
|
16300
|
+
* @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
|
|
16301
|
+
* @param {number} [length] Message length if known beforehand
|
|
16302
|
+
* @returns {sqlanvil.MysqlOptions.Partition.Bound} Bound
|
|
16303
|
+
* @throws {Error} If the payload is not a reader or valid buffer
|
|
16304
|
+
* @throws {$protobuf.util.ProtocolError} If required fields are missing
|
|
16305
|
+
*/
|
|
16306
|
+
Bound.decode = function decode(reader, length, error, long) {
|
|
16307
|
+
if (!(reader instanceof $Reader))
|
|
16308
|
+
reader = $Reader.create(reader);
|
|
16309
|
+
if (long === undefined)
|
|
16310
|
+
long = 0;
|
|
16311
|
+
if (long > $Reader.recursionLimit)
|
|
16312
|
+
throw Error("maximum nesting depth exceeded");
|
|
16313
|
+
let end = length === undefined ? reader.len : reader.pos + length, message = new $root.sqlanvil.MysqlOptions.Partition.Bound();
|
|
16314
|
+
while (reader.pos < end) {
|
|
16315
|
+
let tag = reader.uint32();
|
|
16316
|
+
if (tag === error)
|
|
16317
|
+
break;
|
|
16318
|
+
switch (tag >>> 3) {
|
|
16319
|
+
case 1: {
|
|
16320
|
+
message.name = reader.string();
|
|
16321
|
+
break;
|
|
16322
|
+
}
|
|
16323
|
+
case 2: {
|
|
16324
|
+
message.values = reader.string();
|
|
16325
|
+
break;
|
|
16326
|
+
}
|
|
16327
|
+
default:
|
|
16328
|
+
reader.skipType(tag & 7, long);
|
|
16329
|
+
break;
|
|
16330
|
+
}
|
|
16331
|
+
}
|
|
16332
|
+
return message;
|
|
16333
|
+
};
|
|
16334
|
+
|
|
16335
|
+
/**
|
|
16336
|
+
* Decodes a Bound message from the specified reader or buffer, length delimited.
|
|
16337
|
+
* @function decodeDelimited
|
|
16338
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16339
|
+
* @static
|
|
16340
|
+
* @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
|
|
16341
|
+
* @returns {sqlanvil.MysqlOptions.Partition.Bound} Bound
|
|
16342
|
+
* @throws {Error} If the payload is not a reader or valid buffer
|
|
16343
|
+
* @throws {$protobuf.util.ProtocolError} If required fields are missing
|
|
16344
|
+
*/
|
|
16345
|
+
Bound.decodeDelimited = function decodeDelimited(reader) {
|
|
16346
|
+
if (!(reader instanceof $Reader))
|
|
16347
|
+
reader = new $Reader(reader);
|
|
16348
|
+
return this.decode(reader, reader.uint32());
|
|
16349
|
+
};
|
|
16350
|
+
|
|
16351
|
+
/**
|
|
16352
|
+
* Verifies a Bound message.
|
|
16353
|
+
* @function verify
|
|
16354
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16355
|
+
* @static
|
|
16356
|
+
* @param {Object.<string,*>} message Plain object to verify
|
|
16357
|
+
* @returns {string|null} `null` if valid, otherwise the reason why it is not
|
|
16358
|
+
*/
|
|
16359
|
+
Bound.verify = function verify(message, long) {
|
|
16360
|
+
if (typeof message !== "object" || message === null)
|
|
16361
|
+
return "object expected";
|
|
16362
|
+
if (long === undefined)
|
|
16363
|
+
long = 0;
|
|
16364
|
+
if (long > $util.recursionLimit)
|
|
16365
|
+
return "maximum nesting depth exceeded";
|
|
16366
|
+
if (message.name != null && message.hasOwnProperty("name"))
|
|
16367
|
+
if (!$util.isString(message.name))
|
|
16368
|
+
return "name: string expected";
|
|
16369
|
+
if (message.values != null && message.hasOwnProperty("values"))
|
|
16370
|
+
if (!$util.isString(message.values))
|
|
16371
|
+
return "values: string expected";
|
|
16372
|
+
return null;
|
|
16373
|
+
};
|
|
16374
|
+
|
|
16375
|
+
/**
|
|
16376
|
+
* Creates a Bound message from a plain object. Also converts values to their respective internal types.
|
|
16377
|
+
* @function fromObject
|
|
16378
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16379
|
+
* @static
|
|
16380
|
+
* @param {Object.<string,*>} object Plain object
|
|
16381
|
+
* @returns {sqlanvil.MysqlOptions.Partition.Bound} Bound
|
|
16382
|
+
*/
|
|
16383
|
+
Bound.fromObject = function fromObject(object, long) {
|
|
16384
|
+
if (object instanceof $root.sqlanvil.MysqlOptions.Partition.Bound)
|
|
16385
|
+
return object;
|
|
16386
|
+
if (long === undefined)
|
|
16387
|
+
long = 0;
|
|
16388
|
+
if (long > $util.recursionLimit)
|
|
16389
|
+
throw Error("maximum nesting depth exceeded");
|
|
16390
|
+
let message = new $root.sqlanvil.MysqlOptions.Partition.Bound();
|
|
16391
|
+
if (object.name != null)
|
|
16392
|
+
message.name = String(object.name);
|
|
16393
|
+
if (object.values != null)
|
|
16394
|
+
message.values = String(object.values);
|
|
16395
|
+
return message;
|
|
16396
|
+
};
|
|
16397
|
+
|
|
16398
|
+
/**
|
|
16399
|
+
* Creates a plain object from a Bound message. Also converts values to other types if specified.
|
|
16400
|
+
* @function toObject
|
|
16401
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16402
|
+
* @static
|
|
16403
|
+
* @param {sqlanvil.MysqlOptions.Partition.Bound} message Bound
|
|
16404
|
+
* @param {$protobuf.IConversionOptions} [options] Conversion options
|
|
16405
|
+
* @returns {Object.<string,*>} Plain object
|
|
16406
|
+
*/
|
|
16407
|
+
Bound.toObject = function toObject(message, options) {
|
|
16408
|
+
if (!options)
|
|
16409
|
+
options = {};
|
|
16410
|
+
let object = {};
|
|
16411
|
+
if (options.defaults) {
|
|
16412
|
+
object.name = "";
|
|
16413
|
+
object.values = "";
|
|
16414
|
+
}
|
|
16415
|
+
if (message.name != null && message.hasOwnProperty("name"))
|
|
16416
|
+
object.name = message.name;
|
|
16417
|
+
if (message.values != null && message.hasOwnProperty("values"))
|
|
16418
|
+
object.values = message.values;
|
|
16419
|
+
return object;
|
|
16420
|
+
};
|
|
16421
|
+
|
|
16422
|
+
/**
|
|
16423
|
+
* Converts this Bound to JSON.
|
|
16424
|
+
* @function toJSON
|
|
16425
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16426
|
+
* @instance
|
|
16427
|
+
* @returns {Object.<string,*>} JSON object
|
|
16428
|
+
*/
|
|
16429
|
+
Bound.prototype.toJSON = function toJSON() {
|
|
16430
|
+
return this.constructor.toObject(this, $protobuf__namespace.util.toJSONOptions);
|
|
16431
|
+
};
|
|
16432
|
+
|
|
16433
|
+
/**
|
|
16434
|
+
* Gets the default type url for Bound
|
|
16435
|
+
* @function getTypeUrl
|
|
16436
|
+
* @memberof sqlanvil.MysqlOptions.Partition.Bound
|
|
16437
|
+
* @static
|
|
16438
|
+
* @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com")
|
|
16439
|
+
* @returns {string} The default type url
|
|
16440
|
+
*/
|
|
16441
|
+
Bound.getTypeUrl = function getTypeUrl(typeUrlPrefix) {
|
|
16442
|
+
if (typeUrlPrefix === undefined) {
|
|
16443
|
+
typeUrlPrefix = "type.googleapis.com";
|
|
16444
|
+
}
|
|
16445
|
+
return typeUrlPrefix + "/sqlanvil.MysqlOptions.Partition.Bound";
|
|
16446
|
+
};
|
|
16447
|
+
|
|
16448
|
+
return Bound;
|
|
16449
|
+
})();
|
|
16450
|
+
|
|
16451
|
+
return Partition;
|
|
16452
|
+
})();
|
|
16453
|
+
|
|
15829
16454
|
return MysqlOptions;
|
|
15830
16455
|
})();
|
|
15831
16456
|
|
|
@@ -43245,7 +43870,7 @@ class MysqlExecutionSql {
|
|
|
43245
43870
|
if (table.materialized) {
|
|
43246
43871
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.VIEW)));
|
|
43247
43872
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.TABLE)));
|
|
43248
|
-
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)} as ${table.query}`));
|
|
43873
|
+
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)}${this.partitionClause(table)} as ${table.query}`));
|
|
43249
43874
|
this.createIndexes(table).forEach(stmt => tasks.add(Task.statement(stmt)));
|
|
43250
43875
|
}
|
|
43251
43876
|
else {
|
|
@@ -43256,7 +43881,7 @@ class MysqlExecutionSql {
|
|
|
43256
43881
|
const fresh = !this.shouldWriteIncrementally(table, runConfig, tableMetadata);
|
|
43257
43882
|
if (fresh) {
|
|
43258
43883
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.TABLE)));
|
|
43259
|
-
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)} as ${table.query}`));
|
|
43884
|
+
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)}${this.partitionClause(table)} as ${table.query}`));
|
|
43260
43885
|
if (table.uniqueKey && table.uniqueKey.length > 0) {
|
|
43261
43886
|
const idx = `uq_${table.target.schema}_${table.target.name}`.slice(0, 63);
|
|
43262
43887
|
const cols = table.uniqueKey.map(k => `\`${k}\``).join(", ");
|
|
@@ -43270,7 +43895,7 @@ class MysqlExecutionSql {
|
|
|
43270
43895
|
}
|
|
43271
43896
|
else {
|
|
43272
43897
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.TABLE)));
|
|
43273
|
-
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)} as ${table.query}`));
|
|
43898
|
+
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)}${this.partitionClause(table)} as ${table.query}`));
|
|
43274
43899
|
this.createIndexes(table).forEach(stmt => tasks.add(Task.statement(stmt)));
|
|
43275
43900
|
}
|
|
43276
43901
|
this.postOps(table, runConfig, tableMetadata).forEach(task => tasks.add(task));
|
|
@@ -43328,6 +43953,34 @@ class MysqlExecutionSql {
|
|
|
43328
43953
|
const tail = updates.length > 0 ? ` on duplicate key update ${updates}` : "";
|
|
43329
43954
|
return `insert into ${target} (${backticked.join(", ")}) select ${backticked.join(", ")} from (${query}) as insertions${tail}`;
|
|
43330
43955
|
}
|
|
43956
|
+
partitionKindAsSql(kind) {
|
|
43957
|
+
switch (kind) {
|
|
43958
|
+
case sqlanvil.MysqlOptions.Partition.Kind.LIST:
|
|
43959
|
+
return "list";
|
|
43960
|
+
case sqlanvil.MysqlOptions.Partition.Kind.HASH:
|
|
43961
|
+
return "hash";
|
|
43962
|
+
case sqlanvil.MysqlOptions.Partition.Kind.KEY:
|
|
43963
|
+
return "key";
|
|
43964
|
+
case sqlanvil.MysqlOptions.Partition.Kind.RANGE:
|
|
43965
|
+
default:
|
|
43966
|
+
return "range";
|
|
43967
|
+
}
|
|
43968
|
+
}
|
|
43969
|
+
partitionClause(table) {
|
|
43970
|
+
const partition = table.mysql && table.mysql.partition;
|
|
43971
|
+
if (!partition) {
|
|
43972
|
+
return "";
|
|
43973
|
+
}
|
|
43974
|
+
const kind = this.partitionKindAsSql(partition.kind);
|
|
43975
|
+
const head = ` partition by ${kind} (${partition.expression})`;
|
|
43976
|
+
if (kind === "hash" || kind === "key") {
|
|
43977
|
+
return partition.count ? `${head} partitions ${partition.count}` : head;
|
|
43978
|
+
}
|
|
43979
|
+
const defs = (partition.partitions || [])
|
|
43980
|
+
.map(bound => `partition ${bound.name} ${bound.values}`)
|
|
43981
|
+
.join(", ");
|
|
43982
|
+
return `${head} (${defs})`;
|
|
43983
|
+
}
|
|
43331
43984
|
tableOptions(table) {
|
|
43332
43985
|
const opts = table.mysql;
|
|
43333
43986
|
if (!opts) {
|
|
@@ -43792,7 +44445,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
43792
44445
|
.filter(validationQuery => !!validationQuery.query);
|
|
43793
44446
|
}
|
|
43794
44447
|
|
|
43795
|
-
const version = "1.
|
|
44448
|
+
const version = "1.11.0";
|
|
43796
44449
|
const dataformVersion = "3.0.60";
|
|
43797
44450
|
|
|
43798
44451
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -44291,6 +44944,9 @@ function mapBigQueryType(bqType) {
|
|
|
44291
44944
|
function mapPostgresType(pgType) {
|
|
44292
44945
|
return pgType.trim().toLowerCase();
|
|
44293
44946
|
}
|
|
44947
|
+
function mapMysqlType(mysqlType) {
|
|
44948
|
+
return mysqlType.trim().toLowerCase();
|
|
44949
|
+
}
|
|
44294
44950
|
function keyToken(name) {
|
|
44295
44951
|
return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? name : `"${name}"`;
|
|
44296
44952
|
}
|
|
@@ -44392,6 +45048,40 @@ const readBigQuerySchema = function (resolved, sourceSchema, table) {
|
|
|
44392
45048
|
});
|
|
44393
45049
|
});
|
|
44394
45050
|
};
|
|
45051
|
+
const readMysqlSchema = function (resolved, sourceSchema, table) {
|
|
45052
|
+
const c = resolved.credentials;
|
|
45053
|
+
let conn;
|
|
45054
|
+
return mysql__namespace
|
|
45055
|
+
.createConnection({
|
|
45056
|
+
host: c.host || resolved.definition.host,
|
|
45057
|
+
port: Number(c.port || resolved.definition.port || 3306),
|
|
45058
|
+
database: c.database || resolved.definition.database,
|
|
45059
|
+
user: c.user,
|
|
45060
|
+
password: c.password,
|
|
45061
|
+
ssl: c.sslMode && c.sslMode !== "disable" ? { rejectUnauthorized: false } : undefined
|
|
45062
|
+
})
|
|
45063
|
+
.then(function (connection) {
|
|
45064
|
+
conn = connection;
|
|
45065
|
+
return conn.query(`select column_name, data_type, column_comment
|
|
45066
|
+
from information_schema.columns
|
|
45067
|
+
where table_schema = ? and table_name = ?
|
|
45068
|
+
order by ordinal_position`, [sourceSchema, table]);
|
|
45069
|
+
})
|
|
45070
|
+
.then(function (result) {
|
|
45071
|
+
const rows = result[0] || [];
|
|
45072
|
+
return rows.map(function (r) {
|
|
45073
|
+
const name = r.column_name !== undefined ? r.column_name : r.COLUMN_NAME;
|
|
45074
|
+
const type = r.data_type !== undefined ? r.data_type : r.DATA_TYPE;
|
|
45075
|
+
const comment = r.column_comment !== undefined ? r.column_comment : r.COLUMN_COMMENT;
|
|
45076
|
+
return { name, type, description: comment || undefined };
|
|
45077
|
+
});
|
|
45078
|
+
})
|
|
45079
|
+
.then(function (cols) {
|
|
45080
|
+
return conn.end().then(function () { return cols; });
|
|
45081
|
+
}, function (err) {
|
|
45082
|
+
return (conn ? conn.end() : Promise.resolve()).then(function () { throw err; }, function () { throw err; });
|
|
45083
|
+
});
|
|
45084
|
+
};
|
|
44395
45085
|
function defaultReaderFor(platform) {
|
|
44396
45086
|
if (platform === "bigquery") {
|
|
44397
45087
|
return readBigQuerySchema;
|
|
@@ -44399,6 +45089,9 @@ function defaultReaderFor(platform) {
|
|
|
44399
45089
|
if (platform === "postgres" || platform === "supabase") {
|
|
44400
45090
|
return readPostgresSchema;
|
|
44401
45091
|
}
|
|
45092
|
+
if (platform === "mysql") {
|
|
45093
|
+
return readMysqlSchema;
|
|
45094
|
+
}
|
|
44402
45095
|
throw new Error(`introspect does not support source platform "${platform}".`);
|
|
44403
45096
|
}
|
|
44404
45097
|
function splitTableRef(tableRef) {
|
|
@@ -44417,7 +45110,11 @@ function introspectToSqlx(projectDir, connectionName, tableRef, options) {
|
|
|
44417
45110
|
return Promise.reject(new Error(`Could not determine the source schema for "${tableRef}" on connection "${connectionName}". ` +
|
|
44418
45111
|
`Pass it as "schema.table", or set "dataset"/"defaultSchema" on the connection.`));
|
|
44419
45112
|
}
|
|
44420
|
-
const mapType = resolved.definition.platform === "bigquery"
|
|
45113
|
+
const mapType = resolved.definition.platform === "bigquery"
|
|
45114
|
+
? mapBigQueryType
|
|
45115
|
+
: resolved.definition.platform === "mysql"
|
|
45116
|
+
? mapMysqlType
|
|
45117
|
+
: mapPostgresType;
|
|
44421
45118
|
return reader(resolved, sourceSchema, parts.table).then(function (rawColumns) {
|
|
44422
45119
|
if (rawColumns.length === 0) {
|
|
44423
45120
|
throw new Error(`Source table "${tableRef}" on connection "${connectionName}" has no columns (does it exist?).`);
|
|
@@ -44459,6 +45156,53 @@ function assertConnectionCredentialsAvailable(graph, connections) {
|
|
|
44459
45156
|
}));
|
|
44460
45157
|
}
|
|
44461
45158
|
|
|
45159
|
+
function loadDuckdb() {
|
|
45160
|
+
try {
|
|
45161
|
+
return nativeRequire("@duckdb/node-api");
|
|
45162
|
+
}
|
|
45163
|
+
catch (e) {
|
|
45164
|
+
throw new Error(`This feature requires the optional "@duckdb/node-api" dependency, which failed to load: ` +
|
|
45165
|
+
`${e.message}`);
|
|
45166
|
+
}
|
|
45167
|
+
}
|
|
45168
|
+
async function runAsync(conn, sql) {
|
|
45169
|
+
await conn.run(sql);
|
|
45170
|
+
}
|
|
45171
|
+
async function allAsync(conn, sql) {
|
|
45172
|
+
const reader = await conn.runAndReadAll(sql);
|
|
45173
|
+
return reader.getRowObjects();
|
|
45174
|
+
}
|
|
45175
|
+
async function withDuckdb(fn, dbPath = ":memory:") {
|
|
45176
|
+
const { DuckDBInstance } = loadDuckdb();
|
|
45177
|
+
const instance = await DuckDBInstance.create(dbPath);
|
|
45178
|
+
const conn = await instance.connect();
|
|
45179
|
+
const done = () => {
|
|
45180
|
+
var _a, _b;
|
|
45181
|
+
try {
|
|
45182
|
+
(_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
|
|
45183
|
+
}
|
|
45184
|
+
catch (e) {
|
|
45185
|
+
}
|
|
45186
|
+
try {
|
|
45187
|
+
(_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
|
|
45188
|
+
}
|
|
45189
|
+
catch (e) {
|
|
45190
|
+
}
|
|
45191
|
+
};
|
|
45192
|
+
try {
|
|
45193
|
+
if (!process.env.HOME) {
|
|
45194
|
+
await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
|
|
45195
|
+
}
|
|
45196
|
+
const result = await fn(conn);
|
|
45197
|
+
done();
|
|
45198
|
+
return result;
|
|
45199
|
+
}
|
|
45200
|
+
catch (e) {
|
|
45201
|
+
done();
|
|
45202
|
+
throw e;
|
|
45203
|
+
}
|
|
45204
|
+
}
|
|
45205
|
+
|
|
44462
45206
|
const PG_ATTACH_ALIAS = "pg";
|
|
44463
45207
|
const SECRET_NAME = "sa_export";
|
|
44464
45208
|
function buildAttachSql(pg) {
|
|
@@ -44522,48 +45266,6 @@ function buildCopySql(selectSql, uri, format, options = {}) {
|
|
|
44522
45266
|
const optionList = [`FORMAT ${fmt}`, ...extraOptions].join(", ");
|
|
44523
45267
|
return `COPY (SELECT * FROM postgres_query('${PG_ATTACH_ALIAS}', $sa$${selectSql}$sa$)) TO '${target}' (${optionList})`;
|
|
44524
45268
|
}
|
|
44525
|
-
function loadDuckdb() {
|
|
44526
|
-
try {
|
|
44527
|
-
return nativeRequire("@duckdb/node-api");
|
|
44528
|
-
}
|
|
44529
|
-
catch (e) {
|
|
44530
|
-
throw new Error(`Exporting on Postgres/Supabase requires the optional "@duckdb/node-api" dependency, which ` +
|
|
44531
|
-
`failed to load: ${e.message}`);
|
|
44532
|
-
}
|
|
44533
|
-
}
|
|
44534
|
-
async function runAsync(conn, sql) {
|
|
44535
|
-
await conn.run(sql);
|
|
44536
|
-
}
|
|
44537
|
-
async function withConnection(fn) {
|
|
44538
|
-
const { DuckDBInstance } = loadDuckdb();
|
|
44539
|
-
const instance = await DuckDBInstance.create(":memory:");
|
|
44540
|
-
const conn = await instance.connect();
|
|
44541
|
-
const done = () => {
|
|
44542
|
-
var _a, _b;
|
|
44543
|
-
try {
|
|
44544
|
-
(_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
|
|
44545
|
-
}
|
|
44546
|
-
catch (e) {
|
|
44547
|
-
}
|
|
44548
|
-
try {
|
|
44549
|
-
(_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
|
|
44550
|
-
}
|
|
44551
|
-
catch (e) {
|
|
44552
|
-
}
|
|
44553
|
-
};
|
|
44554
|
-
try {
|
|
44555
|
-
if (!process.env.HOME) {
|
|
44556
|
-
await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
|
|
44557
|
-
}
|
|
44558
|
-
const result = await fn(conn);
|
|
44559
|
-
done();
|
|
44560
|
-
return result;
|
|
44561
|
-
}
|
|
44562
|
-
catch (e) {
|
|
44563
|
-
done();
|
|
44564
|
-
throw e;
|
|
44565
|
-
}
|
|
44566
|
-
}
|
|
44567
45269
|
async function runDuckdbExport(args) {
|
|
44568
45270
|
const { spec, selectSql, pg, storage, actionName } = args;
|
|
44569
45271
|
const uri = resolveExportUri(spec, actionName, { wildcard: false });
|
|
@@ -44572,7 +45274,7 @@ async function runDuckdbExport(args) {
|
|
|
44572
45274
|
throw new Error(`No "${scheme}" storage credentials found in .df-credentials.json (storage.${scheme}) for ` +
|
|
44573
45275
|
`export to ${uri}.`);
|
|
44574
45276
|
}
|
|
44575
|
-
return
|
|
45277
|
+
return withDuckdb(async (conn) => {
|
|
44576
45278
|
await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
|
|
44577
45279
|
await runAsync(conn, buildAttachSql(pg));
|
|
44578
45280
|
if (scheme !== "local") {
|
|
@@ -45204,6 +45906,343 @@ function resolveCredentials(envCredentials, cliCredentials, defaultFilename) {
|
|
|
45204
45906
|
return envCredentials || defaultFilename;
|
|
45205
45907
|
}
|
|
45206
45908
|
|
|
45909
|
+
function key(target) {
|
|
45910
|
+
return targetStringifier.stringify(target);
|
|
45911
|
+
}
|
|
45912
|
+
function enumName(enumObject, value) {
|
|
45913
|
+
if (value === undefined || value === null) {
|
|
45914
|
+
return "UNKNOWN";
|
|
45915
|
+
}
|
|
45916
|
+
const match = Object.keys(enumObject).find(k => enumObject[k] === value);
|
|
45917
|
+
return match || String(value);
|
|
45918
|
+
}
|
|
45919
|
+
function tableType$1(enumType) {
|
|
45920
|
+
switch (enumType) {
|
|
45921
|
+
case sqlanvil.TableType.VIEW:
|
|
45922
|
+
return "view";
|
|
45923
|
+
case sqlanvil.TableType.INCREMENTAL:
|
|
45924
|
+
return "incremental";
|
|
45925
|
+
default:
|
|
45926
|
+
return "table";
|
|
45927
|
+
}
|
|
45928
|
+
}
|
|
45929
|
+
function toMillis(value) {
|
|
45930
|
+
if (value === undefined || value === null) {
|
|
45931
|
+
return 0;
|
|
45932
|
+
}
|
|
45933
|
+
if (typeof value === "number") {
|
|
45934
|
+
return value;
|
|
45935
|
+
}
|
|
45936
|
+
if (typeof value.toNumber === "function") {
|
|
45937
|
+
return value.toNumber();
|
|
45938
|
+
}
|
|
45939
|
+
return Number(value) || 0;
|
|
45940
|
+
}
|
|
45941
|
+
function actionRow(action, type) {
|
|
45942
|
+
const descriptor = action.actionDescriptor || {};
|
|
45943
|
+
return {
|
|
45944
|
+
target_key: key(action.target),
|
|
45945
|
+
database: action.target.database || "",
|
|
45946
|
+
schema: action.target.schema || "",
|
|
45947
|
+
name: action.target.name || "",
|
|
45948
|
+
readable_name: targetAsReadableString(action.target),
|
|
45949
|
+
type,
|
|
45950
|
+
tags: JSON.stringify(action.tags || []),
|
|
45951
|
+
disabled: !!action.disabled,
|
|
45952
|
+
file_name: action.fileName || "",
|
|
45953
|
+
description: descriptor.description || ""
|
|
45954
|
+
};
|
|
45955
|
+
}
|
|
45956
|
+
function pushDeps(action, out) {
|
|
45957
|
+
for (const dep of action.dependencyTargets || []) {
|
|
45958
|
+
out.push({
|
|
45959
|
+
from_target_key: key(action.target),
|
|
45960
|
+
to_target_key: key(dep),
|
|
45961
|
+
from_readable: targetAsReadableString(action.target),
|
|
45962
|
+
to_readable: targetAsReadableString(dep)
|
|
45963
|
+
});
|
|
45964
|
+
}
|
|
45965
|
+
}
|
|
45966
|
+
function pushColumns(action, out) {
|
|
45967
|
+
const columns = (action.actionDescriptor && action.actionDescriptor.columns) || [];
|
|
45968
|
+
for (const column of columns) {
|
|
45969
|
+
out.push({
|
|
45970
|
+
target_key: key(action.target),
|
|
45971
|
+
readable_name: targetAsReadableString(action.target),
|
|
45972
|
+
column_name: (column.path || []).join("."),
|
|
45973
|
+
description: column.description || ""
|
|
45974
|
+
});
|
|
45975
|
+
}
|
|
45976
|
+
}
|
|
45977
|
+
function catalogRows(compiledGraph) {
|
|
45978
|
+
const actions = [];
|
|
45979
|
+
const dependencies = [];
|
|
45980
|
+
const columns = [];
|
|
45981
|
+
const add = (action, type) => {
|
|
45982
|
+
actions.push(actionRow(action, type));
|
|
45983
|
+
pushDeps(action, dependencies);
|
|
45984
|
+
pushColumns(action, columns);
|
|
45985
|
+
};
|
|
45986
|
+
for (const table of compiledGraph.tables || []) {
|
|
45987
|
+
add(table, tableType$1(table.enumType));
|
|
45988
|
+
}
|
|
45989
|
+
for (const operation of compiledGraph.operations || []) {
|
|
45990
|
+
add(operation, "operation");
|
|
45991
|
+
}
|
|
45992
|
+
for (const assertion of compiledGraph.assertions || []) {
|
|
45993
|
+
add(assertion, "assertion");
|
|
45994
|
+
}
|
|
45995
|
+
for (const exp of compiledGraph.exports || []) {
|
|
45996
|
+
add(exp, "export");
|
|
45997
|
+
}
|
|
45998
|
+
for (const declaration of compiledGraph.declarations || []) {
|
|
45999
|
+
add(declaration, "declaration");
|
|
46000
|
+
}
|
|
46001
|
+
return { actions, dependencies, columns };
|
|
46002
|
+
}
|
|
46003
|
+
function runRows(runResult, runId) {
|
|
46004
|
+
const runStatus = enumName(sqlanvil.RunResult.ExecutionStatus, runResult.status);
|
|
46005
|
+
return (runResult.actions || []).map(action => {
|
|
46006
|
+
const start = toMillis(action.timing && action.timing.startTimeMillis);
|
|
46007
|
+
const end = toMillis(action.timing && action.timing.endTimeMillis);
|
|
46008
|
+
const failedTask = (action.tasks || []).find(t => !!t.errorMessage);
|
|
46009
|
+
return {
|
|
46010
|
+
run_id: runId,
|
|
46011
|
+
run_status: runStatus,
|
|
46012
|
+
target_key: key(action.target),
|
|
46013
|
+
readable_name: targetAsReadableString(action.target),
|
|
46014
|
+
status: enumName(sqlanvil.ActionResult.ExecutionStatus, action.status),
|
|
46015
|
+
start_millis: start,
|
|
46016
|
+
end_millis: end,
|
|
46017
|
+
duration_millis: start && end ? end - start : 0,
|
|
46018
|
+
error_message: (failedTask && failedTask.errorMessage) || ""
|
|
46019
|
+
};
|
|
46020
|
+
});
|
|
46021
|
+
}
|
|
46022
|
+
|
|
46023
|
+
let tmpCounter = 0;
|
|
46024
|
+
async function writeParquet(rows, outPath, columns) {
|
|
46025
|
+
await fs__namespace.ensureDir(path__namespace.dirname(outPath));
|
|
46026
|
+
const tmp = path__namespace.join(os__namespace.tmpdir(), `sa_artifact_${process.pid}_${Date.now()}_${tmpCounter++}.json`);
|
|
46027
|
+
const payload = rows.length > 0 ? rows : [columns.reduce((o, c) => ((o[c] = null), o), {})];
|
|
46028
|
+
const whereFalse = rows.length > 0 ? "" : " WHERE false";
|
|
46029
|
+
await fs__namespace.writeFile(tmp, JSON.stringify(payload));
|
|
46030
|
+
try {
|
|
46031
|
+
await withDuckdb(async (conn) => {
|
|
46032
|
+
await runAsync(conn, `COPY (SELECT * FROM read_json_auto('${tmp}')${whereFalse}) TO '${outPath}' (FORMAT parquet)`);
|
|
46033
|
+
});
|
|
46034
|
+
}
|
|
46035
|
+
finally {
|
|
46036
|
+
await fs__namespace.remove(tmp).catch(() => undefined);
|
|
46037
|
+
}
|
|
46038
|
+
}
|
|
46039
|
+
async function queryParquet(sql, views) {
|
|
46040
|
+
return withDuckdb(async (conn) => {
|
|
46041
|
+
for (const view of views) {
|
|
46042
|
+
await runAsync(conn, `create view ${view.name} as select * from read_parquet('${view.glob}')`);
|
|
46043
|
+
}
|
|
46044
|
+
return allAsync(conn, sql);
|
|
46045
|
+
});
|
|
46046
|
+
}
|
|
46047
|
+
|
|
46048
|
+
const TARGET_DIR = "target";
|
|
46049
|
+
const ACTION_COLUMNS = [
|
|
46050
|
+
"target_key",
|
|
46051
|
+
"database",
|
|
46052
|
+
"schema",
|
|
46053
|
+
"name",
|
|
46054
|
+
"readable_name",
|
|
46055
|
+
"type",
|
|
46056
|
+
"tags",
|
|
46057
|
+
"disabled",
|
|
46058
|
+
"file_name",
|
|
46059
|
+
"description"
|
|
46060
|
+
];
|
|
46061
|
+
const DEPENDENCY_COLUMNS = ["from_target_key", "to_target_key", "from_readable", "to_readable"];
|
|
46062
|
+
const COLUMN_COLUMNS = ["target_key", "readable_name", "column_name", "description"];
|
|
46063
|
+
const RUN_COLUMNS = [
|
|
46064
|
+
"run_id",
|
|
46065
|
+
"run_status",
|
|
46066
|
+
"target_key",
|
|
46067
|
+
"readable_name",
|
|
46068
|
+
"status",
|
|
46069
|
+
"start_millis",
|
|
46070
|
+
"end_millis",
|
|
46071
|
+
"duration_millis",
|
|
46072
|
+
"error_message"
|
|
46073
|
+
];
|
|
46074
|
+
async function writeArtifacts(compiledGraph, projectDir, options = {}) {
|
|
46075
|
+
const targetDir = path__namespace.join(projectDir, TARGET_DIR);
|
|
46076
|
+
const catalogDir = path__namespace.join(targetDir, "catalog");
|
|
46077
|
+
const { actions, dependencies, columns } = catalogRows(compiledGraph);
|
|
46078
|
+
await writeParquet(actions, path__namespace.join(catalogDir, "actions.parquet"), ACTION_COLUMNS);
|
|
46079
|
+
await writeParquet(dependencies, path__namespace.join(catalogDir, "dependencies.parquet"), DEPENDENCY_COLUMNS);
|
|
46080
|
+
await writeParquet(columns, path__namespace.join(catalogDir, "columns.parquet"), COLUMN_COLUMNS);
|
|
46081
|
+
if (options.runResult) {
|
|
46082
|
+
const runId = options.runId !== undefined ? options.runId : Date.now();
|
|
46083
|
+
await writeParquet(runRows(options.runResult, runId), path__namespace.join(targetDir, "runs", `run_${runId}.parquet`), RUN_COLUMNS);
|
|
46084
|
+
}
|
|
46085
|
+
return { targetDir };
|
|
46086
|
+
}
|
|
46087
|
+
async function safeWriteArtifacts(compiledGraph, projectDir, options = {}) {
|
|
46088
|
+
try {
|
|
46089
|
+
await writeArtifacts(compiledGraph, projectDir, options);
|
|
46090
|
+
}
|
|
46091
|
+
catch (e) {
|
|
46092
|
+
if (options.warn) {
|
|
46093
|
+
options.warn(`Artifacts skipped: ${e.message}`);
|
|
46094
|
+
}
|
|
46095
|
+
}
|
|
46096
|
+
}
|
|
46097
|
+
|
|
46098
|
+
function escapeHtml(value) {
|
|
46099
|
+
return String(value === null || value === undefined ? "" : value)
|
|
46100
|
+
.replace(/&/g, "&")
|
|
46101
|
+
.replace(/</g, "<")
|
|
46102
|
+
.replace(/>/g, ">")
|
|
46103
|
+
.replace(/"/g, """);
|
|
46104
|
+
}
|
|
46105
|
+
async function buildDocsModel(views, generatedAt) {
|
|
46106
|
+
const hasRuns = views.some(v => v.name === "runs");
|
|
46107
|
+
const actions = await queryParquet("select readable_name, type, tags, description from actions order by type, readable_name", views);
|
|
46108
|
+
const dependencies = await queryParquet("select from_readable, to_readable from dependencies", views);
|
|
46109
|
+
const columns = await queryParquet("select readable_name, column_name, description from columns order by readable_name, column_name", views);
|
|
46110
|
+
let latestRun;
|
|
46111
|
+
const statusByModel = new Map();
|
|
46112
|
+
if (hasRuns) {
|
|
46113
|
+
const head = await queryParquet("select max(run_id) as run_id from runs", views);
|
|
46114
|
+
const runId = head[0] && head[0].run_id !== null ? Number(head[0].run_id) : undefined;
|
|
46115
|
+
if (runId !== undefined) {
|
|
46116
|
+
const overall = await queryParquet(`select run_status from runs where run_id = ${runId} limit 1`, views);
|
|
46117
|
+
latestRun = { runId, status: overall[0] ? overall[0].run_status : "UNKNOWN" };
|
|
46118
|
+
const statuses = await queryParquet(`select readable_name, status from runs where run_id = ${runId}`, views);
|
|
46119
|
+
for (const row of statuses) {
|
|
46120
|
+
statusByModel.set(row.readable_name, row.status);
|
|
46121
|
+
}
|
|
46122
|
+
}
|
|
46123
|
+
}
|
|
46124
|
+
const dependsOn = new Map();
|
|
46125
|
+
for (const dep of dependencies) {
|
|
46126
|
+
dependsOn.set(dep.from_readable, (dependsOn.get(dep.from_readable) || []).concat(dep.to_readable));
|
|
46127
|
+
}
|
|
46128
|
+
const byTypeMap = new Map();
|
|
46129
|
+
const models = actions.map(a => {
|
|
46130
|
+
byTypeMap.set(a.type, (byTypeMap.get(a.type) || 0) + 1);
|
|
46131
|
+
let tags = [];
|
|
46132
|
+
try {
|
|
46133
|
+
tags = JSON.parse(a.tags || "[]");
|
|
46134
|
+
}
|
|
46135
|
+
catch (e) {
|
|
46136
|
+
tags = [];
|
|
46137
|
+
}
|
|
46138
|
+
return {
|
|
46139
|
+
readable: a.readable_name,
|
|
46140
|
+
type: a.type,
|
|
46141
|
+
tags,
|
|
46142
|
+
description: a.description || "",
|
|
46143
|
+
status: statusByModel.get(a.readable_name),
|
|
46144
|
+
dependsOn: dependsOn.get(a.readable_name) || []
|
|
46145
|
+
};
|
|
46146
|
+
});
|
|
46147
|
+
return {
|
|
46148
|
+
generatedAt,
|
|
46149
|
+
summary: {
|
|
46150
|
+
total: models.length,
|
|
46151
|
+
byType: Array.from(byTypeMap.entries())
|
|
46152
|
+
.map(([type, n]) => ({ type, n }))
|
|
46153
|
+
.sort((x, y) => x.type.localeCompare(y.type))
|
|
46154
|
+
},
|
|
46155
|
+
latestRun,
|
|
46156
|
+
models,
|
|
46157
|
+
columns: columns.map(c => ({
|
|
46158
|
+
readable: c.readable_name,
|
|
46159
|
+
column: c.column_name,
|
|
46160
|
+
description: c.description || ""
|
|
46161
|
+
}))
|
|
46162
|
+
};
|
|
46163
|
+
}
|
|
46164
|
+
function renderDocsHtml(model) {
|
|
46165
|
+
const summaryLine = `${model.summary.total} models — ` +
|
|
46166
|
+
model.summary.byType.map(t => `${t.n} ${t.type}`).join(", ");
|
|
46167
|
+
const runLine = model.latestRun
|
|
46168
|
+
? `Last run: <strong>${escapeHtml(model.latestRun.status)}</strong> (run ${model.latestRun.runId})`
|
|
46169
|
+
: "No runs recorded yet.";
|
|
46170
|
+
const statusBadge = (status) => {
|
|
46171
|
+
if (!status) {
|
|
46172
|
+
return "";
|
|
46173
|
+
}
|
|
46174
|
+
const cls = status === "SUCCESSFUL" ? "ok" : status === "FAILED" ? "fail" : "muted";
|
|
46175
|
+
return `<span class="badge ${cls}">${escapeHtml(status)}</span>`;
|
|
46176
|
+
};
|
|
46177
|
+
const modelRows = model.models
|
|
46178
|
+
.map(m => `<tr data-search="${escapeHtml((m.readable + " " + m.type + " " + m.tags.join(" ")).toLowerCase())}">
|
|
46179
|
+
<td><code>${escapeHtml(m.readable)}</code></td>
|
|
46180
|
+
<td>${escapeHtml(m.type)}</td>
|
|
46181
|
+
<td>${m.tags.map(t => `<span class="tag">${escapeHtml(t)}</span>`).join(" ")}</td>
|
|
46182
|
+
<td>${statusBadge(m.status)}</td>
|
|
46183
|
+
<td>${m.dependsOn.map(d => `<code>${escapeHtml(d)}</code>`).join("<br>")}</td>
|
|
46184
|
+
<td>${escapeHtml(m.description)}</td>
|
|
46185
|
+
</tr>`)
|
|
46186
|
+
.join("\n");
|
|
46187
|
+
const columnRows = model.columns
|
|
46188
|
+
.map(c => `<tr><td><code>${escapeHtml(c.readable)}</code></td><td><code>${escapeHtml(c.column)}</code></td><td>${escapeHtml(c.description)}</td></tr>`)
|
|
46189
|
+
.join("\n");
|
|
46190
|
+
return `<!doctype html>
|
|
46191
|
+
<html lang="en">
|
|
46192
|
+
<head>
|
|
46193
|
+
<meta charset="utf-8">
|
|
46194
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
46195
|
+
<title>SQLAnvil catalog</title>
|
|
46196
|
+
<style>
|
|
46197
|
+
:root { color-scheme: light dark; }
|
|
46198
|
+
body { font: 14px/1.5 -apple-system, system-ui, sans-serif; margin: 2rem; max-width: 1100px; }
|
|
46199
|
+
h1 { margin: 0 0 .25rem; }
|
|
46200
|
+
.meta { color: #888; margin-bottom: 1.5rem; }
|
|
46201
|
+
table { border-collapse: collapse; width: 100%; margin: 1rem 0 2rem; }
|
|
46202
|
+
th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #8884; vertical-align: top; }
|
|
46203
|
+
th { font-weight: 600; }
|
|
46204
|
+
code { font-size: 12px; }
|
|
46205
|
+
.tag { background: #8883; border-radius: 4px; padding: 0 .35rem; font-size: 12px; }
|
|
46206
|
+
.badge { border-radius: 4px; padding: 0 .4rem; font-size: 12px; font-weight: 600; }
|
|
46207
|
+
.badge.ok { background: #1a7f37; color: #fff; }
|
|
46208
|
+
.badge.fail { background: #b62324; color: #fff; }
|
|
46209
|
+
.badge.muted { background: #8884; }
|
|
46210
|
+
#q { padding: .4rem .6rem; width: 320px; max-width: 100%; margin-bottom: .5rem; }
|
|
46211
|
+
</style>
|
|
46212
|
+
</head>
|
|
46213
|
+
<body>
|
|
46214
|
+
<h1>SQLAnvil catalog</h1>
|
|
46215
|
+
<div class="meta">${escapeHtml(summaryLine)} · ${runLine} · generated ${escapeHtml(model.generatedAt)}</div>
|
|
46216
|
+
|
|
46217
|
+
<input id="q" type="search" placeholder="Filter models…" oninput="filterModels(this.value)">
|
|
46218
|
+
<table id="models">
|
|
46219
|
+
<thead><tr><th>Model</th><th>Type</th><th>Tags</th><th>Last run</th><th>Depends on</th><th>Description</th></tr></thead>
|
|
46220
|
+
<tbody>
|
|
46221
|
+
${modelRows}
|
|
46222
|
+
</tbody>
|
|
46223
|
+
</table>
|
|
46224
|
+
|
|
46225
|
+
<h2>Columns</h2>
|
|
46226
|
+
<table>
|
|
46227
|
+
<thead><tr><th>Model</th><th>Column</th><th>Description</th></tr></thead>
|
|
46228
|
+
<tbody>
|
|
46229
|
+
${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></tr>'}
|
|
46230
|
+
</tbody>
|
|
46231
|
+
</table>
|
|
46232
|
+
|
|
46233
|
+
<script>
|
|
46234
|
+
function filterModels(q) {
|
|
46235
|
+
q = q.toLowerCase();
|
|
46236
|
+
for (const tr of document.querySelectorAll('#models tbody tr')) {
|
|
46237
|
+
tr.style.display = tr.getAttribute('data-search').includes(q) ? '' : 'none';
|
|
46238
|
+
}
|
|
46239
|
+
}
|
|
46240
|
+
</script>
|
|
46241
|
+
</body>
|
|
46242
|
+
</html>
|
|
46243
|
+
`;
|
|
46244
|
+
}
|
|
46245
|
+
|
|
45207
46246
|
function targetKey(target) {
|
|
45208
46247
|
return [target.database, target.schema, target.name].filter(Boolean).join(".");
|
|
45209
46248
|
}
|
|
@@ -47964,6 +49003,11 @@ const timeoutOption = option("timeout", {
|
|
|
47964
49003
|
default: null,
|
|
47965
49004
|
coerce: (rawTimeoutString) => rawTimeoutString ? parseDuration__default["default"](rawTimeoutString) : null
|
|
47966
49005
|
});
|
|
49006
|
+
const noArtifactsOption = option("no-artifacts", {
|
|
49007
|
+
describe: "Skip writing the queryable Parquet artifacts under target/ (catalog on compile; run history " +
|
|
49008
|
+
"on run).",
|
|
49009
|
+
type: "boolean"
|
|
49010
|
+
});
|
|
47967
49011
|
const keepShadowOption = option("keep-shadow", {
|
|
47968
49012
|
describe: "If set, `validate` leaves its temporary shadow schema(s) in place instead of dropping them " +
|
|
47969
49013
|
"(debugging aid).",
|
|
@@ -48132,6 +49176,106 @@ async function runValidate(argv) {
|
|
|
48132
49176
|
const results = await validate(prunedGraph, deps, { keepShadow: argv[keepShadowOption.name] });
|
|
48133
49177
|
return printValidationResults(results, argv[jsonOutputOption.name]);
|
|
48134
49178
|
}
|
|
49179
|
+
function resolveArtifactViews(projectDir) {
|
|
49180
|
+
const catalogDir = path__namespace.join(projectDir, TARGET_DIR, "catalog");
|
|
49181
|
+
const runsDir = path__namespace.join(projectDir, TARGET_DIR, "runs");
|
|
49182
|
+
const views = [];
|
|
49183
|
+
for (const name of ["actions", "dependencies", "columns"]) {
|
|
49184
|
+
const file = path__namespace.join(catalogDir, `${name}.parquet`);
|
|
49185
|
+
if (fs__namespace$1.existsSync(file)) {
|
|
49186
|
+
views.push({ name, glob: file });
|
|
49187
|
+
}
|
|
49188
|
+
}
|
|
49189
|
+
const hasRuns = fs__namespace$1.existsSync(runsDir) && fs__namespace$1.readdirSync(runsDir).some(f => f.endsWith(".parquet"));
|
|
49190
|
+
if (hasRuns) {
|
|
49191
|
+
views.push({ name: "runs", glob: path__namespace.join(runsDir, "*.parquet") });
|
|
49192
|
+
}
|
|
49193
|
+
return { views, hasCatalog: views.some(v => v.name === "actions"), hasRuns };
|
|
49194
|
+
}
|
|
49195
|
+
function printArtifactRows(rows) {
|
|
49196
|
+
if (!rows || rows.length === 0) {
|
|
49197
|
+
print("(0 rows)");
|
|
49198
|
+
return;
|
|
49199
|
+
}
|
|
49200
|
+
const cols = Object.keys(rows[0]);
|
|
49201
|
+
const widths = cols.map(c => Math.max(c.length, ...rows.map(r => String(r[c] === null || r[c] === undefined ? "" : r[c]).length)));
|
|
49202
|
+
const fmtRow = (vals) => vals.map((v, i) => v.padEnd(widths[i])).join(" ");
|
|
49203
|
+
print(fmtRow(cols));
|
|
49204
|
+
print(fmtRow(widths.map(w => "-".repeat(w))));
|
|
49205
|
+
for (const row of rows) {
|
|
49206
|
+
print(fmtRow(cols.map(c => String(row[c] === null || row[c] === undefined ? "" : row[c]))));
|
|
49207
|
+
}
|
|
49208
|
+
print(`\n(${rows.length} row${rows.length === 1 ? "" : "s"})`);
|
|
49209
|
+
}
|
|
49210
|
+
const NO_ARTIFACTS = "No artifacts found under target/. Run `sqlanvil compile` (or `run`) first.";
|
|
49211
|
+
async function runQuery(projectDir, sql, json) {
|
|
49212
|
+
const { views, hasCatalog } = resolveArtifactViews(projectDir);
|
|
49213
|
+
if (!hasCatalog) {
|
|
49214
|
+
printError(NO_ARTIFACTS);
|
|
49215
|
+
return 1;
|
|
49216
|
+
}
|
|
49217
|
+
const rows = await queryParquet(sql, views);
|
|
49218
|
+
if (json) {
|
|
49219
|
+
print(prettyJsonStringify(rows));
|
|
49220
|
+
}
|
|
49221
|
+
else {
|
|
49222
|
+
printArtifactRows(rows);
|
|
49223
|
+
}
|
|
49224
|
+
return 0;
|
|
49225
|
+
}
|
|
49226
|
+
async function runInspect(projectDir, json) {
|
|
49227
|
+
const { views, hasCatalog, hasRuns } = resolveArtifactViews(projectDir);
|
|
49228
|
+
if (!hasCatalog) {
|
|
49229
|
+
printError(NO_ARTIFACTS);
|
|
49230
|
+
return 1;
|
|
49231
|
+
}
|
|
49232
|
+
const actionsByType = await queryParquet("select type, count(*) as n from actions group by type order by type", views);
|
|
49233
|
+
let latestRun = null;
|
|
49234
|
+
let failures = [];
|
|
49235
|
+
if (hasRuns) {
|
|
49236
|
+
const latest = await queryParquet("select run_id, run_status, " +
|
|
49237
|
+
"count(*) filter (where status = 'SUCCESSFUL') as succeeded, " +
|
|
49238
|
+
"count(*) filter (where status = 'FAILED') as failed, " +
|
|
49239
|
+
"max(end_millis) - min(start_millis) as wall_ms " +
|
|
49240
|
+
"from runs where run_id = (select max(run_id) from runs) group by run_id, run_status", views);
|
|
49241
|
+
latestRun = latest[0] || null;
|
|
49242
|
+
failures = await queryParquet("select readable_name, error_message from runs " +
|
|
49243
|
+
"where run_id = (select max(run_id) from runs) and status = 'FAILED' limit 20", views);
|
|
49244
|
+
}
|
|
49245
|
+
if (json) {
|
|
49246
|
+
print(prettyJsonStringify({ actionsByType, latestRun, failures }));
|
|
49247
|
+
return 0;
|
|
49248
|
+
}
|
|
49249
|
+
print("Actions by type:");
|
|
49250
|
+
printArtifactRows(actionsByType);
|
|
49251
|
+
if (!hasRuns || !latestRun) {
|
|
49252
|
+
print("\nNo runs recorded yet.");
|
|
49253
|
+
}
|
|
49254
|
+
else {
|
|
49255
|
+
print(`\nLatest run (${latestRun.run_status}): ${latestRun.succeeded} succeeded, ` +
|
|
49256
|
+
`${latestRun.failed} failed, ${latestRun.wall_ms}ms`);
|
|
49257
|
+
if (failures.length > 0) {
|
|
49258
|
+
print("\nFailures:");
|
|
49259
|
+
printArtifactRows(failures);
|
|
49260
|
+
}
|
|
49261
|
+
}
|
|
49262
|
+
return 0;
|
|
49263
|
+
}
|
|
49264
|
+
async function runDocs(projectDir) {
|
|
49265
|
+
const { views, hasCatalog } = resolveArtifactViews(projectDir);
|
|
49266
|
+
if (!hasCatalog) {
|
|
49267
|
+
printError(NO_ARTIFACTS);
|
|
49268
|
+
return 1;
|
|
49269
|
+
}
|
|
49270
|
+
const model = await buildDocsModel(views, new Date().toISOString());
|
|
49271
|
+
const html = renderDocsHtml(model);
|
|
49272
|
+
const outDir = path__namespace.join(projectDir, TARGET_DIR, "docs");
|
|
49273
|
+
fs__namespace$1.mkdirSync(outDir, { recursive: true });
|
|
49274
|
+
const outFile = path__namespace.join(outDir, "index.html");
|
|
49275
|
+
fs__namespace$1.writeFileSync(outFile, html);
|
|
49276
|
+
printSuccess(`Wrote catalog to ${outFile}`);
|
|
49277
|
+
return 0;
|
|
49278
|
+
}
|
|
48135
49279
|
function runCli() {
|
|
48136
49280
|
const builtYargs = createYargsCli({
|
|
48137
49281
|
commands: [
|
|
@@ -48262,6 +49406,7 @@ function runCli() {
|
|
|
48262
49406
|
compileTagsOption,
|
|
48263
49407
|
compileIncludeDepsOption,
|
|
48264
49408
|
compileIncludeDependentsOption,
|
|
49409
|
+
noArtifactsOption,
|
|
48265
49410
|
option(verboseOptionName, {
|
|
48266
49411
|
describe: "Enable verbose compilation output. Example usage: 'sqlanvil compile --verbose'",
|
|
48267
49412
|
type: "boolean",
|
|
@@ -48308,6 +49453,9 @@ function runCli() {
|
|
|
48308
49453
|
printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]);
|
|
48309
49454
|
return true;
|
|
48310
49455
|
}
|
|
49456
|
+
if (!argv[noArtifactsOption.name]) {
|
|
49457
|
+
await safeWriteArtifacts(compiledGraph, projectDir, { warn: print });
|
|
49458
|
+
}
|
|
48311
49459
|
return false;
|
|
48312
49460
|
}
|
|
48313
49461
|
const graphHasErrors = await compileAndPrint();
|
|
@@ -48458,6 +49606,7 @@ function runCli() {
|
|
|
48458
49606
|
timeoutOption,
|
|
48459
49607
|
tagsOption,
|
|
48460
49608
|
bigqueryJobLabelsOption,
|
|
49609
|
+
noArtifactsOption,
|
|
48461
49610
|
...ProjectConfigOptions.allYargsOptions
|
|
48462
49611
|
],
|
|
48463
49612
|
processFn: async (argv) => {
|
|
@@ -48572,9 +49721,47 @@ function runCli() {
|
|
|
48572
49721
|
runner.onChange(printExecutedGraph);
|
|
48573
49722
|
const runResult = await runner.result();
|
|
48574
49723
|
printExecutedGraph(runResult);
|
|
49724
|
+
if (!argv[noArtifactsOption.name]) {
|
|
49725
|
+
await safeWriteArtifacts(compiledGraph, argv[projectDirOption.name], {
|
|
49726
|
+
runResult,
|
|
49727
|
+
runId: Date.now(),
|
|
49728
|
+
warn: print
|
|
49729
|
+
});
|
|
49730
|
+
}
|
|
48575
49731
|
return runResult.status === sqlanvil.RunResult.ExecutionStatus.SUCCESSFUL ? 0 : 1;
|
|
48576
49732
|
}
|
|
48577
49733
|
},
|
|
49734
|
+
{
|
|
49735
|
+
format: `query [sql] [${projectDirOption.name}]`,
|
|
49736
|
+
description: "Run SQL over the project's queryable artifacts in target/ (views: actions, " +
|
|
49737
|
+
"dependencies, columns, runs), via the bundled DuckDB.",
|
|
49738
|
+
positionalOptions: [
|
|
49739
|
+
positionalOption("sql", { describe: 'SQL to run, e.g. "select type, count(*) from actions group by 1".' }, (argv) => {
|
|
49740
|
+
if (!argv.sql) {
|
|
49741
|
+
throw new Error('Provide a SQL query, e.g. sqlanvil query "select * from actions".');
|
|
49742
|
+
}
|
|
49743
|
+
}),
|
|
49744
|
+
projectDirOption
|
|
49745
|
+
],
|
|
49746
|
+
options: [jsonOutputOption],
|
|
49747
|
+
processFn: async (argv) => runQuery(argv[projectDirOption.name], argv.sql, argv[jsonOutputOption.name])
|
|
49748
|
+
},
|
|
49749
|
+
{
|
|
49750
|
+
format: `inspect [${projectDirOption.name}]`,
|
|
49751
|
+
description: "Summarize the project's artifacts: action counts by type, the latest run's status/" +
|
|
49752
|
+
"timing, and recent failures.",
|
|
49753
|
+
positionalOptions: [projectDirOption],
|
|
49754
|
+
options: [jsonOutputOption],
|
|
49755
|
+
processFn: async (argv) => runInspect(argv[projectDirOption.name], argv[jsonOutputOption.name])
|
|
49756
|
+
},
|
|
49757
|
+
{
|
|
49758
|
+
format: `docs [${projectDirOption.name}]`,
|
|
49759
|
+
description: "Generate a self-contained HTML catalog of the project (models, columns, dependencies, " +
|
|
49760
|
+
"last-run status) at target/docs/index.html, from the artifacts.",
|
|
49761
|
+
positionalOptions: [projectDirOption],
|
|
49762
|
+
options: [],
|
|
49763
|
+
processFn: async (argv) => runDocs(argv[projectDirOption.name])
|
|
49764
|
+
},
|
|
48578
49765
|
{
|
|
48579
49766
|
format: `format [${projectDirMustExistOption.name}]`,
|
|
48580
49767
|
description: "Format the sqlanvil project's files.",
|