ingestron 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/main.js +770 -376
- package/package.json +3 -2
- package/schemas/flow.schema.yaml +16 -0
- package/schemas/standard.schema.yaml +5 -0
- package/schemas/transform.schema.yaml +15 -0
package/dist/main.js
CHANGED
|
@@ -1184,8 +1184,8 @@ var require_stringifyString = __commonJS({
|
|
|
1184
1184
|
case "u":
|
|
1185
1185
|
{
|
|
1186
1186
|
str += json.slice(start, i);
|
|
1187
|
-
const
|
|
1188
|
-
switch (
|
|
1187
|
+
const code2 = json.substr(i + 2, 4);
|
|
1188
|
+
switch (code2) {
|
|
1189
1189
|
case "0000":
|
|
1190
1190
|
str += "\\0";
|
|
1191
1191
|
break;
|
|
@@ -1211,8 +1211,8 @@ var require_stringifyString = __commonJS({
|
|
|
1211
1211
|
str += "\\P";
|
|
1212
1212
|
break;
|
|
1213
1213
|
default:
|
|
1214
|
-
if (
|
|
1215
|
-
str += "\\x" +
|
|
1214
|
+
if (code2.substr(0, 2) === "00")
|
|
1215
|
+
str += "\\x" + code2.substr(2);
|
|
1216
1216
|
else
|
|
1217
1217
|
str += json.substr(i, 6);
|
|
1218
1218
|
}
|
|
@@ -1280,9 +1280,9 @@ ${indent2}`) + "'";
|
|
|
1280
1280
|
return quotedString(value, ctx);
|
|
1281
1281
|
}
|
|
1282
1282
|
const indent2 = ctx.indent || (ctx.forceBlockIndent || containsDocumentMarker(value) ? " " : "");
|
|
1283
|
-
const
|
|
1283
|
+
const literal3 = blockQuote === "literal" ? true : blockQuote === "folded" || type === Scalar.Scalar.BLOCK_FOLDED ? false : type === Scalar.Scalar.BLOCK_LITERAL ? true : !lineLengthOverLimit(value, lineWidth, indent2.length);
|
|
1284
1284
|
if (!value)
|
|
1285
|
-
return
|
|
1285
|
+
return literal3 ? "|\n" : ">\n";
|
|
1286
1286
|
let chomp;
|
|
1287
1287
|
let endStart;
|
|
1288
1288
|
for (endStart = value.length; endStart > 0; --endStart) {
|
|
@@ -1331,7 +1331,7 @@ ${indent2}`) + "'";
|
|
|
1331
1331
|
if (onComment)
|
|
1332
1332
|
onComment();
|
|
1333
1333
|
}
|
|
1334
|
-
if (!
|
|
1334
|
+
if (!literal3) {
|
|
1335
1335
|
const foldedValue = value.replace(/\n+/g, "\n$&").replace(/(?:^|\n)([\t ].*)(?:([\n\t ]*)\n(?![\n\t ]))?/g, "$1$2").replace(/\n+/g, `$&${indent2}`);
|
|
1336
1336
|
let literalFallback = false;
|
|
1337
1337
|
const foldOptions = getFoldOptions(ctx, true);
|
|
@@ -3711,22 +3711,22 @@ var require_errors = __commonJS({
|
|
|
3711
3711
|
"node_modules/.pnpm/yaml@2.9.1/node_modules/yaml/dist/errors.js"(exports) {
|
|
3712
3712
|
"use strict";
|
|
3713
3713
|
var YAMLError = class extends Error {
|
|
3714
|
-
constructor(name, pos,
|
|
3714
|
+
constructor(name, pos, code2, message) {
|
|
3715
3715
|
super();
|
|
3716
3716
|
this.name = name;
|
|
3717
|
-
this.code =
|
|
3717
|
+
this.code = code2;
|
|
3718
3718
|
this.message = message;
|
|
3719
3719
|
this.pos = pos;
|
|
3720
3720
|
}
|
|
3721
3721
|
};
|
|
3722
3722
|
var YAMLParseError = class extends YAMLError {
|
|
3723
|
-
constructor(pos,
|
|
3724
|
-
super("YAMLParseError", pos,
|
|
3723
|
+
constructor(pos, code2, message) {
|
|
3724
|
+
super("YAMLParseError", pos, code2, message);
|
|
3725
3725
|
}
|
|
3726
3726
|
};
|
|
3727
3727
|
var YAMLWarning = class extends YAMLError {
|
|
3728
|
-
constructor(pos,
|
|
3729
|
-
super("YAMLWarning", pos,
|
|
3728
|
+
constructor(pos, code2, message) {
|
|
3729
|
+
super("YAMLWarning", pos, code2, message);
|
|
3730
3730
|
}
|
|
3731
3731
|
};
|
|
3732
3732
|
var prettifyError = (src, lc) => (error2) => {
|
|
@@ -4635,7 +4635,7 @@ var require_resolve_flow_scalar = __commonJS({
|
|
|
4635
4635
|
const { offset, type, source, end } = scalar2;
|
|
4636
4636
|
let _type;
|
|
4637
4637
|
let value;
|
|
4638
|
-
const _onError = (rel,
|
|
4638
|
+
const _onError = (rel, code2, msg) => onError(offset + rel, code2, msg);
|
|
4639
4639
|
switch (type) {
|
|
4640
4640
|
case "scalar":
|
|
4641
4641
|
_type = Scalar.Scalar.PLAIN;
|
|
@@ -4833,9 +4833,9 @@ var require_resolve_flow_scalar = __commonJS({
|
|
|
4833
4833
|
function parseCharCode(source, offset, length, onError) {
|
|
4834
4834
|
const cc = source.substr(offset, length);
|
|
4835
4835
|
const ok = cc.length === length && /^[0-9a-fA-F]+$/.test(cc);
|
|
4836
|
-
const
|
|
4836
|
+
const code2 = ok ? parseInt(cc, 16) : NaN;
|
|
4837
4837
|
try {
|
|
4838
|
-
return String.fromCodePoint(
|
|
4838
|
+
return String.fromCodePoint(code2);
|
|
4839
4839
|
} catch {
|
|
4840
4840
|
const raw = source.substr(offset - 2, length + 2);
|
|
4841
4841
|
onError(offset - 2, "BAD_DQ_ESCAPE", `Invalid escape sequence ${raw}`);
|
|
@@ -5157,12 +5157,12 @@ var require_composer = __commonJS({
|
|
|
5157
5157
|
this.prelude = [];
|
|
5158
5158
|
this.errors = [];
|
|
5159
5159
|
this.warnings = [];
|
|
5160
|
-
this.onError = (source,
|
|
5160
|
+
this.onError = (source, code2, message, warning3) => {
|
|
5161
5161
|
const pos = getErrorPos(source);
|
|
5162
5162
|
if (warning3)
|
|
5163
|
-
this.warnings.push(new errors.YAMLWarning(pos,
|
|
5163
|
+
this.warnings.push(new errors.YAMLWarning(pos, code2, message));
|
|
5164
5164
|
else
|
|
5165
|
-
this.errors.push(new errors.YAMLParseError(pos,
|
|
5165
|
+
this.errors.push(new errors.YAMLParseError(pos, code2, message));
|
|
5166
5166
|
};
|
|
5167
5167
|
this.directives = new directives.Directives({ version: options2.version || "1.2" });
|
|
5168
5168
|
this.options = options2;
|
|
@@ -5324,12 +5324,12 @@ var require_cst_scalar = __commonJS({
|
|
|
5324
5324
|
var stringifyString = require_stringifyString();
|
|
5325
5325
|
function resolveAsScalar(token2, strict = true, onError) {
|
|
5326
5326
|
if (token2) {
|
|
5327
|
-
const _onError = (pos,
|
|
5327
|
+
const _onError = (pos, code2, message) => {
|
|
5328
5328
|
const offset = typeof pos === "number" ? pos : Array.isArray(pos) ? pos[0] : pos.offset;
|
|
5329
5329
|
if (onError)
|
|
5330
|
-
onError(offset,
|
|
5330
|
+
onError(offset, code2, message);
|
|
5331
5331
|
else
|
|
5332
|
-
throw new errors.YAMLParseError([offset, offset + 1],
|
|
5332
|
+
throw new errors.YAMLParseError([offset, offset + 1], code2, message);
|
|
5333
5333
|
};
|
|
5334
5334
|
switch (token2.type) {
|
|
5335
5335
|
case "scalar":
|
|
@@ -6042,9 +6042,9 @@ var require_lexer = __commonJS({
|
|
|
6042
6042
|
}
|
|
6043
6043
|
}
|
|
6044
6044
|
*parseQuotedScalar() {
|
|
6045
|
-
const
|
|
6046
|
-
let end = this.buffer.indexOf(
|
|
6047
|
-
if (
|
|
6045
|
+
const quote2 = this.charAt(0);
|
|
6046
|
+
let end = this.buffer.indexOf(quote2, this.pos + 1);
|
|
6047
|
+
if (quote2 === "'") {
|
|
6048
6048
|
while (end !== -1 && this.buffer[end + 1] === "'")
|
|
6049
6049
|
end = this.buffer.indexOf("'", end + 2);
|
|
6050
6050
|
} else {
|
|
@@ -7396,9 +7396,9 @@ var require_code = __commonJS({
|
|
|
7396
7396
|
};
|
|
7397
7397
|
exports.Name = Name;
|
|
7398
7398
|
var _Code = class extends _CodeOrName {
|
|
7399
|
-
constructor(
|
|
7399
|
+
constructor(code2) {
|
|
7400
7400
|
super();
|
|
7401
|
-
this._items = typeof
|
|
7401
|
+
this._items = typeof code2 === "string" ? [code2] : code2;
|
|
7402
7402
|
}
|
|
7403
7403
|
toString() {
|
|
7404
7404
|
return this.str;
|
|
@@ -7425,13 +7425,13 @@ var require_code = __commonJS({
|
|
|
7425
7425
|
exports._Code = _Code;
|
|
7426
7426
|
exports.nil = new _Code("");
|
|
7427
7427
|
function _(strs, ...args) {
|
|
7428
|
-
const
|
|
7428
|
+
const code2 = [strs[0]];
|
|
7429
7429
|
let i = 0;
|
|
7430
7430
|
while (i < args.length) {
|
|
7431
|
-
addCodeArg(
|
|
7432
|
-
|
|
7431
|
+
addCodeArg(code2, args[i]);
|
|
7432
|
+
code2.push(strs[++i]);
|
|
7433
7433
|
}
|
|
7434
|
-
return new _Code(
|
|
7434
|
+
return new _Code(code2);
|
|
7435
7435
|
}
|
|
7436
7436
|
exports._ = _;
|
|
7437
7437
|
var plus = new _Code("+");
|
|
@@ -7447,13 +7447,13 @@ var require_code = __commonJS({
|
|
|
7447
7447
|
return new _Code(expr);
|
|
7448
7448
|
}
|
|
7449
7449
|
exports.str = str;
|
|
7450
|
-
function addCodeArg(
|
|
7450
|
+
function addCodeArg(code2, arg) {
|
|
7451
7451
|
if (arg instanceof _Code)
|
|
7452
|
-
|
|
7452
|
+
code2.push(...arg._items);
|
|
7453
7453
|
else if (arg instanceof Name)
|
|
7454
|
-
|
|
7454
|
+
code2.push(arg);
|
|
7455
7455
|
else
|
|
7456
|
-
|
|
7456
|
+
code2.push(interpolate(arg));
|
|
7457
7457
|
}
|
|
7458
7458
|
exports.addCodeArg = addCodeArg;
|
|
7459
7459
|
function optimize(expr) {
|
|
@@ -7637,7 +7637,7 @@ var require_scope = __commonJS({
|
|
|
7637
7637
|
}, usedValues, getCode);
|
|
7638
7638
|
}
|
|
7639
7639
|
_reduceValues(values2, valueCode, usedValues = {}, getCode) {
|
|
7640
|
-
let
|
|
7640
|
+
let code2 = code_1.nil;
|
|
7641
7641
|
for (const prefix in values2) {
|
|
7642
7642
|
const vs = values2[prefix];
|
|
7643
7643
|
if (!vs)
|
|
@@ -7650,16 +7650,16 @@ var require_scope = __commonJS({
|
|
|
7650
7650
|
let c = valueCode(name);
|
|
7651
7651
|
if (c) {
|
|
7652
7652
|
const def = this.opts.es5 ? exports.varKinds.var : exports.varKinds.const;
|
|
7653
|
-
|
|
7653
|
+
code2 = (0, code_1._)`${code2}${def} ${name} = ${c};${this.opts._n}`;
|
|
7654
7654
|
} else if (c = getCode === null || getCode === void 0 ? void 0 : getCode(name)) {
|
|
7655
|
-
|
|
7655
|
+
code2 = (0, code_1._)`${code2}${c}${this.opts._n}`;
|
|
7656
7656
|
} else {
|
|
7657
7657
|
throw new ValueError(name);
|
|
7658
7658
|
}
|
|
7659
7659
|
nameSet.set(name, UsedValueState.Completed);
|
|
7660
7660
|
});
|
|
7661
7661
|
}
|
|
7662
|
-
return
|
|
7662
|
+
return code2;
|
|
7663
7663
|
}
|
|
7664
7664
|
};
|
|
7665
7665
|
exports.ValueScope = ValueScope;
|
|
@@ -7819,9 +7819,9 @@ var require_codegen = __commonJS({
|
|
|
7819
7819
|
}
|
|
7820
7820
|
};
|
|
7821
7821
|
var AnyCode = class extends Node {
|
|
7822
|
-
constructor(
|
|
7822
|
+
constructor(code2) {
|
|
7823
7823
|
super();
|
|
7824
|
-
this.code =
|
|
7824
|
+
this.code = code2;
|
|
7825
7825
|
}
|
|
7826
7826
|
render({ _n }) {
|
|
7827
7827
|
return `${this.code};` + _n;
|
|
@@ -7843,7 +7843,7 @@ var require_codegen = __commonJS({
|
|
|
7843
7843
|
this.nodes = nodes;
|
|
7844
7844
|
}
|
|
7845
7845
|
render(opts) {
|
|
7846
|
-
return this.nodes.reduce((
|
|
7846
|
+
return this.nodes.reduce((code2, n) => code2 + n.render(opts), "");
|
|
7847
7847
|
}
|
|
7848
7848
|
optimizeNodes() {
|
|
7849
7849
|
const { nodes } = this;
|
|
@@ -7891,10 +7891,10 @@ var require_codegen = __commonJS({
|
|
|
7891
7891
|
this.condition = condition;
|
|
7892
7892
|
}
|
|
7893
7893
|
render(opts) {
|
|
7894
|
-
let
|
|
7894
|
+
let code2 = `if(${this.condition})` + super.render(opts);
|
|
7895
7895
|
if (this.else)
|
|
7896
|
-
|
|
7897
|
-
return
|
|
7896
|
+
code2 += "else " + this.else.render(opts);
|
|
7897
|
+
return code2;
|
|
7898
7898
|
}
|
|
7899
7899
|
optimizeNodes() {
|
|
7900
7900
|
super.optimizeNodes();
|
|
@@ -8015,12 +8015,12 @@ var require_codegen = __commonJS({
|
|
|
8015
8015
|
Return.kind = "return";
|
|
8016
8016
|
var Try = class extends BlockNode {
|
|
8017
8017
|
render(opts) {
|
|
8018
|
-
let
|
|
8018
|
+
let code2 = "try" + super.render(opts);
|
|
8019
8019
|
if (this.catch)
|
|
8020
|
-
|
|
8020
|
+
code2 += this.catch.render(opts);
|
|
8021
8021
|
if (this.finally)
|
|
8022
|
-
|
|
8023
|
-
return
|
|
8022
|
+
code2 += this.finally.render(opts);
|
|
8023
|
+
return code2;
|
|
8024
8024
|
}
|
|
8025
8025
|
optimizeNodes() {
|
|
8026
8026
|
var _a, _b;
|
|
@@ -8137,18 +8137,18 @@ var require_codegen = __commonJS({
|
|
|
8137
8137
|
}
|
|
8138
8138
|
// returns code for object literal for the passed argument list of key-value pairs
|
|
8139
8139
|
object(...keyValues) {
|
|
8140
|
-
const
|
|
8140
|
+
const code2 = ["{"];
|
|
8141
8141
|
for (const [key, value] of keyValues) {
|
|
8142
|
-
if (
|
|
8143
|
-
|
|
8144
|
-
|
|
8142
|
+
if (code2.length > 1)
|
|
8143
|
+
code2.push(",");
|
|
8144
|
+
code2.push(key);
|
|
8145
8145
|
if (key !== value || this.opts.es5) {
|
|
8146
|
-
|
|
8147
|
-
(0, code_1.addCodeArg)(
|
|
8146
|
+
code2.push(":");
|
|
8147
|
+
(0, code_1.addCodeArg)(code2, value);
|
|
8148
8148
|
}
|
|
8149
8149
|
}
|
|
8150
|
-
|
|
8151
|
-
return new code_1._Code(
|
|
8150
|
+
code2.push("}");
|
|
8151
|
+
return new code_1._Code(code2);
|
|
8152
8152
|
}
|
|
8153
8153
|
// `if` clause (or statement if `thenBody` and, optionally, `elseBody` are passed)
|
|
8154
8154
|
if(condition, thenBody, elseBody) {
|
|
@@ -10467,22 +10467,22 @@ var require_utils = __commonJS({
|
|
|
10467
10467
|
}
|
|
10468
10468
|
function stringArrayToHexStripped(input) {
|
|
10469
10469
|
let acc = "";
|
|
10470
|
-
let
|
|
10470
|
+
let code2 = 0;
|
|
10471
10471
|
let i = 0;
|
|
10472
10472
|
for (i = 0; i < input.length; i++) {
|
|
10473
|
-
|
|
10474
|
-
if (
|
|
10473
|
+
code2 = input[i].charCodeAt(0);
|
|
10474
|
+
if (code2 === 48) {
|
|
10475
10475
|
continue;
|
|
10476
10476
|
}
|
|
10477
|
-
if (!(
|
|
10477
|
+
if (!(code2 >= 48 && code2 <= 57 || code2 >= 65 && code2 <= 70 || code2 >= 97 && code2 <= 102)) {
|
|
10478
10478
|
return "";
|
|
10479
10479
|
}
|
|
10480
10480
|
acc += input[i];
|
|
10481
10481
|
break;
|
|
10482
10482
|
}
|
|
10483
10483
|
for (i += 1; i < input.length; i++) {
|
|
10484
|
-
|
|
10485
|
-
if (!(
|
|
10484
|
+
code2 = input[i].charCodeAt(0);
|
|
10485
|
+
if (!(code2 >= 48 && code2 <= 57 || code2 >= 65 && code2 <= 70 || code2 >= 97 && code2 <= 102)) {
|
|
10486
10486
|
return "";
|
|
10487
10487
|
}
|
|
10488
10488
|
acc += input[i];
|
|
@@ -10755,15 +10755,15 @@ var require_utils = __commonJS({
|
|
|
10755
10755
|
if (isPathCharacter(ch)) {
|
|
10756
10756
|
output += ch;
|
|
10757
10757
|
} else {
|
|
10758
|
-
const
|
|
10759
|
-
if (
|
|
10760
|
-
output += isEscapeSafe(
|
|
10761
|
-
} else if (
|
|
10762
|
-
output += percentEncodeNonAscii(
|
|
10763
|
-
} else if (
|
|
10758
|
+
const code2 = input.charCodeAt(i);
|
|
10759
|
+
if (code2 < 128) {
|
|
10760
|
+
output += isEscapeSafe(code2) ? ch : BYTE_HEX[code2];
|
|
10761
|
+
} else if (code2 < 55296 || code2 > 57343) {
|
|
10762
|
+
output += percentEncodeNonAscii(code2);
|
|
10763
|
+
} else if (code2 <= 56319 && i + 1 < input.length) {
|
|
10764
10764
|
const low = input.charCodeAt(i + 1);
|
|
10765
10765
|
if (low >= 56320 && low <= 57343) {
|
|
10766
|
-
output += percentEncodeNonAscii(65536 + (
|
|
10766
|
+
output += percentEncodeNonAscii(65536 + (code2 - 55296 << 10) + (low - 56320));
|
|
10767
10767
|
i++;
|
|
10768
10768
|
} else {
|
|
10769
10769
|
output += percentEncodeNonAscii(65533);
|
|
@@ -10794,15 +10794,15 @@ var require_utils = __commonJS({
|
|
|
10794
10794
|
if (isPathCharacter(ch) && (ch !== ":" || !firstSegment)) {
|
|
10795
10795
|
output += ch;
|
|
10796
10796
|
} else {
|
|
10797
|
-
const
|
|
10798
|
-
if (
|
|
10799
|
-
output += BYTE_HEX[
|
|
10800
|
-
} else if (
|
|
10801
|
-
output += percentEncodeNonAscii(
|
|
10802
|
-
} else if (
|
|
10797
|
+
const code2 = input.charCodeAt(i);
|
|
10798
|
+
if (code2 < 128) {
|
|
10799
|
+
output += BYTE_HEX[code2];
|
|
10800
|
+
} else if (code2 < 55296 || code2 > 57343) {
|
|
10801
|
+
output += percentEncodeNonAscii(code2);
|
|
10802
|
+
} else if (code2 <= 56319 && i + 1 < input.length) {
|
|
10803
10803
|
const low = input.charCodeAt(i + 1);
|
|
10804
10804
|
if (low >= 56320 && low <= 57343) {
|
|
10805
|
-
output += percentEncodeNonAscii(65536 + (
|
|
10805
|
+
output += percentEncodeNonAscii(65536 + (code2 - 55296 << 10) + (low - 56320));
|
|
10806
10806
|
i++;
|
|
10807
10807
|
} else {
|
|
10808
10808
|
output += percentEncodeNonAscii(65533);
|
|
@@ -10829,15 +10829,15 @@ var require_utils = __commonJS({
|
|
|
10829
10829
|
if (isAllowed2(ch)) {
|
|
10830
10830
|
output += ch;
|
|
10831
10831
|
} else {
|
|
10832
|
-
const
|
|
10833
|
-
if (
|
|
10834
|
-
output += BYTE_HEX[
|
|
10835
|
-
} else if (
|
|
10836
|
-
output += percentEncodeNonAscii(
|
|
10837
|
-
} else if (
|
|
10832
|
+
const code2 = input.charCodeAt(i);
|
|
10833
|
+
if (code2 < 128) {
|
|
10834
|
+
output += BYTE_HEX[code2];
|
|
10835
|
+
} else if (code2 < 55296 || code2 > 57343) {
|
|
10836
|
+
output += percentEncodeNonAscii(code2);
|
|
10837
|
+
} else if (code2 <= 56319 && i + 1 < input.length) {
|
|
10838
10838
|
const low = input.charCodeAt(i + 1);
|
|
10839
10839
|
if (low >= 56320 && low <= 57343) {
|
|
10840
|
-
output += percentEncodeNonAscii(65536 + (
|
|
10840
|
+
output += percentEncodeNonAscii(65536 + (code2 - 55296 << 10) + (low - 56320));
|
|
10841
10841
|
i++;
|
|
10842
10842
|
} else {
|
|
10843
10843
|
output += percentEncodeNonAscii(65533);
|
|
@@ -10882,15 +10882,15 @@ var require_utils = __commonJS({
|
|
|
10882
10882
|
if (isQueryFragmentCharacter(ch)) {
|
|
10883
10883
|
output += ch;
|
|
10884
10884
|
} else {
|
|
10885
|
-
const
|
|
10886
|
-
if (
|
|
10887
|
-
output += isEscapeSafe(
|
|
10888
|
-
} else if (
|
|
10889
|
-
output += percentEncodeNonAscii(
|
|
10890
|
-
} else if (
|
|
10885
|
+
const code2 = input.charCodeAt(i);
|
|
10886
|
+
if (code2 < 128) {
|
|
10887
|
+
output += isEscapeSafe(code2) ? ch : BYTE_HEX[code2];
|
|
10888
|
+
} else if (code2 < 55296 || code2 > 57343) {
|
|
10889
|
+
output += percentEncodeNonAscii(code2);
|
|
10890
|
+
} else if (code2 <= 56319 && i + 1 < input.length) {
|
|
10891
10891
|
const low = input.charCodeAt(i + 1);
|
|
10892
10892
|
if (low >= 56320 && low <= 57343) {
|
|
10893
|
-
output += percentEncodeNonAscii(65536 + (
|
|
10893
|
+
output += percentEncodeNonAscii(65536 + (code2 - 55296 << 10) + (low - 56320));
|
|
10894
10894
|
i++;
|
|
10895
10895
|
} else {
|
|
10896
10896
|
output += percentEncodeNonAscii(65533);
|
|
@@ -13848,8 +13848,8 @@ var require_format = __commonJS({
|
|
|
13848
13848
|
}
|
|
13849
13849
|
}
|
|
13850
13850
|
function getFormat(fmtDef) {
|
|
13851
|
-
const
|
|
13852
|
-
const fmt = gen.scopeValue("formats", { key: schema, ref: fmtDef, code });
|
|
13851
|
+
const code2 = fmtDef instanceof RegExp ? (0, codegen_1.regexpCode)(fmtDef) : opts.code.formats ? (0, codegen_1._)`${opts.code.formats}${(0, codegen_1.getProperty)(schema)}` : void 0;
|
|
13852
|
+
const fmt = gen.scopeValue("formats", { key: schema, ref: fmtDef, code: code2 });
|
|
13853
13853
|
if (typeof fmtDef == "object" && !(fmtDef instanceof RegExp)) {
|
|
13854
13854
|
return [fmtDef.type || "string", fmtDef.validate, (0, codegen_1._)`${fmt}.validate`];
|
|
13855
13855
|
}
|
|
@@ -15888,8 +15888,8 @@ var require_liquid_node = __commonJS({
|
|
|
15888
15888
|
var SIGN = 64;
|
|
15889
15889
|
var PUNCTUATION = 128;
|
|
15890
15890
|
function isWord(char) {
|
|
15891
|
-
const
|
|
15892
|
-
return
|
|
15891
|
+
const code2 = char.charCodeAt(0);
|
|
15892
|
+
return code2 >= 128 ? !TYPES[code2] : !!(TYPES[code2] & WORD);
|
|
15893
15893
|
}
|
|
15894
15894
|
TYPES[160] = TYPES[5760] = TYPES[6158] = TYPES[8192] = TYPES[8193] = TYPES[8194] = TYPES[8195] = TYPES[8196] = TYPES[8197] = TYPES[8198] = TYPES[8199] = TYPES[8200] = TYPES[8201] = TYPES[8202] = TYPES[8232] = TYPES[8233] = TYPES[8239] = TYPES[8287] = TYPES[12288] = BLANK;
|
|
15895
15895
|
TYPES[8220] = TYPES[8221] = PUNCTUATION;
|
|
@@ -16605,12 +16605,12 @@ var require_liquid_node = __commonJS({
|
|
|
16605
16605
|
v: "\v"
|
|
16606
16606
|
};
|
|
16607
16607
|
function hexVal(c) {
|
|
16608
|
-
const
|
|
16609
|
-
if (
|
|
16610
|
-
return
|
|
16611
|
-
if (
|
|
16612
|
-
return
|
|
16613
|
-
return
|
|
16608
|
+
const code2 = c.charCodeAt(0);
|
|
16609
|
+
if (code2 >= 97)
|
|
16610
|
+
return code2 - 87;
|
|
16611
|
+
if (code2 >= 65)
|
|
16612
|
+
return code2 - 55;
|
|
16613
|
+
return code2 - 48;
|
|
16614
16614
|
}
|
|
16615
16615
|
function parseStringLiteral(str) {
|
|
16616
16616
|
let ret = "";
|
|
@@ -17731,9 +17731,9 @@ var require_liquid_node = __commonJS({
|
|
|
17731
17731
|
const end = this.matchTrie(this.literalTrie);
|
|
17732
17732
|
if (end === -1)
|
|
17733
17733
|
return;
|
|
17734
|
-
const
|
|
17734
|
+
const literal3 = new LiteralToken(this.input, this.p, end, this.file);
|
|
17735
17735
|
this.p = end;
|
|
17736
|
-
return
|
|
17736
|
+
return literal3;
|
|
17737
17737
|
}
|
|
17738
17738
|
readGroupOrRange() {
|
|
17739
17739
|
this.skipBlank();
|
|
@@ -20609,11 +20609,11 @@ function answersOf(pairs) {
|
|
|
20609
20609
|
import { join as join8, relative as relative4, resolve, sep as sep4 } from "node:path";
|
|
20610
20610
|
|
|
20611
20611
|
// packages/config/src/diagnostics.ts
|
|
20612
|
-
function error(
|
|
20613
|
-
return { code, severity: "error", message, location, hint };
|
|
20612
|
+
function error(code2, message, location, hint) {
|
|
20613
|
+
return { code: code2, severity: "error", message, location, hint };
|
|
20614
20614
|
}
|
|
20615
|
-
function warning(
|
|
20616
|
-
return { code, severity: "warning", message, location, hint };
|
|
20615
|
+
function warning(code2, message, location, hint) {
|
|
20616
|
+
return { code: code2, severity: "warning", message, location, hint };
|
|
20617
20617
|
}
|
|
20618
20618
|
function hasErrors(diagnostics) {
|
|
20619
20619
|
return diagnostics.some((diagnostic) => diagnostic.severity === "error");
|
|
@@ -20784,27 +20784,27 @@ var referenceRoots = ["values", "environment", "project"];
|
|
|
20784
20784
|
var resolverNames = ["file", "envvar", "secret"];
|
|
20785
20785
|
function parseDefinitionString(text2) {
|
|
20786
20786
|
const segments = [];
|
|
20787
|
-
let
|
|
20787
|
+
let literal3 = "";
|
|
20788
20788
|
let index = 0;
|
|
20789
20789
|
while (index < text2.length) {
|
|
20790
20790
|
if (text2.startsWith("$${", index)) {
|
|
20791
|
-
|
|
20791
|
+
literal3 += "${";
|
|
20792
20792
|
index += 3;
|
|
20793
20793
|
continue;
|
|
20794
20794
|
}
|
|
20795
20795
|
if (!text2.startsWith("${", index)) {
|
|
20796
|
-
|
|
20796
|
+
literal3 += text2[index];
|
|
20797
20797
|
index += 1;
|
|
20798
20798
|
continue;
|
|
20799
20799
|
}
|
|
20800
|
-
const
|
|
20801
|
-
if (!
|
|
20802
|
-
if (
|
|
20803
|
-
|
|
20804
|
-
segments.push(
|
|
20805
|
-
index =
|
|
20800
|
+
const reference4 = readReference(text2, index);
|
|
20801
|
+
if (!reference4.ok) return reference4;
|
|
20802
|
+
if (literal3) segments.push({ type: "text", text: literal3 });
|
|
20803
|
+
literal3 = "";
|
|
20804
|
+
segments.push(reference4.segment);
|
|
20805
|
+
index = reference4.end;
|
|
20806
20806
|
}
|
|
20807
|
-
if (
|
|
20807
|
+
if (literal3 || segments.length === 0) segments.push({ type: "text", text: literal3 });
|
|
20808
20808
|
return { ok: true, segments };
|
|
20809
20809
|
}
|
|
20810
20810
|
function readReference(text2, start) {
|
|
@@ -20826,8 +20826,8 @@ function readReference(text2, start) {
|
|
|
20826
20826
|
}
|
|
20827
20827
|
return { ok: true, segment: { type: "reference", source, expression }, end: close + 1 };
|
|
20828
20828
|
}
|
|
20829
|
-
function refusal(
|
|
20830
|
-
return { ok: false, code, message, hint: keepText(snippet) };
|
|
20829
|
+
function refusal(code2, message, snippet) {
|
|
20830
|
+
return { ok: false, code: code2, message, hint: keepText(snippet) };
|
|
20831
20831
|
}
|
|
20832
20832
|
var keepText = (snippet) => `To keep "${snippet}" as text, write "$${snippet}".`;
|
|
20833
20833
|
var forms = "A reference is ${values.<name>}, ${environment.<name>}, ${project.<name>}, ${file:<path>}, ${envvar:<name>} or ${secret:<name>}.";
|
|
@@ -21354,11 +21354,11 @@ function schemasAt(root, keys2) {
|
|
|
21354
21354
|
return current;
|
|
21355
21355
|
}
|
|
21356
21356
|
function expand(root, schema) {
|
|
21357
|
-
const
|
|
21358
|
-
return
|
|
21357
|
+
const reference4 = typeof schema.$ref === "string" ? lookup(root, schema.$ref) : void 0;
|
|
21358
|
+
return reference4 ? [schema, ...expand(root, reference4)] : [schema];
|
|
21359
21359
|
}
|
|
21360
|
-
function lookup(root,
|
|
21361
|
-
const [, group, name] = /^#\/(\$defs|definitions)\/(.+)$/.exec(
|
|
21360
|
+
function lookup(root, reference4) {
|
|
21361
|
+
const [, group, name] = /^#\/(\$defs|definitions)\/(.+)$/.exec(reference4) ?? [];
|
|
21362
21362
|
const defs = root[group ?? ""];
|
|
21363
21363
|
return name ? defs?.[name] : void 0;
|
|
21364
21364
|
}
|
|
@@ -22105,6 +22105,9 @@ function endsAtFiles(context2, named2, facts) {
|
|
|
22105
22105
|
if (pick?.id !== NO_STORE) return void 0;
|
|
22106
22106
|
return isAllowed(context2, "store", named2, pick) && located(pick.chosenBy);
|
|
22107
22107
|
}
|
|
22108
|
+
function hasChoice(context2, action, named2, facts) {
|
|
22109
|
+
return findPick(context2, action, named2, facts) !== void 0;
|
|
22110
|
+
}
|
|
22108
22111
|
function located({ at, ...rest }) {
|
|
22109
22112
|
const location = at?.document.locate(at.pointer);
|
|
22110
22113
|
return { ...rest, ...location && { file: location.file, line: location.line } };
|
|
@@ -22442,9 +22445,9 @@ function walkContract(context2, document, value, pointer) {
|
|
|
22442
22445
|
}
|
|
22443
22446
|
}
|
|
22444
22447
|
function offCapability(context2, document, at, path, uses) {
|
|
22445
|
-
const
|
|
22446
|
-
const needed =
|
|
22447
|
-
if (
|
|
22448
|
+
const used4 = uses.flatMap(({ use }) => use.use === "used" ? [use] : []);
|
|
22449
|
+
const needed = used4.map((use) => use.needs).find((id) => id && !isOn(context2, id));
|
|
22450
|
+
if (used4.length === 0 || !needed || used4.some((use) => !use.needs || isOn(context2, use.needs)))
|
|
22448
22451
|
return false;
|
|
22449
22452
|
warnOff(
|
|
22450
22453
|
context2,
|
|
@@ -22461,14 +22464,14 @@ function warnUnused(context2, document, at, path, parent) {
|
|
|
22461
22464
|
return use ? [{ pack: pack.id, use }] : [];
|
|
22462
22465
|
});
|
|
22463
22466
|
if (offCapability(context2, document, at, path, uses)) return true;
|
|
22464
|
-
const
|
|
22465
|
-
if (
|
|
22467
|
+
const notYet3 = uses.filter(({ use }) => use.use === "not-yet");
|
|
22468
|
+
if (notYet3.length === 0 || uses.some(({ use }) => use.use === "used")) return false;
|
|
22466
22469
|
const owner2 = typeof parent.name === "string" ? ` of "${parent.name}"` : "";
|
|
22467
22470
|
context2.reporter.warning(
|
|
22468
22471
|
"CONTRACT_FIELD_UNUSED",
|
|
22469
22472
|
`${at.at(-1)}${owner2} in contract "${document.id}" is not used yet, so it changes nothing.`,
|
|
22470
22473
|
{ document, pointer: `/${at.join("/")}` },
|
|
22471
|
-
|
|
22474
|
+
notYet3.map(({ pack, use }) => `${pack}: ${use.reason}`).join("\n")
|
|
22472
22475
|
);
|
|
22473
22476
|
return true;
|
|
22474
22477
|
}
|
|
@@ -22545,7 +22548,7 @@ function withExamples(schema, examples) {
|
|
|
22545
22548
|
}
|
|
22546
22549
|
|
|
22547
22550
|
// packages/definitions/src/settings-keys.ts
|
|
22548
|
-
var actions = ["ingest", "transform", "store"];
|
|
22551
|
+
var actions = ["ingest", "transform", "store", "serve"];
|
|
22549
22552
|
function checkSettingsKeys(context2) {
|
|
22550
22553
|
const { standard, platform, pack } = context2;
|
|
22551
22554
|
for (const action of actions) {
|
|
@@ -22634,8 +22637,8 @@ function oneStandard(project, available, reporter) {
|
|
|
22634
22637
|
return standard;
|
|
22635
22638
|
}
|
|
22636
22639
|
function selectPacks(standard, available, reporter) {
|
|
22637
|
-
return standard.value.packs.flatMap((
|
|
22638
|
-
const [id, version] =
|
|
22640
|
+
return standard.value.packs.flatMap((reference4, index) => {
|
|
22641
|
+
const [id, version] = reference4.split("@");
|
|
22639
22642
|
const at = { document: standard, pointer: `/packs/${index}` };
|
|
22640
22643
|
const pack = available.find((candidate) => candidate.id === id);
|
|
22641
22644
|
if (!pack) {
|
|
@@ -22650,7 +22653,7 @@ function selectPacks(standard, available, reporter) {
|
|
|
22650
22653
|
if (String(pack.version) !== version) {
|
|
22651
22654
|
reporter.error(
|
|
22652
22655
|
"WRONG_PACK",
|
|
22653
|
-
`This standard is for "${
|
|
22656
|
+
`This standard is for "${reference4}", but the pack is ${pack.id}@${pack.version}.`,
|
|
22654
22657
|
at
|
|
22655
22658
|
);
|
|
22656
22659
|
return [];
|
|
@@ -22674,7 +22677,7 @@ function methodOwners(standard, packs2, reporter) {
|
|
|
22674
22677
|
}
|
|
22675
22678
|
return owner2;
|
|
22676
22679
|
}
|
|
22677
|
-
var actions2 = ["ingest", "transform", "store"];
|
|
22680
|
+
var actions2 = ["ingest", "transform", "store", "serve"];
|
|
22678
22681
|
function combine(packs2) {
|
|
22679
22682
|
const [first] = packs2;
|
|
22680
22683
|
const rules = Object.fromEntries(
|
|
@@ -22698,10 +22701,10 @@ function combine(packs2) {
|
|
|
22698
22701
|
function createReporter(start = []) {
|
|
22699
22702
|
const diagnostics = [...start];
|
|
22700
22703
|
const seen = /* @__PURE__ */ new Set();
|
|
22701
|
-
const report = (severity,
|
|
22704
|
+
const report = (severity, code2, message, at, hint, fixes) => {
|
|
22702
22705
|
const location = at?.document.locate(at.pointer);
|
|
22703
22706
|
const diagnostic = {
|
|
22704
|
-
code,
|
|
22707
|
+
code: code2,
|
|
22705
22708
|
severity,
|
|
22706
22709
|
message,
|
|
22707
22710
|
...hint && { hint },
|
|
@@ -22716,7 +22719,7 @@ function createReporter(start = []) {
|
|
|
22716
22719
|
return {
|
|
22717
22720
|
diagnostics,
|
|
22718
22721
|
error: (...args) => report("error", ...args),
|
|
22719
|
-
warning: (
|
|
22722
|
+
warning: (code2, message, at, hint, fixes) => report("warning", code2, message, at, hint, fixes),
|
|
22720
22723
|
failed: () => diagnostics.some((diagnostic) => diagnostic.severity === "error")
|
|
22721
22724
|
};
|
|
22722
22725
|
}
|
|
@@ -22733,7 +22736,7 @@ var operators = [
|
|
|
22733
22736
|
"mustNotBeBetween"
|
|
22734
22737
|
];
|
|
22735
22738
|
var metrics = ["nullValues", "missingValues", "invalidValues", "duplicateValues", "rowCount"];
|
|
22736
|
-
function tableCheckOf(rule, name,
|
|
22739
|
+
function tableCheckOf(rule, name, column6, action) {
|
|
22737
22740
|
const operator = operators.find((key) => key in rule);
|
|
22738
22741
|
const metric = metricOf(rule);
|
|
22739
22742
|
if (!operator || !metric) return void 0;
|
|
@@ -22742,7 +22745,7 @@ function tableCheckOf(rule, name, column5, action) {
|
|
|
22742
22745
|
return {
|
|
22743
22746
|
name,
|
|
22744
22747
|
metric,
|
|
22745
|
-
columns:
|
|
22748
|
+
columns: column6 ? [column6] : combination,
|
|
22746
22749
|
arguments: args,
|
|
22747
22750
|
...metric === "sql" && { query: rule.query },
|
|
22748
22751
|
operator,
|
|
@@ -22772,14 +22775,14 @@ var options = {
|
|
|
22772
22775
|
maxItems: (name, value) => `size(${name}) <= ${literal(value)}`,
|
|
22773
22776
|
uniqueItems: (name, value) => value === true ? `size(array_distinct(${name})) = size(${name})` : ""
|
|
22774
22777
|
};
|
|
22775
|
-
function optionChecks(
|
|
22778
|
+
function optionChecks(column6, value) {
|
|
22776
22779
|
if (!isRecord2(value)) return [];
|
|
22777
|
-
const name = `\`${
|
|
22780
|
+
const name = `\`${column6}\``;
|
|
22778
22781
|
return Object.entries(value).flatMap(([option, limit]) => {
|
|
22779
22782
|
const check2 = options[option]?.(name, limit);
|
|
22780
22783
|
if (!check2) return [];
|
|
22781
22784
|
const suffix = option.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
|
22782
|
-
return [{ name: `${
|
|
22785
|
+
return [{ name: `${column6}_${suffix}`.toLowerCase(), check: `${name} IS NULL OR ${check2}` }];
|
|
22783
22786
|
});
|
|
22784
22787
|
}
|
|
22785
22788
|
function literal(value) {
|
|
@@ -22797,12 +22800,12 @@ function ruleChecks(context2, entry, at) {
|
|
|
22797
22800
|
const where = { ...at, pointer: `${at.pointer}/quality/${index}` };
|
|
22798
22801
|
overTable(context2, found, tableCheckOf(rule, name, void 0, action), rule, where);
|
|
22799
22802
|
});
|
|
22800
|
-
(entry.properties ?? []).forEach((property,
|
|
22803
|
+
(entry.properties ?? []).forEach((property, column6) => {
|
|
22801
22804
|
for (const check2 of optionChecks(property.name, property.logicalTypeOptions))
|
|
22802
22805
|
found.rows.push({ ...check2, name: unique(names, check2.name) });
|
|
22803
22806
|
rulesOf(property.quality).forEach((rule, index) => {
|
|
22804
22807
|
const name = unique(names, nameOf(property.name, rule));
|
|
22805
|
-
const where = { ...at, pointer: `${at.pointer}/properties/${
|
|
22808
|
+
const where = { ...at, pointer: `${at.pointer}/properties/${column6}/quality/${index}` };
|
|
22806
22809
|
const check2 = rowCheck(property.name, rule);
|
|
22807
22810
|
if (check2) found.rows.push({ name, check: check2 });
|
|
22808
22811
|
else overTable(context2, found, tableCheckOf(rule, name, property.name, action), rule, where);
|
|
@@ -22818,8 +22821,8 @@ function overTable(context2, found, check2, rule, at) {
|
|
|
22818
22821
|
if (check2 && isOn(context2, "checks")) found.table.push(check2);
|
|
22819
22822
|
else notChecked(context2, rule, at, check2 !== void 0);
|
|
22820
22823
|
}
|
|
22821
|
-
function uniqueCheck(name,
|
|
22822
|
-
const base = { name, metric: "duplicateValues", columns: [
|
|
22824
|
+
function uniqueCheck(name, column6, action) {
|
|
22825
|
+
const base = { name, metric: "duplicateValues", columns: [column6], arguments: {} };
|
|
22823
22826
|
return { ...base, operator: "mustBe", value: 0, unit: "rows", action };
|
|
22824
22827
|
}
|
|
22825
22828
|
function tableAction(context2) {
|
|
@@ -22829,8 +22832,8 @@ function tableAction(context2) {
|
|
|
22829
22832
|
function rulesOf(value) {
|
|
22830
22833
|
return Array.isArray(value) ? value.filter(isRecord2) : [];
|
|
22831
22834
|
}
|
|
22832
|
-
function nameOf(
|
|
22833
|
-
const given = typeof rule.name === "string" ? rule.name : `${
|
|
22835
|
+
function nameOf(column6, rule) {
|
|
22836
|
+
const given = typeof rule.name === "string" ? rule.name : `${column6}_${String(rule.metric)}`;
|
|
22834
22837
|
return given.toLowerCase().replace(/[^a-z0-9_]+/g, "_");
|
|
22835
22838
|
}
|
|
22836
22839
|
function unique(names, name) {
|
|
@@ -22843,10 +22846,10 @@ function allowsNone(rule) {
|
|
|
22843
22846
|
const rows = rule.unit === void 0 || rule.unit === "rows";
|
|
22844
22847
|
return rule.mustBe === 0 || rule.mustBeLessOrEqualTo === 0 || rows && rule.mustBeLessThan === 1;
|
|
22845
22848
|
}
|
|
22846
|
-
function rowCheck(
|
|
22849
|
+
function rowCheck(column6, rule) {
|
|
22847
22850
|
if ((rule.type ?? "library") !== "library" || !allowsNone(rule)) return void 0;
|
|
22848
22851
|
const args = isRecord2(rule.arguments) ? rule.arguments : {};
|
|
22849
|
-
return metrics2[String(rule.metric)]?.(`\`${
|
|
22852
|
+
return metrics2[String(rule.metric)]?.(`\`${column6}\``, args);
|
|
22850
22853
|
}
|
|
22851
22854
|
var metrics2 = {
|
|
22852
22855
|
nullValues: (name) => `${name} IS NOT NULL`,
|
|
@@ -22876,8 +22879,8 @@ function notCheckedAtFiles(context2, entry, at) {
|
|
|
22876
22879
|
};
|
|
22877
22880
|
rulesOf(entry.quality).forEach((rule, index) => warn(rule, `${at.pointer}/quality/${index}`));
|
|
22878
22881
|
(entry.properties ?? []).forEach(
|
|
22879
|
-
(property,
|
|
22880
|
-
(rule, index) => warn(rule, `${at.pointer}/properties/${
|
|
22882
|
+
(property, column6) => rulesOf(property.quality).forEach(
|
|
22883
|
+
(rule, index) => warn(rule, `${at.pointer}/properties/${column6}/quality/${index}`)
|
|
22881
22884
|
)
|
|
22882
22885
|
);
|
|
22883
22886
|
}
|
|
@@ -22978,15 +22981,15 @@ function expectationsNotChecked(context2, own, at) {
|
|
|
22978
22981
|
)
|
|
22979
22982
|
);
|
|
22980
22983
|
}
|
|
22981
|
-
function expectationsFor(context2,
|
|
22984
|
+
function expectationsFor(context2, contract4, own = []) {
|
|
22982
22985
|
const quality = { ...context2.pack.defaults.quality, ...context2.standard.value.quality };
|
|
22983
|
-
const required =
|
|
22984
|
-
name: `${
|
|
22985
|
-
check: `\`${
|
|
22986
|
+
const required = contract4.columns.filter((column6) => !column6.nullable).map((column6) => ({
|
|
22987
|
+
name: `${column6.name}_not_null`,
|
|
22988
|
+
check: `\`${column6.name}\` IS NOT NULL`,
|
|
22986
22989
|
action: quality.required,
|
|
22987
22990
|
from: "contract"
|
|
22988
22991
|
}));
|
|
22989
|
-
const rules =
|
|
22992
|
+
const rules = contract4.checks.map((item) => ({
|
|
22990
22993
|
...item,
|
|
22991
22994
|
action: quality.default,
|
|
22992
22995
|
from: "contract"
|
|
@@ -23004,11 +23007,11 @@ function expectationsFor(context2, contract3, own = []) {
|
|
|
23004
23007
|
}
|
|
23005
23008
|
|
|
23006
23009
|
// packages/definitions/src/method-fixes.ts
|
|
23007
|
-
function methodFixes(context2, choice, facts, named2,
|
|
23010
|
+
function methodFixes(context2, choice, facts, named2, code2) {
|
|
23008
23011
|
const flow = (id) => ({ file: named2.owner.file, pointer: "/ingest/uses", set: id });
|
|
23009
23012
|
const own = readers(context2.pack, facts).filter((id) => id !== choice.id).flatMap((id) => [
|
|
23010
23013
|
{ title: `Use "${id}" for this flow`, edits: [flow(id)] },
|
|
23011
|
-
standardFix(context2, choice, facts, id,
|
|
23014
|
+
standardFix(context2, choice, facts, id, code2)
|
|
23012
23015
|
]);
|
|
23013
23016
|
const others = context2.available.filter((pack) => !context2.packs.some((own2) => own2.id === pack.id)).flatMap(
|
|
23014
23017
|
(pack) => readers(pack, facts).map((id) => ({
|
|
@@ -23024,14 +23027,14 @@ function methodFixes(context2, choice, facts, named2, code) {
|
|
|
23024
23027
|
function readers(pack, facts) {
|
|
23025
23028
|
return Object.entries(pack.methods).filter(([, method]) => method.action === "ingest" && canServe(method, facts)).map(([id]) => id);
|
|
23026
23029
|
}
|
|
23027
|
-
function standardFix(context2, choice, facts, id,
|
|
23030
|
+
function standardFix(context2, choice, facts, id, code2) {
|
|
23028
23031
|
const rule = choice.origin && /\/when\/\d+\/use$/.test(choice.origin.pointer) ? choice.origin : void 0;
|
|
23029
23032
|
if (rule)
|
|
23030
23033
|
return {
|
|
23031
23034
|
title: `Use "${id}" in the standard's rule`,
|
|
23032
23035
|
edits: [{ file: rule.document.file, pointer: rule.pointer, set: id }]
|
|
23033
23036
|
};
|
|
23034
|
-
const key =
|
|
23037
|
+
const key = code2 === "CONNECTIVITY_NOT_SUPPORTED" ? "connectivity" : "connector";
|
|
23035
23038
|
return {
|
|
23036
23039
|
title: `Use "${id}" for every ${facts[key]} source`,
|
|
23037
23040
|
edits: [
|
|
@@ -23112,12 +23115,12 @@ function mergeSettings(context2, action, id, method, layers, user) {
|
|
|
23112
23115
|
fixed
|
|
23113
23116
|
];
|
|
23114
23117
|
if (!fixedRespected(context2, layers, fixed.value)) return void 0;
|
|
23115
|
-
const
|
|
23118
|
+
const settings2 = all.reduce(
|
|
23116
23119
|
(result, layer) => merge(result, layer.value),
|
|
23117
23120
|
{}
|
|
23118
23121
|
);
|
|
23119
|
-
if (!checkSchema2(context2, { action, id, method, user },
|
|
23120
|
-
return { settings, origins: originsOf(all) };
|
|
23122
|
+
if (!checkSchema2(context2, { action, id, method, user }, settings2, all)) return void 0;
|
|
23123
|
+
return { settings: settings2, origins: originsOf(all) };
|
|
23121
23124
|
}
|
|
23122
23125
|
function organisationLayers(context2, action, id) {
|
|
23123
23126
|
const rules = context2.standard.value[action];
|
|
@@ -23188,14 +23191,14 @@ function changedFixed(fixed, value, path) {
|
|
|
23188
23191
|
return sameValue(wanted, given) ? [] : [{ path: [...path, key], value: wanted }];
|
|
23189
23192
|
});
|
|
23190
23193
|
}
|
|
23191
|
-
function checkSchema2(context2, use,
|
|
23194
|
+
function checkSchema2(context2, use, settings2, layers) {
|
|
23192
23195
|
const { id, method } = use;
|
|
23193
23196
|
let validate = validators2.get(method);
|
|
23194
23197
|
if (!validate) {
|
|
23195
23198
|
validate = ajv.compile(method.settings);
|
|
23196
23199
|
validators2.set(method, validate);
|
|
23197
23200
|
}
|
|
23198
|
-
if (validate(
|
|
23201
|
+
if (validate(settings2)) return true;
|
|
23199
23202
|
const failures = validate.errors ?? [];
|
|
23200
23203
|
const missing = failures.filter((failure2) => failure2.keyword === "required" && failure2.instancePath === "").map((failure2) => String(failure2.params.missingProperty));
|
|
23201
23204
|
if (missing.length > 0) reportMissing(context2, use, missing);
|
|
@@ -23272,10 +23275,10 @@ function resolveStep(context2, action, named2, facts, tableLayers = [], previous
|
|
|
23272
23275
|
function canRead(context2, action, choice, facts, previous, named2) {
|
|
23273
23276
|
const problem2 = readProblem(action, choice, facts, previous, context2.pack.methods);
|
|
23274
23277
|
if (!problem2) return true;
|
|
23275
|
-
const { code } = problem2;
|
|
23276
|
-
const fixes =
|
|
23278
|
+
const { code: code2 } = problem2;
|
|
23279
|
+
const fixes = code2 === "CONNECTOR_NOT_SUPPORTED" || code2 === "CONNECTIVITY_NOT_SUPPORTED" ? methodFixes(context2, choice, facts, named2, code2) : [];
|
|
23277
23280
|
const at = choice.origin ?? { document: named2.owner, pointer: named2.at ?? "" };
|
|
23278
|
-
context2.reporter.error(
|
|
23281
|
+
context2.reporter.error(code2, problem2.message, at, problem2.hint, fixes);
|
|
23279
23282
|
return false;
|
|
23280
23283
|
}
|
|
23281
23284
|
function readProblem(action, choice, facts, previous, methods) {
|
|
@@ -23315,8 +23318,15 @@ function handoverProblem({ id, method }, previous) {
|
|
|
23315
23318
|
};
|
|
23316
23319
|
}
|
|
23317
23320
|
function stepOf(result) {
|
|
23318
|
-
const { method, pack, settings, origins, chosenBy } = result;
|
|
23319
|
-
return { method, pack, settings, origins, chosenBy };
|
|
23321
|
+
const { method, pack, settings: settings2, origins, chosenBy } = result;
|
|
23322
|
+
return { method, pack, settings: settings2, origins, chosenBy };
|
|
23323
|
+
}
|
|
23324
|
+
|
|
23325
|
+
// packages/definitions/src/serve.ts
|
|
23326
|
+
function serveStep(context2, named2, facts, last, layers = []) {
|
|
23327
|
+
const serveFacts = { ...facts, input: last.produces };
|
|
23328
|
+
if (!hasChoice(context2, "serve", named2, serveFacts)) return void 0;
|
|
23329
|
+
return resolveStep(context2, "serve", named2, serveFacts, layers, last) ?? false;
|
|
23320
23330
|
}
|
|
23321
23331
|
|
|
23322
23332
|
// packages/definitions/src/tables.ts
|
|
@@ -23398,34 +23408,35 @@ function flowTable(context2, flow, name, namer) {
|
|
|
23398
23408
|
if (!doc || !layer || !connection) return void 0;
|
|
23399
23409
|
const facts = { connectivity: connection.connectivity, connector: connection.connector, layer };
|
|
23400
23410
|
const steps = flowSteps(context2, flow, name, facts);
|
|
23401
|
-
const
|
|
23411
|
+
const contract4 = flowContract(context2, flow, name, steps?.endsAtFiles !== void 0);
|
|
23402
23412
|
const fullName = renderTableName(context2, namer, {
|
|
23403
23413
|
layer,
|
|
23404
23414
|
table: name,
|
|
23405
23415
|
flow: flow.value.id,
|
|
23406
23416
|
source: flow.value.source
|
|
23407
23417
|
});
|
|
23408
|
-
if (!steps || !
|
|
23418
|
+
if (!steps || !contract4 || !fullName) return void 0;
|
|
23409
23419
|
return {
|
|
23410
23420
|
name,
|
|
23411
23421
|
layer,
|
|
23412
23422
|
fullName,
|
|
23413
23423
|
origin: { kind: "Flow", id: flow.value.id, file: flow.file },
|
|
23414
|
-
contract:
|
|
23415
|
-
expectations: expectationsFor(context2,
|
|
23424
|
+
contract: contract4,
|
|
23425
|
+
expectations: expectationsFor(context2, contract4, doc.expectations),
|
|
23416
23426
|
...tableSteps(steps, connection, doc.object)
|
|
23417
23427
|
};
|
|
23418
23428
|
}
|
|
23419
|
-
function tableSteps(steps, connection,
|
|
23429
|
+
function tableSteps(steps, connection, object5) {
|
|
23420
23430
|
return {
|
|
23421
23431
|
ingest: {
|
|
23422
23432
|
...stepOf(steps.ingest),
|
|
23423
23433
|
connection: connectionOf(connection),
|
|
23424
|
-
object:
|
|
23434
|
+
object: object5,
|
|
23425
23435
|
renamed: steps.ingest.renames
|
|
23426
23436
|
},
|
|
23427
23437
|
...steps.store && { store: stepOf(steps.store) },
|
|
23428
|
-
...steps.endsAtFiles && { endsAtFiles: steps.endsAtFiles }
|
|
23438
|
+
...steps.endsAtFiles && { endsAtFiles: steps.endsAtFiles },
|
|
23439
|
+
...steps.serve && { serve: stepOf(steps.serve) }
|
|
23429
23440
|
};
|
|
23430
23441
|
}
|
|
23431
23442
|
function flowContract(context2, flow, name, atFiles) {
|
|
@@ -23435,6 +23446,20 @@ function flowContract(context2, flow, name, atFiles) {
|
|
|
23435
23446
|
return contractFor(context2, name, doc.contract, at("contract"), atFiles);
|
|
23436
23447
|
}
|
|
23437
23448
|
function flowSteps(context2, flow, name, facts) {
|
|
23449
|
+
const steps = keepSteps(context2, flow, name, facts);
|
|
23450
|
+
if (!steps) return void 0;
|
|
23451
|
+
const last = steps.store ?? steps.ingest;
|
|
23452
|
+
const serve = serveStep(
|
|
23453
|
+
context2,
|
|
23454
|
+
named(flow, "serve"),
|
|
23455
|
+
facts,
|
|
23456
|
+
last,
|
|
23457
|
+
tableLayer(flow, name, "serve")
|
|
23458
|
+
);
|
|
23459
|
+
if (serve === false) return void 0;
|
|
23460
|
+
return serve ? { ...steps, serve } : steps;
|
|
23461
|
+
}
|
|
23462
|
+
function keepSteps(context2, flow, name, facts) {
|
|
23438
23463
|
const ingest = resolveStep(
|
|
23439
23464
|
context2,
|
|
23440
23465
|
"ingest",
|
|
@@ -23464,7 +23489,7 @@ function afterTable(context2, flow, ingest) {
|
|
|
23464
23489
|
}
|
|
23465
23490
|
function named(flow, action) {
|
|
23466
23491
|
const doc = flow.value[action];
|
|
23467
|
-
return { owner: flow, doc, at: action
|
|
23492
|
+
return { owner: flow, doc, at: action !== "ingest" && doc ? `/${action}` : "/source" };
|
|
23468
23493
|
}
|
|
23469
23494
|
function reportStoreAfterTable(context2, flow, method) {
|
|
23470
23495
|
context2.reporter.error(
|
|
@@ -23580,10 +23605,10 @@ function scheduleOf(doc, mode) {
|
|
|
23580
23605
|
}
|
|
23581
23606
|
|
|
23582
23607
|
// packages/definitions/src/code-values.ts
|
|
23583
|
-
function codeWithValues(context2, document,
|
|
23608
|
+
function codeWithValues(context2, document, code2) {
|
|
23584
23609
|
const values2 = context2.project.values;
|
|
23585
23610
|
let complete = true;
|
|
23586
|
-
const replaced2 =
|
|
23611
|
+
const replaced2 = code2.replace(/\$\{values\.([A-Za-z_][\w.]*)\}/g, (written, name) => {
|
|
23587
23612
|
const value = valueAt(values2, name.split("."));
|
|
23588
23613
|
if (value !== void 0 && !isRecord2(value)) return String(value);
|
|
23589
23614
|
complete = false;
|
|
@@ -23672,13 +23697,13 @@ function transformTable(context2, namer, doc, resolved) {
|
|
|
23672
23697
|
const layer = namer.layerOf(value.layer, 1, { document: doc, pointer: "/layer" });
|
|
23673
23698
|
if (!layer) return void 0;
|
|
23674
23699
|
const steps = transformSteps(context2, doc, layer);
|
|
23675
|
-
const
|
|
23700
|
+
const contract4 = contractFor(context2, value.id, value.contract, {
|
|
23676
23701
|
document: doc,
|
|
23677
23702
|
pointer: "/contract"
|
|
23678
23703
|
});
|
|
23679
23704
|
const fullName = renderTableName(context2, namer, { layer, table: value.id });
|
|
23680
|
-
const
|
|
23681
|
-
if (!steps || !
|
|
23705
|
+
const code2 = codeWithValues(context2, doc, value.code);
|
|
23706
|
+
if (!steps || !contract4 || !fullName || code2 === void 0) return void 0;
|
|
23682
23707
|
if (!readsTables(context2, doc, resolved)) return void 0;
|
|
23683
23708
|
const inputs = Object.fromEntries(
|
|
23684
23709
|
value.inputs.map((input) => [input, inputName(resolved, input)])
|
|
@@ -23688,14 +23713,15 @@ function transformTable(context2, namer, doc, resolved) {
|
|
|
23688
23713
|
layer,
|
|
23689
23714
|
fullName,
|
|
23690
23715
|
origin: { kind: "Transform", id: value.id, file: doc.file },
|
|
23691
|
-
contract:
|
|
23692
|
-
expectations: expectationsFor(context2,
|
|
23716
|
+
contract: contract4,
|
|
23717
|
+
expectations: expectationsFor(context2, contract4, value.expectations),
|
|
23693
23718
|
transform: {
|
|
23694
23719
|
...stepOf(steps.transform),
|
|
23695
|
-
code,
|
|
23720
|
+
code: code2,
|
|
23696
23721
|
inputs
|
|
23697
23722
|
},
|
|
23698
|
-
store: stepOf(steps.store)
|
|
23723
|
+
store: stepOf(steps.store),
|
|
23724
|
+
...steps.serve && { serve: stepOf(steps.serve) }
|
|
23699
23725
|
};
|
|
23700
23726
|
}
|
|
23701
23727
|
function transformSteps(context2, doc, layer) {
|
|
@@ -23713,7 +23739,10 @@ function transformSteps(context2, doc, layer) {
|
|
|
23713
23739
|
[],
|
|
23714
23740
|
transform2
|
|
23715
23741
|
);
|
|
23716
|
-
|
|
23742
|
+
if (!transform2 || !store) return void 0;
|
|
23743
|
+
const serve = serveStep(context2, { owner: doc, doc: doc.value.serve }, { layer }, store);
|
|
23744
|
+
if (serve === false) return void 0;
|
|
23745
|
+
return serve ? { transform: transform2, store, serve } : { transform: transform2, store };
|
|
23717
23746
|
}
|
|
23718
23747
|
function readsTables(context2, doc, resolved) {
|
|
23719
23748
|
let tables = true;
|
|
@@ -23788,6 +23817,7 @@ function resolveLoaded(project, packs2, warnings) {
|
|
|
23788
23817
|
const tables = [...flows, ...transformTables(context2, namer, flows)];
|
|
23789
23818
|
checkDuplicateNames(context2, tables, (table) => originOf(context2, table));
|
|
23790
23819
|
recordOutputs(context2, tables);
|
|
23820
|
+
checkServeInputs(context2, tables);
|
|
23791
23821
|
runMethodChecks(context2, tables);
|
|
23792
23822
|
checkContractFields(context2, new Set(tables.map((table) => table.contract.id)));
|
|
23793
23823
|
const model = reporter.failed() ? void 0 : buildModel(context2, namer, tables);
|
|
@@ -23820,15 +23850,29 @@ function runPackCheck(context2, model) {
|
|
|
23820
23850
|
}
|
|
23821
23851
|
function recordOutputs(context2, tables) {
|
|
23822
23852
|
for (const table of tables) {
|
|
23823
|
-
for (const step of [table.ingest, table.transform, table.store]) {
|
|
23853
|
+
for (const step of [table.ingest, table.transform, table.store, table.serve]) {
|
|
23824
23854
|
const output = step && context2.pack.methods[step.method]?.output?.(table);
|
|
23825
23855
|
if (step && output) step.output = output;
|
|
23826
23856
|
}
|
|
23827
23857
|
}
|
|
23828
23858
|
}
|
|
23859
|
+
function checkServeInputs(context2, tables) {
|
|
23860
|
+
for (const table of tables) {
|
|
23861
|
+
const serve = table.serve;
|
|
23862
|
+
const last = table.store ?? table.ingest;
|
|
23863
|
+
if (!serve || !last || last.output || !context2.pack.methods[serve.method]?.needsOutput)
|
|
23864
|
+
continue;
|
|
23865
|
+
context2.reporter.error(
|
|
23866
|
+
"HANDOVER",
|
|
23867
|
+
`"${serve.method}" reads where "${last.method}" puts its data, and "${last.method}" does not say.`,
|
|
23868
|
+
originOf(context2, table),
|
|
23869
|
+
"Serve it with a method that does not need to know where its data lands."
|
|
23870
|
+
);
|
|
23871
|
+
}
|
|
23872
|
+
}
|
|
23829
23873
|
function runMethodChecks(context2, tables) {
|
|
23830
23874
|
for (const table of tables) {
|
|
23831
|
-
for (const step of [table.ingest, table.transform, table.store]) {
|
|
23875
|
+
for (const step of [table.ingest, table.transform, table.store, table.serve]) {
|
|
23832
23876
|
const method = step && context2.pack.methods[step.method];
|
|
23833
23877
|
for (const message of method?.check?.(table) ?? []) {
|
|
23834
23878
|
context2.reporter.error(
|
|
@@ -23856,8 +23900,8 @@ function digestOf(content) {
|
|
|
23856
23900
|
}
|
|
23857
23901
|
|
|
23858
23902
|
// packages/generate/src/problems.ts
|
|
23859
|
-
function problem(
|
|
23860
|
-
return path === void 0 ? { code, message } : { code, path, message };
|
|
23903
|
+
function problem(code2, message, path) {
|
|
23904
|
+
return path === void 0 ? { code: code2, message } : { code: code2, path, message };
|
|
23861
23905
|
}
|
|
23862
23906
|
|
|
23863
23907
|
// packages/generate/src/paths.ts
|
|
@@ -24121,8 +24165,8 @@ function formatPlan(plan, options2 = {}) {
|
|
|
24121
24165
|
const detail = change.change === "conflict" ? explain(change, plan) : diffOf(change, options2);
|
|
24122
24166
|
if (detail) lines.push(...detail.split("\n").map((line2) => indent + line2));
|
|
24123
24167
|
}
|
|
24124
|
-
for (const { code, path, message } of plan.problems) {
|
|
24125
|
-
lines.push(`${"problem".padEnd(10)}${path ?? ""}${path ? " " : ""}${
|
|
24168
|
+
for (const { code: code2, path, message } of plan.problems) {
|
|
24169
|
+
lines.push(`${"problem".padEnd(10)}${path ?? ""}${path ? " " : ""}${code2}`, indent + message);
|
|
24126
24170
|
}
|
|
24127
24171
|
lines.push("", summary(plan));
|
|
24128
24172
|
return lines.join("\n");
|
|
@@ -24354,42 +24398,42 @@ function sqlAuthentication(connector) {
|
|
|
24354
24398
|
}
|
|
24355
24399
|
function sqlConnectionProblems(table) {
|
|
24356
24400
|
const connection = table.ingest.connection;
|
|
24357
|
-
const
|
|
24358
|
-
const mode = String(
|
|
24401
|
+
const settings2 = connection.settings;
|
|
24402
|
+
const mode = String(settings2.authentication ?? "SQL");
|
|
24359
24403
|
const fields = modeFields[mode] ?? [];
|
|
24360
24404
|
const problems = [];
|
|
24361
24405
|
if (!sqlAuthentication(connection.connector).includes(mode))
|
|
24362
24406
|
problems.push(`the connection's authentication must be ${sqlAuthentication(connection.connector).join(", ")}`);
|
|
24363
24407
|
if (mode === "Windows" && table.ingest.settings.runtime_kind !== "self-hosted")
|
|
24364
24408
|
problems.push("Windows authentication requires runtime_kind: self-hosted");
|
|
24365
|
-
problems.push(...credentialFields(
|
|
24409
|
+
problems.push(...credentialFields(settings2, fields));
|
|
24366
24410
|
const allowed2 = ["server", "authentication", "host_name_in_certificate", ...fields];
|
|
24367
|
-
for (const key of Object.keys(
|
|
24411
|
+
for (const key of Object.keys(settings2))
|
|
24368
24412
|
if (!allowed2.includes(key))
|
|
24369
24413
|
problems.push(`the connection setting "${key}" is not supported for ${mode} authentication`);
|
|
24370
|
-
if (
|
|
24414
|
+
if (settings2.host_name_in_certificate !== void 0 && !nonEmpty(settings2.host_name_in_certificate))
|
|
24371
24415
|
problems.push('the connection needs non-empty "settings.host_name_in_certificate"');
|
|
24372
24416
|
return problems;
|
|
24373
24417
|
}
|
|
24374
|
-
function sqlCredentials(
|
|
24375
|
-
const authenticationType =
|
|
24418
|
+
function sqlCredentials(settings2, keyVault) {
|
|
24419
|
+
const authenticationType = settings2.authentication ?? "SQL";
|
|
24376
24420
|
if (authenticationType === "SQL" || authenticationType === "Windows")
|
|
24377
24421
|
return {
|
|
24378
24422
|
authenticationType,
|
|
24379
|
-
userName:
|
|
24380
|
-
password: vaultSecret(
|
|
24423
|
+
userName: settings2.user,
|
|
24424
|
+
password: vaultSecret(settings2.password, keyVault)
|
|
24381
24425
|
};
|
|
24382
24426
|
if (authenticationType === "ServicePrincipal")
|
|
24383
24427
|
return {
|
|
24384
24428
|
authenticationType,
|
|
24385
|
-
tenant:
|
|
24386
|
-
servicePrincipalId:
|
|
24387
|
-
servicePrincipalCredential: vaultSecret(
|
|
24429
|
+
tenant: settings2.tenant,
|
|
24430
|
+
servicePrincipalId: settings2.client_id,
|
|
24431
|
+
servicePrincipalCredential: vaultSecret(settings2.client_secret, keyVault)
|
|
24388
24432
|
};
|
|
24389
24433
|
if (authenticationType === "UserAssignedManagedIdentity")
|
|
24390
24434
|
return {
|
|
24391
24435
|
authenticationType,
|
|
24392
|
-
credential: { referenceName:
|
|
24436
|
+
credential: { referenceName: settings2.credential, type: "CredentialReference" }
|
|
24393
24437
|
};
|
|
24394
24438
|
return { authenticationType };
|
|
24395
24439
|
}
|
|
@@ -24403,7 +24447,7 @@ function nativeValue(value, field, keyVault) {
|
|
|
24403
24447
|
]));
|
|
24404
24448
|
return value;
|
|
24405
24449
|
}
|
|
24406
|
-
function credentialFields(
|
|
24450
|
+
function credentialFields(settings2, fields) {
|
|
24407
24451
|
const problems = [];
|
|
24408
24452
|
for (const key of [
|
|
24409
24453
|
...fields.filter((key2) => secretFields.has(key2)),
|
|
@@ -24411,9 +24455,9 @@ function credentialFields(settings, fields) {
|
|
|
24411
24455
|
...fields.filter((key2) => !secretFields.has(key2))
|
|
24412
24456
|
]) {
|
|
24413
24457
|
if (secretFields.has(key)) {
|
|
24414
|
-
if (!isSecret(
|
|
24458
|
+
if (!isSecret(settings2[key]))
|
|
24415
24459
|
problems.push(`the connection's ${key} must be \${secret:\u2026}, the name of a Key Vault secret`);
|
|
24416
|
-
} else if (!nonEmpty(
|
|
24460
|
+
} else if (!nonEmpty(settings2[key]))
|
|
24417
24461
|
problems.push(`the connection needs "settings.${key}"`);
|
|
24418
24462
|
}
|
|
24419
24463
|
return problems;
|
|
@@ -24470,29 +24514,29 @@ function replacesFile(table) {
|
|
|
24470
24514
|
const layout = layoutOf(table);
|
|
24471
24515
|
if (layout !== "path")
|
|
24472
24516
|
return layout === "snapshot";
|
|
24473
|
-
const
|
|
24474
|
-
if (
|
|
24517
|
+
const used4 = [...String(table.ingest.settings.path).matchAll(tokenPattern)].map((match) => match[1]);
|
|
24518
|
+
if (used4.includes("run"))
|
|
24475
24519
|
return false;
|
|
24476
24520
|
const frequency = table.ingest.settings.every?.frequency;
|
|
24477
24521
|
const needed = windowTokens[frequency ?? ""];
|
|
24478
|
-
return !needed || !needed.every((token2) =>
|
|
24522
|
+
return !needed || !needed.every((token2) => used4.includes(token2));
|
|
24479
24523
|
}
|
|
24480
24524
|
function pathProblems(path, extension) {
|
|
24481
24525
|
const quoted = `the path pattern "${path}"`;
|
|
24482
24526
|
if (path.endsWith("/") || path.split("/").at(-1) === "")
|
|
24483
24527
|
return [`${quoted} must end in a file name`];
|
|
24484
24528
|
const unknown = [...path.matchAll(tokenPattern)].map((match) => match[1] ?? "").filter((name) => !tokens.includes(name)).map((name) => `${quoted} uses an unknown token {${name}}; tokens: ${tokens.join(", ")}`);
|
|
24485
|
-
const
|
|
24529
|
+
const object5 = path.includes("{object}") ? [] : [`${quoted} must include {object}`];
|
|
24486
24530
|
const suffix = path.endsWith(`.${extension}`) ? [] : [`${quoted} must end in .${extension}`];
|
|
24487
|
-
return [...unknown, ...
|
|
24531
|
+
return [...unknown, ...object5, ...suffix];
|
|
24488
24532
|
}
|
|
24489
24533
|
function relativePath(table, extension, replace) {
|
|
24490
|
-
const
|
|
24534
|
+
const object5 = table.ingest.object;
|
|
24491
24535
|
const layout = layoutOf(table);
|
|
24492
24536
|
if (layout === "snapshot")
|
|
24493
|
-
return `${
|
|
24537
|
+
return `${object5}/${object5}.${extension}`;
|
|
24494
24538
|
if (layout === "increments")
|
|
24495
|
-
return `${
|
|
24539
|
+
return `${object5}/*.${extension}`;
|
|
24496
24540
|
return String(table.ingest.settings.path).replace(tokenPattern, (_match, name) => replace(name));
|
|
24497
24541
|
}
|
|
24498
24542
|
function expressionOf(table, token2) {
|
|
@@ -24635,11 +24679,11 @@ var layoutSettings = {
|
|
|
24635
24679
|
}
|
|
24636
24680
|
};
|
|
24637
24681
|
function credentialProblems(table, keys2) {
|
|
24638
|
-
const
|
|
24639
|
-
const password =
|
|
24682
|
+
const settings2 = table.ingest?.connection.settings ?? {};
|
|
24683
|
+
const password = settings2.password;
|
|
24640
24684
|
const problems = typeof password?.$secret === "string" ? [] : ["the connection's password must be ${secret:\u2026}, the name of a Key Vault secret"];
|
|
24641
24685
|
for (const key of keys2)
|
|
24642
|
-
if (typeof
|
|
24686
|
+
if (typeof settings2[key] !== "string")
|
|
24643
24687
|
problems.push(`the connection needs "settings.${key}"`);
|
|
24644
24688
|
return problems;
|
|
24645
24689
|
}
|
|
@@ -24776,16 +24820,16 @@ var authFields = {
|
|
|
24776
24820
|
Basic: ["user", "password"],
|
|
24777
24821
|
OAuth2ClientCredential: ["client_id", "client_secret", "token_endpoint"]
|
|
24778
24822
|
};
|
|
24779
|
-
function webAuthProblems(connector,
|
|
24780
|
-
const mode =
|
|
24823
|
+
function webAuthProblems(connector, settings2) {
|
|
24824
|
+
const mode = settings2.authentication ?? "Anonymous";
|
|
24781
24825
|
const fields = authFields[String(mode)] ?? [];
|
|
24782
24826
|
const problems = [];
|
|
24783
24827
|
if (connector === "http" && mode === "OAuth2ClientCredential")
|
|
24784
24828
|
problems.push("HTTP supports Anonymous or Basic authentication");
|
|
24785
24829
|
for (const field of fields)
|
|
24786
|
-
if (!validCredential(field,
|
|
24830
|
+
if (!validCredential(field, settings2[field]))
|
|
24787
24831
|
problems.push(`${mode} authentication requires settings.${field}`);
|
|
24788
|
-
for (const key of Object.keys(
|
|
24832
|
+
for (const key of Object.keys(settings2))
|
|
24789
24833
|
if (![
|
|
24790
24834
|
"url",
|
|
24791
24835
|
"auth_headers",
|
|
@@ -24801,15 +24845,15 @@ function validCredential(field, value) {
|
|
|
24801
24845
|
}
|
|
24802
24846
|
|
|
24803
24847
|
// node_modules/.pnpm/@ingestron+pack-adf@1.1.0_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-adf/dist/connection-check.js
|
|
24804
|
-
function connectionProblems(source,
|
|
24805
|
-
const problems = source.id === "rest" || source.id === "http" ? webAuthProblems(source.id,
|
|
24806
|
-
for (const key of Object.keys(
|
|
24848
|
+
function connectionProblems(source, settings2) {
|
|
24849
|
+
const problems = source.id === "rest" || source.id === "http" ? webAuthProblems(source.id, settings2) : [];
|
|
24850
|
+
for (const key of Object.keys(settings2))
|
|
24807
24851
|
if (!Object.hasOwn(source.fields, key))
|
|
24808
24852
|
problems.push(`the connection setting "${key}" is not supported for ${source.id}`);
|
|
24809
24853
|
for (const [key, field] of Object.entries(source.fields)) {
|
|
24810
|
-
if (
|
|
24854
|
+
if (settings2[key] === void 0 && field.optional)
|
|
24811
24855
|
continue;
|
|
24812
|
-
if (!validField(field,
|
|
24856
|
+
if (!validField(field, settings2[key]))
|
|
24813
24857
|
problems.push(`the connection needs "settings.${key}" as ${fieldDescription(field)}`);
|
|
24814
24858
|
}
|
|
24815
24859
|
return problems;
|
|
@@ -25213,17 +25257,17 @@ function sourceOptionNames(source) {
|
|
|
25213
25257
|
];
|
|
25214
25258
|
}
|
|
25215
25259
|
function sourceOptionProblems(table, source) {
|
|
25216
|
-
const
|
|
25217
|
-
const options2 =
|
|
25260
|
+
const settings2 = table.ingest.settings;
|
|
25261
|
+
const options2 = settings2.source_options ?? {};
|
|
25218
25262
|
const problems = Object.keys(options2).filter((key) => !sourceOptionNames(source).includes(key)).map((key) => `source_options.${key} is not supported for ${source.id}`);
|
|
25219
|
-
if (source.runtime &&
|
|
25263
|
+
if (source.runtime && settings2.runtime_kind !== source.runtime)
|
|
25220
25264
|
problems.push(`${source.id} requires runtime_kind: ${source.runtime}`);
|
|
25221
25265
|
if (source.id === "web-table" && options2.table_index === void 0)
|
|
25222
25266
|
problems.push("web-table requires source_options.table_index");
|
|
25223
25267
|
problems.push(...stagingProblems(source, options2), ...mappingProblems(table, options2));
|
|
25224
25268
|
if (source.id === "microsoft-365")
|
|
25225
25269
|
problems.push(...officeProblems(table, options2));
|
|
25226
|
-
else if (
|
|
25270
|
+
else if (settings2.format !== "parquet")
|
|
25227
25271
|
problems.push(`${source.id} requires format: parquet`);
|
|
25228
25272
|
return problems;
|
|
25229
25273
|
}
|
|
@@ -25237,7 +25281,7 @@ function officeProblems(table, options2) {
|
|
|
25237
25281
|
problems.push("this Microsoft 365 dataset requires date_filter_column");
|
|
25238
25282
|
if (options2.allowed_groups && options2.user_scope_filter)
|
|
25239
25283
|
problems.push("choose allowed_groups or user_scope_filter, not both");
|
|
25240
|
-
if (table.contract.columns.some((
|
|
25284
|
+
if (table.contract.columns.some((column6) => column6.physicalName && column6.physicalName !== column6.name))
|
|
25241
25285
|
problems.push("Microsoft 365 cannot rename columns during extraction; contract names must match source names");
|
|
25242
25286
|
return problems;
|
|
25243
25287
|
}
|
|
@@ -25250,7 +25294,7 @@ function stagingProblems(source, options2) {
|
|
|
25250
25294
|
return problems;
|
|
25251
25295
|
}
|
|
25252
25296
|
function mappingProblems(table, options2) {
|
|
25253
|
-
const columns = new Set(table.contract.columns.map((
|
|
25297
|
+
const columns = new Set(table.contract.columns.map((column6) => column6.name));
|
|
25254
25298
|
const problems = Object.keys(options2.column_paths ?? {}).filter((key) => !columns.has(key)).map((key) => `column_paths.${key} is not a contract column`);
|
|
25255
25299
|
problems.push(...webRequestProblems(options2));
|
|
25256
25300
|
return problems;
|
|
@@ -26129,7 +26173,7 @@ var bracket = (name) => `[${name.replaceAll("]", "]]")}]`;
|
|
|
26129
26173
|
function query(table) {
|
|
26130
26174
|
const ingest = table.ingest;
|
|
26131
26175
|
const [, schema, name] = ingest.object.split(".");
|
|
26132
|
-
const columns = table.contract.columns.map((
|
|
26176
|
+
const columns = table.contract.columns.map((column6) => column6.physicalName && column6.physicalName !== column6.name ? `${bracket(column6.physicalName)} AS ${bracket(column6.name)}` : bracket(column6.name)).join(", ");
|
|
26133
26177
|
const select = `SELECT ${columns} FROM ${bracket(schema ?? "")}.${bracket(name ?? "")}`;
|
|
26134
26178
|
if (ingest.settings.mode !== "incremental")
|
|
26135
26179
|
return select;
|
|
@@ -26153,7 +26197,7 @@ function landingService(shared, landing) {
|
|
|
26153
26197
|
properties: { type: "AzureBlobFS", typeProperties: { url: parseLanding(landing).url } }
|
|
26154
26198
|
};
|
|
26155
26199
|
}
|
|
26156
|
-
function sqlServerService(group, shared,
|
|
26200
|
+
function sqlServerService(group, shared, settings2) {
|
|
26157
26201
|
const connection = group.connection.settings;
|
|
26158
26202
|
return {
|
|
26159
26203
|
name: group.linkedService,
|
|
@@ -26162,14 +26206,14 @@ function sqlServerService(group, shared, settings) {
|
|
|
26162
26206
|
typeProperties: {
|
|
26163
26207
|
server: connection.server,
|
|
26164
26208
|
database: group.database,
|
|
26165
|
-
encrypt:
|
|
26209
|
+
encrypt: settings2.encrypt,
|
|
26166
26210
|
trustServerCertificate: false,
|
|
26167
26211
|
...sqlCredentials(connection, shared.keyVault),
|
|
26168
26212
|
...Boolean(connection.host_name_in_certificate) && {
|
|
26169
26213
|
hostNameInCertificate: connection.host_name_in_certificate
|
|
26170
26214
|
}
|
|
26171
26215
|
},
|
|
26172
|
-
connectVia: reference2(String(
|
|
26216
|
+
connectVia: reference2(String(settings2.integration_runtime), "IntegrationRuntimeReference")
|
|
26173
26217
|
}
|
|
26174
26218
|
};
|
|
26175
26219
|
}
|
|
@@ -26237,19 +26281,19 @@ function copyActivity(table, group, shared) {
|
|
|
26237
26281
|
};
|
|
26238
26282
|
}
|
|
26239
26283
|
function windowTrigger(model, group, pipeline, name) {
|
|
26240
|
-
const
|
|
26241
|
-
const every =
|
|
26284
|
+
const settings2 = group.tables[0].ingest.settings;
|
|
26285
|
+
const every = settings2.every;
|
|
26242
26286
|
return {
|
|
26243
26287
|
name,
|
|
26244
26288
|
properties: {
|
|
26245
26289
|
type: "TumblingWindowTrigger",
|
|
26246
|
-
runtimeState: paused(model,
|
|
26290
|
+
runtimeState: paused(model, settings2) ? "Stopped" : "Started",
|
|
26247
26291
|
typeProperties: {
|
|
26248
26292
|
frequency: every.frequency,
|
|
26249
26293
|
interval: every.interval,
|
|
26250
|
-
startTime:
|
|
26294
|
+
startTime: settings2.start,
|
|
26251
26295
|
maxConcurrency: 1,
|
|
26252
|
-
retryPolicy: { count:
|
|
26296
|
+
retryPolicy: { count: settings2.retries, intervalInSeconds: 300 }
|
|
26253
26297
|
},
|
|
26254
26298
|
pipeline: {
|
|
26255
26299
|
pipelineReference: reference2(pipeline, "PipelineReference"),
|
|
@@ -26261,19 +26305,19 @@ function windowTrigger(model, group, pipeline, name) {
|
|
|
26261
26305
|
}
|
|
26262
26306
|
};
|
|
26263
26307
|
}
|
|
26264
|
-
function paused(model,
|
|
26265
|
-
if (typeof
|
|
26266
|
-
return
|
|
26308
|
+
function paused(model, settings2) {
|
|
26309
|
+
if (typeof settings2.paused === "boolean")
|
|
26310
|
+
return settings2.paused;
|
|
26267
26311
|
return model.platform.schedule?.paused ?? model.platform.mode !== "production";
|
|
26268
26312
|
}
|
|
26269
26313
|
|
|
26270
26314
|
// node_modules/.pnpm/@ingestron+pack-adf@1.1.0_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-adf/dist/table-copy-options.js
|
|
26271
26315
|
function nativeSource(table, source) {
|
|
26272
|
-
const
|
|
26273
|
-
const options2 =
|
|
26316
|
+
const settings2 = table.ingest.settings;
|
|
26317
|
+
const options2 = settings2.source_options ?? {};
|
|
26274
26318
|
const result = { type: source.source };
|
|
26275
|
-
if (
|
|
26276
|
-
result[source.queryProperty ?? "query"] =
|
|
26319
|
+
if (settings2.query !== void 0)
|
|
26320
|
+
result[source.queryProperty ?? "query"] = settings2.query;
|
|
26277
26321
|
if (source.id === "sap-bw-open-hub")
|
|
26278
26322
|
result.excludeLastRequest = options2.exclude_last_request ?? false;
|
|
26279
26323
|
if (source.id === "snowflake")
|
|
@@ -26288,7 +26332,7 @@ function nativeSource(table, source) {
|
|
|
26288
26332
|
}
|
|
26289
26333
|
function officeSource(table, options2) {
|
|
26290
26334
|
return {
|
|
26291
|
-
outputColumns: table.contract.columns.map((
|
|
26335
|
+
outputColumns: table.contract.columns.map((column6) => ({ name: column6.name })),
|
|
26292
26336
|
...Boolean(options2.date_filter_column) && {
|
|
26293
26337
|
dateFilterColumn: options2.date_filter_column,
|
|
26294
26338
|
startTime: { value: "@pipeline().parameters.windowStart", type: "Expression" },
|
|
@@ -26321,11 +26365,11 @@ function tableTranslator(table, source) {
|
|
|
26321
26365
|
...Boolean(options2.collection_reference) && {
|
|
26322
26366
|
collectionReference: options2.collection_reference
|
|
26323
26367
|
},
|
|
26324
|
-
mappings: table.contract.columns.map((
|
|
26368
|
+
mappings: table.contract.columns.map((column6) => ({
|
|
26325
26369
|
source: source.hierarchical ? {
|
|
26326
|
-
path: options2.column_paths?.[
|
|
26327
|
-
} : { name:
|
|
26328
|
-
sink: { name:
|
|
26370
|
+
path: options2.column_paths?.[column6.name] ?? `${options2.collection_reference ? "" : "$"}['${(column6.physicalName ?? column6.name).replaceAll("\\", "\\\\").replaceAll("'", "\\'")}']`
|
|
26371
|
+
} : { name: column6.physicalName ?? column6.name },
|
|
26372
|
+
sink: { name: column6.name }
|
|
26329
26373
|
}))
|
|
26330
26374
|
}
|
|
26331
26375
|
};
|
|
@@ -26734,7 +26778,7 @@ var nativeFileConnectors = fileStores.map((source) => ({
|
|
|
26734
26778
|
maturity: { status: "GA", checked: "2026-10-28" },
|
|
26735
26779
|
docs: `https://learn.microsoft.com/en-us/azure/data-factory/connector-${source.docs}`
|
|
26736
26780
|
}));
|
|
26737
|
-
function nativeFileService(group, shared,
|
|
26781
|
+
function nativeFileService(group, shared, settings2) {
|
|
26738
26782
|
const source = fileStores.find((item) => item.id === group.connection.connector);
|
|
26739
26783
|
const typeProperties = { ...source.defaults };
|
|
26740
26784
|
for (const [key, field] of Object.entries(source.fields)) {
|
|
@@ -26747,8 +26791,8 @@ function nativeFileService(group, shared, settings) {
|
|
|
26747
26791
|
properties: {
|
|
26748
26792
|
type: source.service,
|
|
26749
26793
|
typeProperties,
|
|
26750
|
-
...
|
|
26751
|
-
connectVia: reference2(String(
|
|
26794
|
+
...settings2.runtime_kind === "azure" && settings2.integration_runtime === "AutoResolveIntegrationRuntime" ? {} : {
|
|
26795
|
+
connectVia: reference2(String(settings2.integration_runtime), "IntegrationRuntimeReference")
|
|
26752
26796
|
}
|
|
26753
26797
|
}
|
|
26754
26798
|
};
|
|
@@ -26766,11 +26810,11 @@ function nativeFileLocation(connector) {
|
|
|
26766
26810
|
folderPath: { value: "@dataset().folder", type: "Expression" }
|
|
26767
26811
|
};
|
|
26768
26812
|
}
|
|
26769
|
-
function nativeFileParameters(connector,
|
|
26813
|
+
function nativeFileParameters(connector, object5) {
|
|
26770
26814
|
const source = fileStores.find((item) => item.id === connector);
|
|
26771
26815
|
if (!source?.root)
|
|
26772
|
-
return { folder: connector === "files" ?
|
|
26773
|
-
const [root = "", ...parts] =
|
|
26816
|
+
return { folder: connector === "files" ? object5.replaceAll("/", "\\") : object5 };
|
|
26817
|
+
const [root = "", ...parts] = object5.split("/");
|
|
26774
26818
|
return { root, folder: parts.join("/") };
|
|
26775
26819
|
}
|
|
26776
26820
|
|
|
@@ -26851,9 +26895,9 @@ var connectors = [
|
|
|
26851
26895
|
|
|
26852
26896
|
// node_modules/.pnpm/@ingestron+pack-adf@1.1.0_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-adf/dist/files-resources.js
|
|
26853
26897
|
var folderParameter = { value: "@dataset().folder", type: "Expression" };
|
|
26854
|
-
function fileServerService(group, shared,
|
|
26898
|
+
function fileServerService(group, shared, settings2) {
|
|
26855
26899
|
if (group.connection.connector !== "files")
|
|
26856
|
-
return nativeFileService(group, shared,
|
|
26900
|
+
return nativeFileService(group, shared, settings2);
|
|
26857
26901
|
const connection = group.connection.settings;
|
|
26858
26902
|
const password = connection.password;
|
|
26859
26903
|
return {
|
|
@@ -26869,7 +26913,7 @@ function fileServerService(group, shared, settings) {
|
|
|
26869
26913
|
secretName: password.$secret
|
|
26870
26914
|
}
|
|
26871
26915
|
},
|
|
26872
|
-
connectVia: reference2(String(
|
|
26916
|
+
connectVia: reference2(String(settings2.integration_runtime), "IntegrationRuntimeReference")
|
|
26873
26917
|
}
|
|
26874
26918
|
};
|
|
26875
26919
|
}
|
|
@@ -26927,21 +26971,21 @@ function windowFilter(mode) {
|
|
|
26927
26971
|
function filesActivity(table, group, shared) {
|
|
26928
26972
|
const ingest = table.ingest;
|
|
26929
26973
|
const { path } = parseLanding(String(ingest.settings.landing));
|
|
26930
|
-
const
|
|
26974
|
+
const object5 = ingest.object.replace(/\/+$/, "");
|
|
26931
26975
|
return {
|
|
26932
26976
|
name: resourceName(table, "copy_activity", `copy_${table.name}`),
|
|
26933
26977
|
type: "Copy",
|
|
26934
26978
|
inputs: [
|
|
26935
26979
|
{
|
|
26936
26980
|
...reference2(group.dataset, "DatasetReference"),
|
|
26937
|
-
parameters: nativeFileParameters(group.connection.connector,
|
|
26981
|
+
parameters: nativeFileParameters(group.connection.connector, object5)
|
|
26938
26982
|
}
|
|
26939
26983
|
],
|
|
26940
26984
|
outputs: [
|
|
26941
26985
|
{
|
|
26942
26986
|
...reference2(group.connection.connector === "http" ? shared.landingHttp : shared.landingFiles, "DatasetReference"),
|
|
26943
26987
|
parameters: {
|
|
26944
|
-
folder: [path,
|
|
26988
|
+
folder: [path, object5].filter((part) => part !== "").join("/"),
|
|
26945
26989
|
...group.connection.connector === "http" && { file: ingest.settings.files }
|
|
26946
26990
|
}
|
|
26947
26991
|
}
|
|
@@ -27011,9 +27055,9 @@ function copyResources(model, copies, shared, files2) {
|
|
|
27011
27055
|
const project = model.project.id;
|
|
27012
27056
|
const add = (folder, resource) => addResource(files2, folder, resource);
|
|
27013
27057
|
for (const group of copies) {
|
|
27014
|
-
const
|
|
27058
|
+
const settings2 = group.tables[0].ingest.settings;
|
|
27015
27059
|
const name = `${project}_${group.connection.id}${group.database === group.connection.id ? "" : `_${group.database}`}`;
|
|
27016
|
-
add("linkedService", sqlServerService(group, shared,
|
|
27060
|
+
add("linkedService", sqlServerService(group, shared, settings2));
|
|
27017
27061
|
add("dataset", sourceDataset(group));
|
|
27018
27062
|
const pipeline = resourceName(group.tables, "pipeline", `pl_${name}`);
|
|
27019
27063
|
add("pipeline", copyPipeline(model, group, shared, pipeline));
|
|
@@ -27167,22 +27211,22 @@ function checkFiles(table) {
|
|
|
27167
27211
|
problems.push("secret references require key_vault in the adf-files settings");
|
|
27168
27212
|
if (!source)
|
|
27169
27213
|
problems.push(...Object.keys(connection.settings).filter((key) => !["path", "user", "password"].includes(key)).map((key) => `the connection setting "${key}" is not supported for files`));
|
|
27170
|
-
const
|
|
27171
|
-
if (!
|
|
27214
|
+
const object5 = table.ingest?.object ?? "";
|
|
27215
|
+
if (!object5.trim() || object5.split("/").some((part) => part === "." || part === ""))
|
|
27172
27216
|
problems.push("the source object must be a non-empty relative path");
|
|
27173
|
-
if (/^[\\/]|\\|@|(^|\/)\.\.(\/|$)/.test(
|
|
27174
|
-
problems.push(`a files object is a folder under the share written with "/", not "${
|
|
27217
|
+
if (/^[\\/]|\\|@|(^|\/)\.\.(\/|$)/.test(object5))
|
|
27218
|
+
problems.push(`a files object is a folder under the share written with "/", not "${object5}"`);
|
|
27175
27219
|
return problems;
|
|
27176
27220
|
}
|
|
27177
27221
|
function fileOptionsProblems(table) {
|
|
27178
27222
|
const connection = table.ingest.connection;
|
|
27179
27223
|
const problems = [];
|
|
27180
|
-
const
|
|
27181
|
-
if (["ftp", "http"].includes(connection.connector) &&
|
|
27224
|
+
const settings2 = table.ingest.settings;
|
|
27225
|
+
if (["ftp", "http"].includes(connection.connector) && settings2.mode !== "all")
|
|
27182
27226
|
problems.push(`${connection.connector} requires mode: all; native modified-time filtering is unavailable`);
|
|
27183
|
-
if (connection.connector === "http" && (!
|
|
27227
|
+
if (connection.connector === "http" && (!settings2.files || /[?*\\/@]/.test(String(settings2.files))))
|
|
27184
27228
|
problems.push("HTTP requires files to name one output file without wildcards");
|
|
27185
|
-
if (["files", "hdfs"].includes(connection.connector) &&
|
|
27229
|
+
if (["files", "hdfs"].includes(connection.connector) && settings2.runtime_kind !== "self-hosted")
|
|
27186
27230
|
problems.push(`${connection.connector} requires runtime_kind: self-hosted`);
|
|
27187
27231
|
return problems;
|
|
27188
27232
|
}
|
|
@@ -27264,9 +27308,9 @@ var adf = {
|
|
|
27264
27308
|
};
|
|
27265
27309
|
function checkShared(model) {
|
|
27266
27310
|
const tables = model.tables.filter((table) => /^adf-(copy|files|table)$/.test(table.ingest?.method ?? ""));
|
|
27267
|
-
const project = differing(tables, (
|
|
27268
|
-
const { url, fileSystem } = parseLanding(String(
|
|
27269
|
-
return [
|
|
27311
|
+
const project = differing(tables, (settings2) => {
|
|
27312
|
+
const { url, fileSystem } = parseLanding(String(settings2.landing));
|
|
27313
|
+
return [settings2.key_vault, url, fileSystem];
|
|
27270
27314
|
});
|
|
27271
27315
|
const problems = project ? [`every ADF table must use the same key_vault and landing storage; ${project}`] : [];
|
|
27272
27316
|
try {
|
|
@@ -27275,12 +27319,12 @@ function checkShared(model) {
|
|
|
27275
27319
|
return [...problems, error2.message];
|
|
27276
27320
|
}
|
|
27277
27321
|
for (const group of copyGroups(model)) {
|
|
27278
|
-
const different = differing(group.tables, (
|
|
27322
|
+
const different = differing(group.tables, (settings2) => groupKeys.map((key) => settings2[key]));
|
|
27279
27323
|
if (different)
|
|
27280
27324
|
problems.push(`the adf-copy tables of ${group.connection.id}.${group.database} share one linked service, pipeline and trigger, so ${groupKeys.slice(0, -1).join(", ")} and ${groupKeys.at(-1)} must be the same; ${different}`);
|
|
27281
27325
|
}
|
|
27282
27326
|
for (const group of filesGroups(model)) {
|
|
27283
|
-
const different = differing(group.tables, (
|
|
27327
|
+
const different = differing(group.tables, (settings2) => shareKeys.map((key) => settings2[key]));
|
|
27284
27328
|
if (different)
|
|
27285
27329
|
problems.push(`the adf-files tables of ${group.connection.id} share one linked service, pipeline and trigger, so ${shareKeys.slice(0, -1).join(", ")} and ${shareKeys.at(-1)} must be the same; ${different}`);
|
|
27286
27330
|
}
|
|
@@ -27321,9 +27365,9 @@ function landingConflicts(tables) {
|
|
|
27321
27365
|
function nativeGroupProblems(model) {
|
|
27322
27366
|
const problems = [];
|
|
27323
27367
|
for (const group of tableGroups(model)) {
|
|
27324
|
-
const different = differing(group, (
|
|
27325
|
-
...shareKeys.map((key) =>
|
|
27326
|
-
|
|
27368
|
+
const different = differing(group, (settings2) => [
|
|
27369
|
+
...shareKeys.map((key) => settings2[key]),
|
|
27370
|
+
settings2.source_options?.staging_sas
|
|
27327
27371
|
]);
|
|
27328
27372
|
if (different)
|
|
27329
27373
|
problems.push(`the adf-table tables of ${group[0].ingest.connection.id} share one runtime and schedule; ${different}`);
|
|
@@ -27625,7 +27669,7 @@ var lakeflowConnect = {
|
|
|
27625
27669
|
check: checkConnect
|
|
27626
27670
|
};
|
|
27627
27671
|
function renamesOf(table) {
|
|
27628
|
-
return table.contract.columns.filter((
|
|
27672
|
+
return table.contract.columns.filter((column6) => column6.physicalName && column6.physicalName !== column6.name).map((column6) => `Lakeflow Connect keeps the source's column names, so it cannot read "${column6.physicalName}" as "${column6.name}"; remove physicalName, or rename the column in a transform`);
|
|
27629
27673
|
}
|
|
27630
27674
|
function checkConnect(table) {
|
|
27631
27675
|
const ingest = table.ingest;
|
|
@@ -27650,10 +27694,10 @@ function checkConnect(table) {
|
|
|
27650
27694
|
// node_modules/.pnpm/@ingestron+pack-databricks@1.0.0_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-databricks/dist/query.js
|
|
27651
27695
|
var import_liquidjs2 = __toESM(require_liquid_node(), 1);
|
|
27652
27696
|
var engine2 = new import_liquidjs2.Liquid({ strictVariables: true, strictFilters: true });
|
|
27653
|
-
function renderQuery(
|
|
27697
|
+
function renderQuery(code2, inputs) {
|
|
27654
27698
|
const quoted = Object.fromEntries(Object.entries(inputs).map(([name, full]) => [name, quoteName(full)]));
|
|
27655
27699
|
try {
|
|
27656
|
-
const sql3 = engine2.parseAndRenderSync(
|
|
27700
|
+
const sql3 = engine2.parseAndRenderSync(code2, { tables: quoted });
|
|
27657
27701
|
return { sql: sql3.trim().replace(/;\s*$/, "") };
|
|
27658
27702
|
} catch (cause) {
|
|
27659
27703
|
const reason = cause instanceof Error ? cause.message.split("\n")[0] : String(cause);
|
|
@@ -27772,24 +27816,24 @@ var scdTypes = {
|
|
|
27772
27816
|
history: "SCD_TYPE_2"
|
|
27773
27817
|
};
|
|
27774
27818
|
function tableConfiguration(table) {
|
|
27775
|
-
const
|
|
27776
|
-
const keyed =
|
|
27819
|
+
const settings2 = table.ingest.settings;
|
|
27820
|
+
const keyed = settings2.mode !== "append" && table.contract.primaryKey.length > 0;
|
|
27777
27821
|
const kafka = table.ingest.connection.connector === "kafka";
|
|
27778
27822
|
return {
|
|
27779
|
-
scd_type: scdTypes[String(
|
|
27780
|
-
...
|
|
27823
|
+
scd_type: scdTypes[String(settings2.mode)],
|
|
27824
|
+
...settings2.columns === "contract" && !kafka && { include_columns: table.contract.columns.map((column6) => column6.name) },
|
|
27781
27825
|
...keyed && { primary_keys: table.contract.primaryKey }
|
|
27782
27826
|
};
|
|
27783
27827
|
}
|
|
27784
27828
|
function kafkaOptions(table) {
|
|
27785
|
-
const
|
|
27786
|
-
const format = String(
|
|
27787
|
-
const hints = table.contract.columns.filter((
|
|
27829
|
+
const settings2 = table.ingest.settings;
|
|
27830
|
+
const format = String(settings2.value_format ?? "json");
|
|
27831
|
+
const hints = table.contract.columns.filter((column6) => column6.type).map((column6) => `${quoteName(column6.name)} ${column6.type}`);
|
|
27788
27832
|
const json = format === "json" && hints.length > 0 ? { json_options: { schema_hints: hints.join(", ") } } : {};
|
|
27789
27833
|
return {
|
|
27790
27834
|
topics: [table.ingest.object],
|
|
27791
|
-
...typeof
|
|
27792
|
-
starting_offset:
|
|
27835
|
+
...typeof settings2.starting_offset === "string" && {
|
|
27836
|
+
starting_offset: settings2.starting_offset
|
|
27793
27837
|
},
|
|
27794
27838
|
value_transformer: { format: format.toUpperCase(), ...json }
|
|
27795
27839
|
};
|
|
@@ -27806,7 +27850,7 @@ function sqlString(text2) {
|
|
|
27806
27850
|
return `'${text2.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
|
|
27807
27851
|
}
|
|
27808
27852
|
function tableData(table, kind) {
|
|
27809
|
-
const
|
|
27853
|
+
const settings2 = table.store?.settings ?? {};
|
|
27810
27854
|
return {
|
|
27811
27855
|
kind,
|
|
27812
27856
|
target: quoteName(fullNameText(table.fullName)),
|
|
@@ -27816,15 +27860,15 @@ function tableData(table, kind) {
|
|
|
27816
27860
|
on_violation: onViolation[item.action]
|
|
27817
27861
|
})),
|
|
27818
27862
|
comment: table.contract.description ? sqlString(table.contract.description) : "",
|
|
27819
|
-
cluster: clusterOf(
|
|
27820
|
-
properties: Object.entries(
|
|
27863
|
+
cluster: clusterOf(settings2.cluster_by),
|
|
27864
|
+
properties: Object.entries(settings2.properties ?? {}).map(([key, value]) => ({ key: sqlString(key), value: sqlString(value) }))
|
|
27821
27865
|
};
|
|
27822
27866
|
}
|
|
27823
27867
|
function clusterOf(value) {
|
|
27824
27868
|
if (value === "auto")
|
|
27825
27869
|
return "AUTO";
|
|
27826
27870
|
if (Array.isArray(value))
|
|
27827
|
-
return `(${value.map((
|
|
27871
|
+
return `(${value.map((column6) => quoteName(String(column6))).join(", ")})`;
|
|
27828
27872
|
return "";
|
|
27829
27873
|
}
|
|
27830
27874
|
function sourceData(table, stream) {
|
|
@@ -27838,60 +27882,60 @@ function sourceData(table, stream) {
|
|
|
27838
27882
|
function renamedColumns(table) {
|
|
27839
27883
|
if (table.ingest?.renamed)
|
|
27840
27884
|
return [];
|
|
27841
|
-
return table.contract.columns.filter((
|
|
27885
|
+
return table.contract.columns.filter((column6) => column6.physicalName && column6.physicalName !== column6.name);
|
|
27842
27886
|
}
|
|
27843
27887
|
function selectOf(table) {
|
|
27844
27888
|
const renamed = renamedColumns(table);
|
|
27845
27889
|
if (renamed.length === 0)
|
|
27846
27890
|
return "*";
|
|
27847
|
-
const sources = renamed.map((
|
|
27848
|
-
const aliases = renamed.map((
|
|
27891
|
+
const sources = renamed.map((column6) => quoteName(column6.physicalName));
|
|
27892
|
+
const aliases = renamed.map((column6) => `${quoteName(column6.physicalName)} AS ${quoteName(column6.name)}`);
|
|
27849
27893
|
return `* EXCEPT (${sources.join(", ")}), ${aliases.join(", ")}`;
|
|
27850
27894
|
}
|
|
27851
27895
|
function pathOf(table) {
|
|
27852
27896
|
const ingest = table.ingest;
|
|
27853
27897
|
const folder = ingest?.settings.landing ?? ingest?.connection.settings.path ?? "";
|
|
27854
27898
|
const base = String(folder).replace(/\/+$/, "");
|
|
27855
|
-
const
|
|
27856
|
-
return `${base}/${
|
|
27899
|
+
const object5 = (table.ingest?.object ?? "").replace(/^\/+/, "");
|
|
27900
|
+
return `${base}/${object5}`;
|
|
27857
27901
|
}
|
|
27858
27902
|
function optionsOf(table) {
|
|
27859
|
-
const
|
|
27903
|
+
const settings2 = table.ingest?.settings ?? {};
|
|
27860
27904
|
const options2 = [
|
|
27861
|
-
{ key: "format", value: sqlString(String(
|
|
27905
|
+
{ key: "format", value: sqlString(String(settings2.format)) }
|
|
27862
27906
|
];
|
|
27863
27907
|
const source = new Set(renamedColumns(table));
|
|
27864
|
-
const hints = table.contract.columns.filter((
|
|
27865
|
-
const name = source.has(
|
|
27866
|
-
return `${quoteName(name)} ${
|
|
27908
|
+
const hints = table.contract.columns.filter((column6) => column6.type).map((column6) => {
|
|
27909
|
+
const name = source.has(column6) ? column6.physicalName : column6.name;
|
|
27910
|
+
return `${quoteName(name)} ${column6.type}`;
|
|
27867
27911
|
});
|
|
27868
|
-
if (
|
|
27912
|
+
if (settings2.schema_hints === true && hints.length > 0)
|
|
27869
27913
|
options2.push({ key: "schemaHints", value: sqlString(hints.join(", ")) });
|
|
27870
|
-
if (
|
|
27914
|
+
if (settings2.schema_evolution)
|
|
27871
27915
|
options2.push({
|
|
27872
27916
|
key: "schemaEvolutionMode",
|
|
27873
|
-
value: sqlString(String(
|
|
27917
|
+
value: sqlString(String(settings2.schema_evolution))
|
|
27874
27918
|
});
|
|
27875
|
-
for (const [key, value] of Object.entries(
|
|
27919
|
+
for (const [key, value] of Object.entries(settings2.options ?? {})) {
|
|
27876
27920
|
options2.push({ key, value: typeof value === "string" ? sqlString(value) : String(value) });
|
|
27877
27921
|
}
|
|
27878
27922
|
return options2;
|
|
27879
27923
|
}
|
|
27880
27924
|
function changeData(table) {
|
|
27881
|
-
const
|
|
27925
|
+
const settings2 = table.store?.settings ?? {};
|
|
27882
27926
|
const changes = {
|
|
27883
27927
|
...table.fullName,
|
|
27884
|
-
table: `${table.fullName.table}${String(
|
|
27928
|
+
table: `${table.fullName.table}${String(settings2.changes_suffix)}`
|
|
27885
27929
|
};
|
|
27886
|
-
const except =
|
|
27930
|
+
const except = settings2.except_columns ?? [];
|
|
27887
27931
|
return {
|
|
27888
27932
|
changes: quoteName(fullNameText(changes)),
|
|
27889
27933
|
flow: quoteName(`${table.fullName.table}_cdc`),
|
|
27890
27934
|
keys: table.contract.primaryKey.map(quoteName).join(", "),
|
|
27891
|
-
sequence: quoteName(String(
|
|
27892
|
-
delete_when: typeof
|
|
27935
|
+
sequence: quoteName(String(settings2.sequence_by)),
|
|
27936
|
+
delete_when: typeof settings2.delete_when === "string" ? settings2.delete_when : "",
|
|
27893
27937
|
except_columns: except.map(quoteName).join(", "),
|
|
27894
|
-
scd:
|
|
27938
|
+
scd: settings2.mode === "history" ? 2 : 1
|
|
27895
27939
|
};
|
|
27896
27940
|
}
|
|
27897
27941
|
|
|
@@ -27919,18 +27963,18 @@ var tagList = (tags2) => tags2.map(tagParts).map(([key, value]) => `${sqlString(
|
|
|
27919
27963
|
function tagStatements(table) {
|
|
27920
27964
|
const alter = `ALTER ${kindOf(table)} ${nameOf2(table)}`;
|
|
27921
27965
|
const own = table.contract.tags.length > 0 ? [`${alter} SET TAGS (${tagList(table.contract.tags)})`] : [];
|
|
27922
|
-
const columns = table.contract.columns.flatMap((
|
|
27966
|
+
const columns = table.contract.columns.flatMap((column6) => {
|
|
27923
27967
|
const tags2 = [
|
|
27924
|
-
...
|
|
27925
|
-
...
|
|
27968
|
+
...column6.tags ?? [],
|
|
27969
|
+
...column6.classification ? [`classification:${column6.classification}`] : []
|
|
27926
27970
|
];
|
|
27927
|
-
return tags2.length > 0 ? [`${alter} ALTER COLUMN ${quoteName(
|
|
27971
|
+
return tags2.length > 0 ? [`${alter} ALTER COLUMN ${quoteName(column6.name)} SET TAGS (${tagList(tags2)})`] : [];
|
|
27928
27972
|
});
|
|
27929
27973
|
return [...own, ...columns];
|
|
27930
27974
|
}
|
|
27931
27975
|
function commentStatements(table) {
|
|
27932
|
-
return table.contract.columns.flatMap((
|
|
27933
|
-
`COMMENT ON COLUMN ${nameOf2(table)}.${quoteName(
|
|
27976
|
+
return table.contract.columns.flatMap((column6) => column6.description ? [
|
|
27977
|
+
`COMMENT ON COLUMN ${nameOf2(table)}.${quoteName(column6.name)} IS ${sqlString(column6.description)}`
|
|
27934
27978
|
] : []);
|
|
27935
27979
|
}
|
|
27936
27980
|
function checkRuns(table) {
|
|
@@ -27953,25 +27997,25 @@ function countQuery(check2, from) {
|
|
|
27953
27997
|
}
|
|
27954
27998
|
function countOf(check2) {
|
|
27955
27999
|
const columns = check2.columns.map(quoteName);
|
|
27956
|
-
const [
|
|
28000
|
+
const [column6 = "*"] = columns;
|
|
27957
28001
|
switch (check2.metric) {
|
|
27958
28002
|
case "rowCount":
|
|
27959
28003
|
return "count(*)";
|
|
27960
28004
|
case "duplicateValues":
|
|
27961
28005
|
return `count(*) - count(DISTINCT ${columns.length > 0 ? columns.join(", ") : "*"})`;
|
|
27962
28006
|
case "nullValues":
|
|
27963
|
-
return `count_if(${
|
|
28007
|
+
return `count_if(${column6} IS NULL)`;
|
|
27964
28008
|
case "missingValues":
|
|
27965
|
-
return Array.isArray(check2.arguments.missingValues) ? `count_if(${
|
|
28009
|
+
return Array.isArray(check2.arguments.missingValues) ? `count_if(${column6} IS NULL OR ${column6} IN (${valuesOf(check2.arguments.missingValues)}))` : `count_if(${column6} IS NULL)`;
|
|
27966
28010
|
default:
|
|
27967
|
-
return `count_if(${
|
|
28011
|
+
return `count_if(${column6} IS NOT NULL AND NOT ${validOf(check2, column6)})`;
|
|
27968
28012
|
}
|
|
27969
28013
|
}
|
|
27970
|
-
function validOf(check2,
|
|
28014
|
+
function validOf(check2, column6) {
|
|
27971
28015
|
const { validValues, pattern: pattern3 } = check2.arguments;
|
|
27972
28016
|
if (typeof pattern3 === "string")
|
|
27973
|
-
return `regexp_like(${
|
|
27974
|
-
return `${
|
|
28017
|
+
return `regexp_like(${column6}, ${sqlString(pattern3)})`;
|
|
28018
|
+
return `${column6} IN (${valuesOf(validValues)})`;
|
|
27975
28019
|
}
|
|
27976
28020
|
function valuesOf(values2) {
|
|
27977
28021
|
const list2 = Array.isArray(values2) && values2.length > 0 ? values2 : [""];
|
|
@@ -28018,7 +28062,7 @@ function tagProblems(model) {
|
|
|
28018
28062
|
return [];
|
|
28019
28063
|
const tags2 = model.tables.flatMap((table) => [
|
|
28020
28064
|
...table.contract.tags,
|
|
28021
|
-
...table.contract.columns.flatMap((
|
|
28065
|
+
...table.contract.columns.flatMap((column6) => column6.tags ?? [])
|
|
28022
28066
|
]);
|
|
28023
28067
|
return [...new Set(tags2)].filter((tag) => /[.,\-=/:]/.test(tagParts(tag)[0]) || tagParts(tag)[0].length > 255).map((tag) => `the tag "${tag}" has a key Unity Catalog does not allow: keys have no . , - = / : and at most 255 characters; write key:value for a value`);
|
|
28024
28068
|
}
|
|
@@ -28412,14 +28456,14 @@ var sql = {
|
|
|
28412
28456
|
}
|
|
28413
28457
|
};
|
|
28414
28458
|
function checkChangeData(table) {
|
|
28415
|
-
const
|
|
28416
|
-
if (
|
|
28459
|
+
const settings2 = table.store?.settings ?? {};
|
|
28460
|
+
if (settings2.mode === "append")
|
|
28417
28461
|
return [];
|
|
28418
28462
|
const problems = [];
|
|
28419
28463
|
if (table.contract.primaryKey.length === 0)
|
|
28420
|
-
problems.push(`mode ${String(
|
|
28421
|
-
if (typeof
|
|
28422
|
-
problems.push(`mode ${String(
|
|
28464
|
+
problems.push(`mode ${String(settings2.mode)} needs primary key columns in the contract`);
|
|
28465
|
+
if (typeof settings2.sequence_by !== "string")
|
|
28466
|
+
problems.push(`mode ${String(settings2.mode)} needs "sequence_by"`);
|
|
28423
28467
|
return problems;
|
|
28424
28468
|
}
|
|
28425
28469
|
|
|
@@ -28443,17 +28487,17 @@ var pyspark = {
|
|
|
28443
28487
|
}
|
|
28444
28488
|
},
|
|
28445
28489
|
check: (table) => {
|
|
28446
|
-
const
|
|
28490
|
+
const code2 = table.transform?.code ?? "";
|
|
28447
28491
|
const name = String(table.transform?.settings.function);
|
|
28448
|
-
const defined = new RegExp(`^def ${name}\\(`, "m").test(
|
|
28492
|
+
const defined = new RegExp(`^def ${name}\\(`, "m").test(code2);
|
|
28449
28493
|
return [
|
|
28450
28494
|
...defined ? [] : [`the code has no function "def ${name}(tables)"`],
|
|
28451
|
-
...unknownInputs(
|
|
28495
|
+
...unknownInputs(code2, Object.keys(table.transform?.inputs ?? {}))
|
|
28452
28496
|
];
|
|
28453
28497
|
}
|
|
28454
28498
|
};
|
|
28455
|
-
function unknownInputs(
|
|
28456
|
-
const lines =
|
|
28499
|
+
function unknownInputs(code2, inputs) {
|
|
28500
|
+
const lines = code2.split("\n").map((line2) => line2.replace(/#.*/, ""));
|
|
28457
28501
|
const read = [...lines.join("\n").matchAll(/\btables\[\s*["']([^"']+)["']\s*\]/g)].map((match) => match[1]);
|
|
28458
28502
|
return [...new Set(read)].filter((key) => !inputs.includes(key)).map((key) => `the code reads tables["${key}"], which is not an input; the inputs are ${inputs.map((input) => `"${input}"`).join(", ") || "none"}`);
|
|
28459
28503
|
}
|
|
@@ -28516,8 +28560,357 @@ var databricks = {
|
|
|
28516
28560
|
request: request2
|
|
28517
28561
|
};
|
|
28518
28562
|
|
|
28563
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/locations.js
|
|
28564
|
+
function sourceStep(table) {
|
|
28565
|
+
return table.serve ? table.store ?? table.ingest : table.ingest;
|
|
28566
|
+
}
|
|
28567
|
+
function relativePath2(output, settings2) {
|
|
28568
|
+
const root = String(settings2.data_source_location ?? "").replace(/\/+$/, "");
|
|
28569
|
+
if (!output.location.startsWith(`${root}/`))
|
|
28570
|
+
return {
|
|
28571
|
+
problem: `${output.location} is not under the data source ${String(settings2.data_source)} (${root})`
|
|
28572
|
+
};
|
|
28573
|
+
return { path: output.location.slice(root.length + 1) };
|
|
28574
|
+
}
|
|
28575
|
+
function filesProblems(table, settings2) {
|
|
28576
|
+
const output = sourceStep(table)?.output;
|
|
28577
|
+
if (!output)
|
|
28578
|
+
return [];
|
|
28579
|
+
if (output.format !== "parquet" && output.format !== "csv")
|
|
28580
|
+
return [`it reads parquet and csv files, not ${output.format}`];
|
|
28581
|
+
const relative10 = relativePath2(output, settings2);
|
|
28582
|
+
return "problem" in relative10 ? [relative10.problem] : [];
|
|
28583
|
+
}
|
|
28584
|
+
|
|
28585
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/types.js
|
|
28586
|
+
var synapseTypes = {
|
|
28587
|
+
string: "VARCHAR(8000) COLLATE Latin1_General_100_BIN2_UTF8",
|
|
28588
|
+
integer: "BIGINT",
|
|
28589
|
+
number: "FLOAT",
|
|
28590
|
+
boolean: "BIT",
|
|
28591
|
+
date: "DATE",
|
|
28592
|
+
timestamp: "DATETIME2(7)"
|
|
28593
|
+
};
|
|
28594
|
+
|
|
28595
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/sql.js
|
|
28596
|
+
var fallbackType = synapseTypes.string;
|
|
28597
|
+
function quote(name) {
|
|
28598
|
+
return `[${name.replaceAll("]", "]]")}]`;
|
|
28599
|
+
}
|
|
28600
|
+
function columnList(columns, options2 = {}) {
|
|
28601
|
+
return columns.map((column6) => ` ${quote(fileName(column6, options2))} ${typeOf(column6, options2)}`).join(",\n");
|
|
28602
|
+
}
|
|
28603
|
+
function selectList(columns, options2) {
|
|
28604
|
+
if (columns.every((column6) => fileName(column6, options2) === column6.name))
|
|
28605
|
+
return "*";
|
|
28606
|
+
return columns.map((column6) => {
|
|
28607
|
+
const file = fileName(column6, options2);
|
|
28608
|
+
return file === column6.name ? quote(column6.name) : `${quote(file)} AS ${quote(column6.name)}`;
|
|
28609
|
+
}).join(", ");
|
|
28610
|
+
}
|
|
28611
|
+
function fileName(column6, { renamed = true }) {
|
|
28612
|
+
return !renamed && column6.physicalName ? column6.physicalName : column6.name;
|
|
28613
|
+
}
|
|
28614
|
+
function typeOf(column6, { resolvedTypes = false }) {
|
|
28615
|
+
const mapped = column6.logicalType ? synapseTypes[column6.logicalType] : void 0;
|
|
28616
|
+
const type = resolvedTypes ? column6.type ?? mapped : mapped ?? column6.type;
|
|
28617
|
+
return type ?? fallbackType;
|
|
28618
|
+
}
|
|
28619
|
+
function filesQuery(input) {
|
|
28620
|
+
const options2 = [
|
|
28621
|
+
` BULK ${literal2(input.path)}`,
|
|
28622
|
+
` DATA_SOURCE = ${literal2(input.dataSource)}`,
|
|
28623
|
+
...formatOptions(input)
|
|
28624
|
+
];
|
|
28625
|
+
return [
|
|
28626
|
+
`SELECT ${selectList(input.columns, input)}`,
|
|
28627
|
+
"FROM OPENROWSET(",
|
|
28628
|
+
options2.join(",\n"),
|
|
28629
|
+
") WITH (",
|
|
28630
|
+
columnList(input.columns, input),
|
|
28631
|
+
") AS [rows]"
|
|
28632
|
+
].join("\n");
|
|
28633
|
+
}
|
|
28634
|
+
function formatOptions({ format, csv }) {
|
|
28635
|
+
if (format === "parquet")
|
|
28636
|
+
return [" FORMAT = 'PARQUET'"];
|
|
28637
|
+
const { header, delimiter } = csv ?? { header: true, delimiter: "," };
|
|
28638
|
+
return [
|
|
28639
|
+
" FORMAT = 'CSV'",
|
|
28640
|
+
" PARSER_VERSION = '2.0'",
|
|
28641
|
+
...header ? [" HEADER_ROW = TRUE"] : [],
|
|
28642
|
+
` FIELDTERMINATOR = ${literal2(delimiter)}`
|
|
28643
|
+
];
|
|
28644
|
+
}
|
|
28645
|
+
function viewStatement(fullName, query2) {
|
|
28646
|
+
const create2 = `CREATE SCHEMA ${quote(fullName.schema)}`;
|
|
28647
|
+
return [
|
|
28648
|
+
`IF SCHEMA_ID(N${literal2(fullName.schema)}) IS NULL EXEC(N${literal2(create2)});`,
|
|
28649
|
+
"GO",
|
|
28650
|
+
`CREATE OR ALTER VIEW ${quote(fullName.schema)}.${quote(fullName.table)} AS`,
|
|
28651
|
+
`${query2};`,
|
|
28652
|
+
"GO"
|
|
28653
|
+
].join("\n");
|
|
28654
|
+
}
|
|
28655
|
+
function literal2(text2) {
|
|
28656
|
+
return `'${text2.replaceAll("'", "''")}'`;
|
|
28657
|
+
}
|
|
28658
|
+
|
|
28659
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/build.js
|
|
28660
|
+
function build3(model) {
|
|
28661
|
+
const statements = model.tables.flatMap((table) => viewOf(model, table));
|
|
28662
|
+
if (statements.length === 0)
|
|
28663
|
+
return {};
|
|
28664
|
+
const header = [
|
|
28665
|
+
`-- Generated by Ingestron for ${model.project.id} (${model.project.environment}): Synapse serverless SQL views.`,
|
|
28666
|
+
"-- Run against the serverless database that holds the views."
|
|
28667
|
+
].join("\n");
|
|
28668
|
+
return { "views.sql": `${[header, ...statements].join("\n\n")}
|
|
28669
|
+
` };
|
|
28670
|
+
}
|
|
28671
|
+
function hasView(table) {
|
|
28672
|
+
return table.store?.method === "synapse-view" || table.serve?.method === "synapse-serve";
|
|
28673
|
+
}
|
|
28674
|
+
function viewOf(model, table) {
|
|
28675
|
+
const name = { schema: table.fullName.schema, table: table.fullName.table };
|
|
28676
|
+
const resolvedTypes = model.standard.packs[0]?.startsWith("synapse@") ?? false;
|
|
28677
|
+
if (table.serve?.method === "synapse-serve")
|
|
28678
|
+
return [viewStatement(name, filesView(table, table.serve, { resolvedTypes }))];
|
|
28679
|
+
if (table.store?.method !== "synapse-view")
|
|
28680
|
+
return [];
|
|
28681
|
+
const renamed = table.ingest?.renamed ?? true;
|
|
28682
|
+
const query2 = table.transform ? transformView(model, table) : filesView(table, table.store, { renamed, resolvedTypes });
|
|
28683
|
+
return [viewStatement(name, query2)];
|
|
28684
|
+
}
|
|
28685
|
+
function filesView(table, step, options2) {
|
|
28686
|
+
const output = sourceStep(table).output;
|
|
28687
|
+
const relative10 = relativePath2(output, step.settings);
|
|
28688
|
+
if ("problem" in relative10)
|
|
28689
|
+
throw new Error(relative10.problem);
|
|
28690
|
+
return filesQuery({
|
|
28691
|
+
...options2,
|
|
28692
|
+
path: relative10.path,
|
|
28693
|
+
dataSource: String(step.settings.data_source),
|
|
28694
|
+
format: output.format === "csv" ? "csv" : "parquet",
|
|
28695
|
+
columns: table.contract.columns,
|
|
28696
|
+
csv: step.settings.csv
|
|
28697
|
+
});
|
|
28698
|
+
}
|
|
28699
|
+
function transformView(model, table) {
|
|
28700
|
+
const code2 = withoutSemicolon(table.transform.code).replace(/\{\{\s*tables\.([A-Za-z0-9_-]+)\s*\}\}/g, (written, input) => {
|
|
28701
|
+
const found = model.tables.find((item) => item.name === input);
|
|
28702
|
+
return found ? `${quote(found.fullName.schema)}.${quote(found.fullName.table)}` : written;
|
|
28703
|
+
});
|
|
28704
|
+
const columns = table.contract.columns.map((column6) => quote(column6.name)).join(", ");
|
|
28705
|
+
return [`SELECT ${columns}`, "FROM (", code2, ") AS [source]"].join("\n");
|
|
28706
|
+
}
|
|
28707
|
+
function withoutSemicolon(code2) {
|
|
28708
|
+
return code2.trim().replace(/;\s*$/, "");
|
|
28709
|
+
}
|
|
28710
|
+
|
|
28711
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/contract.js
|
|
28712
|
+
var used3 = (does) => ({ use: "used", does });
|
|
28713
|
+
var notYet2 = (reason) => ({ use: "not-yet", reason });
|
|
28714
|
+
var noRows = notYet2("Synapse views check no rows.");
|
|
28715
|
+
var object4 = "schema.*";
|
|
28716
|
+
var column5 = "schema.*.properties.*";
|
|
28717
|
+
var contract3 = {
|
|
28718
|
+
...contractCommon,
|
|
28719
|
+
schema: used3("One object per table; each table uses the object named like it."),
|
|
28720
|
+
[`${object4}.name`]: used3("Matched to the table by its name."),
|
|
28721
|
+
[`${object4}.logicalType`]: { use: "documentation" },
|
|
28722
|
+
[`${object4}.physicalName`]: {
|
|
28723
|
+
use: "elsewhere",
|
|
28724
|
+
reason: "The view is named by the standard's naming."
|
|
28725
|
+
},
|
|
28726
|
+
[`${object4}.physicalType`]: { use: "documentation" },
|
|
28727
|
+
[`${object4}.description`]: { use: "documentation" },
|
|
28728
|
+
[`${object4}.properties`]: used3("Each property is a column of the view."),
|
|
28729
|
+
[`${object4}.relationships`]: notYet2("Synapse views declare no relationships."),
|
|
28730
|
+
[`${object4}.quality`]: noRows,
|
|
28731
|
+
[`${object4}.tags`]: notYet2("Synapse views carry no tags."),
|
|
28732
|
+
[`${column5}.name`]: used3("The column's name in the view and in the files."),
|
|
28733
|
+
[`${column5}.physicalName`]: {
|
|
28734
|
+
use: "elsewhere",
|
|
28735
|
+
reason: "Files hold columns under their contract names; the pack that lands them renames."
|
|
28736
|
+
},
|
|
28737
|
+
[`${column5}.logicalType`]: used3("Types the column in the view."),
|
|
28738
|
+
[`${column5}.physicalType`]: used3("Types a column that has no logicalType."),
|
|
28739
|
+
[`${column5}.quality`]: noRows,
|
|
28740
|
+
[`${column5}.description`]: { use: "documentation" },
|
|
28741
|
+
...Object.fromEntries([
|
|
28742
|
+
"required",
|
|
28743
|
+
"primaryKey",
|
|
28744
|
+
"primaryKeyPosition",
|
|
28745
|
+
"logicalTypeOptions",
|
|
28746
|
+
"unique",
|
|
28747
|
+
"partitioned",
|
|
28748
|
+
"partitionKeyPosition",
|
|
28749
|
+
"classification",
|
|
28750
|
+
"encryptedName",
|
|
28751
|
+
"relationships",
|
|
28752
|
+
"tags",
|
|
28753
|
+
"properties",
|
|
28754
|
+
"items"
|
|
28755
|
+
].map((key) => [`${column5}.${key}`, { use: "documentation" }]))
|
|
28756
|
+
};
|
|
28757
|
+
|
|
28758
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/request.js
|
|
28759
|
+
function request3(model) {
|
|
28760
|
+
const steps = model.tables.flatMap(synapseStep);
|
|
28761
|
+
if (steps.length === 0)
|
|
28762
|
+
return [];
|
|
28763
|
+
const databases = unique3(steps.map(({ table }) => table.fullName.catalog));
|
|
28764
|
+
const sources = unique3(steps.map(({ step }) => `${String(step.settings.data_source)}
|
|
28765
|
+
${String(step.settings.data_source_location)}`)).map((key) => key.split("\n"));
|
|
28766
|
+
const schemas = unique3(steps.map(({ table }) => table.fullName.schema));
|
|
28767
|
+
const where = databases.map(code).join(", ");
|
|
28768
|
+
return [
|
|
28769
|
+
{
|
|
28770
|
+
title: "Synapse serverless database",
|
|
28771
|
+
lines: databases.map((name) => `Create the serverless SQL database ${code(name)}; the views live in it.`)
|
|
28772
|
+
},
|
|
28773
|
+
{
|
|
28774
|
+
title: "External data source",
|
|
28775
|
+
lines: sources.flatMap(([name, location]) => [
|
|
28776
|
+
`In ${where}, create a database scoped credential with \`IDENTITY = 'Managed Identity'\`, and the external data source ${code(name)} at ${code(location)} using it.`,
|
|
28777
|
+
`Give the Synapse workspace's managed identity Storage Blob Data Reader on the storage behind ${code(location)}.`
|
|
28778
|
+
])
|
|
28779
|
+
},
|
|
28780
|
+
{
|
|
28781
|
+
title: "Report users",
|
|
28782
|
+
lines: [
|
|
28783
|
+
`In ${where}, give each report user SELECT on the ${schemas.length === 1 ? "schema" : "schemas"} ${schemas.map(code).join(", ")}, REFERENCES on the database scoped credential, and ADMINISTER DATABASE BULK OPERATIONS, which views over files need.`
|
|
28784
|
+
]
|
|
28785
|
+
}
|
|
28786
|
+
];
|
|
28787
|
+
}
|
|
28788
|
+
function synapseStep(table) {
|
|
28789
|
+
const step = table.serve?.method === "synapse-serve" ? table.serve : table.store;
|
|
28790
|
+
return step && /^synapse-(view|serve)$/.test(step.method) ? [{ table, step }] : [];
|
|
28791
|
+
}
|
|
28792
|
+
var unique3 = (values2) => [...new Set(values2)];
|
|
28793
|
+
var code = (text2) => `\`${text2}\``;
|
|
28794
|
+
|
|
28795
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/methods.js
|
|
28796
|
+
var placement2 = { platform: ["data_source", "data_source_location"], standard: ["csv"] };
|
|
28797
|
+
var settings = {
|
|
28798
|
+
type: "object",
|
|
28799
|
+
additionalProperties: false,
|
|
28800
|
+
required: ["data_source", "data_source_location"],
|
|
28801
|
+
properties: {
|
|
28802
|
+
data_source: {
|
|
28803
|
+
type: "string",
|
|
28804
|
+
description: "The external data source's name in the serverless database.",
|
|
28805
|
+
examples: ["landing"]
|
|
28806
|
+
},
|
|
28807
|
+
data_source_location: {
|
|
28808
|
+
type: "string",
|
|
28809
|
+
pattern: "^(abfss|https)://",
|
|
28810
|
+
description: "The data source's location; every file a view reads must be under it.",
|
|
28811
|
+
examples: ["abfss://landing@acct.dfs.core.windows.net"]
|
|
28812
|
+
},
|
|
28813
|
+
csv: {
|
|
28814
|
+
type: "object",
|
|
28815
|
+
additionalProperties: false,
|
|
28816
|
+
description: "How CSV files are read.",
|
|
28817
|
+
properties: {
|
|
28818
|
+
header: { type: "boolean", description: "The first row names the columns." },
|
|
28819
|
+
delimiter: {
|
|
28820
|
+
type: "string",
|
|
28821
|
+
description: "The field separator.",
|
|
28822
|
+
examples: [",", ";", "|"]
|
|
28823
|
+
}
|
|
28824
|
+
}
|
|
28825
|
+
}
|
|
28826
|
+
}
|
|
28827
|
+
};
|
|
28828
|
+
var defaults = { csv: { header: true, delimiter: "," } };
|
|
28829
|
+
function checkView(table) {
|
|
28830
|
+
const transform2 = table.transform?.method;
|
|
28831
|
+
if (transform2)
|
|
28832
|
+
return transform2 === "synapse-sql" ? [] : [`it keeps synapse-sql transforms only, not "${transform2}"`];
|
|
28833
|
+
if (!table.ingest?.output)
|
|
28834
|
+
return [
|
|
28835
|
+
`"${table.ingest?.method}" does not say where its files land, so no view can read them`
|
|
28836
|
+
];
|
|
28837
|
+
return filesProblems(table, table.store.settings);
|
|
28838
|
+
}
|
|
28839
|
+
var reference3 = /\{\{\s*tables\.([A-Za-z0-9_-]+)\s*\}\}/g;
|
|
28840
|
+
function checkSql(table) {
|
|
28841
|
+
const { code: code2, inputs } = table.transform;
|
|
28842
|
+
const leadingWith = /^\s*with\b/i.test(code2) ? ["a view's SQL cannot start with WITH; write the common table expression as a subquery"] : [];
|
|
28843
|
+
const unlisted = [...code2.matchAll(reference3)].map((match) => match[1]).filter((input) => !(input in inputs)).map((input) => `the SQL names {{ tables.${input} }}, which is not in inputs`);
|
|
28844
|
+
return [...leadingWith, ...unlisted];
|
|
28845
|
+
}
|
|
28846
|
+
var synapseView = {
|
|
28847
|
+
action: "store",
|
|
28848
|
+
description: "A view in Synapse serverless SQL: over the table's landed files, or over its transform's query. Nothing runs after a load.",
|
|
28849
|
+
accepts: ["files", "query"],
|
|
28850
|
+
produces: "table",
|
|
28851
|
+
placement: placement2,
|
|
28852
|
+
defaults,
|
|
28853
|
+
settings,
|
|
28854
|
+
check: checkView
|
|
28855
|
+
};
|
|
28856
|
+
var synapseSql = {
|
|
28857
|
+
action: "transform",
|
|
28858
|
+
description: "The transform's SQL in T-SQL, over other tables' views; synapse-view keeps it.",
|
|
28859
|
+
accepts: ["table"],
|
|
28860
|
+
produces: "query",
|
|
28861
|
+
settings: { type: "object", additionalProperties: false, properties: {} },
|
|
28862
|
+
check: checkSql
|
|
28863
|
+
};
|
|
28864
|
+
var synapseServe = {
|
|
28865
|
+
action: "serve",
|
|
28866
|
+
description: "A view in Synapse serverless SQL over the files the table's last step records, such as a table another pack stores.",
|
|
28867
|
+
accepts: ["table", "files"],
|
|
28868
|
+
produces: "table",
|
|
28869
|
+
needsOutput: true,
|
|
28870
|
+
placement: placement2,
|
|
28871
|
+
defaults,
|
|
28872
|
+
settings,
|
|
28873
|
+
check: (table) => filesProblems(table, table.serve.settings)
|
|
28874
|
+
};
|
|
28875
|
+
|
|
28876
|
+
// node_modules/.pnpm/@ingestron+pack-synapse@1.0.1_@ingestron+definitions@packages+definitions/node_modules/@ingestron/pack-synapse/dist/pack.js
|
|
28877
|
+
var synapse = {
|
|
28878
|
+
id: "synapse",
|
|
28879
|
+
version: 1,
|
|
28880
|
+
defaults: {
|
|
28881
|
+
naming: {
|
|
28882
|
+
catalog: "{{ project }}_{{ environment }}",
|
|
28883
|
+
schema: "{{ layer }}",
|
|
28884
|
+
table: "{{ table }}",
|
|
28885
|
+
pipeline: "{{ project }}",
|
|
28886
|
+
job: "{{ project }}_refresh"
|
|
28887
|
+
},
|
|
28888
|
+
layers: ["bronze", "silver", "gold"],
|
|
28889
|
+
types: synapseTypes,
|
|
28890
|
+
quality: { required: "warn", default: "warn" }
|
|
28891
|
+
},
|
|
28892
|
+
rules: { store: { default: "synapse-view" }, transform: { default: "synapse-sql" } },
|
|
28893
|
+
methods: {
|
|
28894
|
+
"synapse-view": synapseView,
|
|
28895
|
+
"synapse-sql": synapseSql,
|
|
28896
|
+
"synapse-serve": synapseServe
|
|
28897
|
+
},
|
|
28898
|
+
build: build3,
|
|
28899
|
+
check: transformInputs,
|
|
28900
|
+
request: request3,
|
|
28901
|
+
contract: contract3
|
|
28902
|
+
};
|
|
28903
|
+
function transformInputs(model) {
|
|
28904
|
+
return model.tables.filter((table) => table.transform?.method === "synapse-sql").flatMap((table) => [
|
|
28905
|
+
...table.store?.method === "synapse-view" ? [] : [
|
|
28906
|
+
`transform "${table.name}" is T-SQL (synapse-sql), so only synapse-view can keep it, not "${table.store?.method}"`
|
|
28907
|
+
],
|
|
28908
|
+
...Object.keys(table.transform.inputs).filter((input) => !model.tables.some((item) => item.name === input && hasView(item))).map((input) => `transform "${table.name}" reads "${input}", which has no Synapse view; store it with synapse-view or serve it with synapse-serve`)
|
|
28909
|
+
]);
|
|
28910
|
+
}
|
|
28911
|
+
|
|
28519
28912
|
// packages/cli/src/packs.ts
|
|
28520
|
-
var packs = [databricks, adf];
|
|
28913
|
+
var packs = [databricks, adf, synapse];
|
|
28521
28914
|
|
|
28522
28915
|
// packages/cli/src/environments.ts
|
|
28523
28916
|
var import_yaml9 = __toESM(require_dist(), 1);
|
|
@@ -28555,8 +28948,8 @@ function extraFiles(root, known3, others) {
|
|
|
28555
28948
|
if (!skip.has(path) && !existsSync2(join7(path, manifestFile))) walk2(path);
|
|
28556
28949
|
continue;
|
|
28557
28950
|
}
|
|
28558
|
-
const
|
|
28559
|
-
if (!known3.has(
|
|
28951
|
+
const relativePath3 = relative3(root, path).split(sep3).join("/");
|
|
28952
|
+
if (!known3.has(relativePath3) && !ignored.has(name)) extra.push(relativePath3);
|
|
28560
28953
|
}
|
|
28561
28954
|
};
|
|
28562
28955
|
if (isFolder(root)) walk2(root);
|
|
@@ -28628,10 +29021,10 @@ ${formatDiagnostics(result.diagnostics)}
|
|
|
28628
29021
|
});
|
|
28629
29022
|
}
|
|
28630
29023
|
function generatedOutputs(model, environment2) {
|
|
28631
|
-
const
|
|
28632
|
-
return
|
|
29024
|
+
const used4 = packs.filter((pack) => model.standard.packs.includes(`${pack.id}@${pack.version}`));
|
|
29025
|
+
return used4.map((pack) => ({
|
|
28633
29026
|
pack: pack.id,
|
|
28634
|
-
label:
|
|
29027
|
+
label: used4.length > 1 ? `${environment2}/${pack.id}` : environment2,
|
|
28635
29028
|
files: pack.build(model)
|
|
28636
29029
|
}));
|
|
28637
29030
|
}
|
|
@@ -28886,13 +29279,13 @@ var import_yaml13 = __toESM(require_dist(), 1);
|
|
|
28886
29279
|
function projectQuestions(files2, connectionFile, flowFiles, method) {
|
|
28887
29280
|
const questions = [];
|
|
28888
29281
|
const project = (0, import_yaml13.parse)(String(files2["project.yaml"]));
|
|
28889
|
-
const ask = (
|
|
28890
|
-
const value =
|
|
29282
|
+
const ask = (object5, key, name, label) => {
|
|
29283
|
+
const value = object5[key];
|
|
28891
29284
|
if (typeof value !== "string") return;
|
|
28892
|
-
const
|
|
28893
|
-
const fallback =
|
|
29285
|
+
const reference4 = /^\$\{values\.([\w]+)\}$/.exec(value);
|
|
29286
|
+
const fallback = reference4 ? project.values?.[reference4[1]] ?? value : value;
|
|
28894
29287
|
questions.push({ value: name, ask: label, default: fallback });
|
|
28895
|
-
|
|
29288
|
+
object5[key] = `\${values.${name}}`;
|
|
28896
29289
|
};
|
|
28897
29290
|
const connection = (0, import_yaml13.parse)(String(files2[connectionFile]));
|
|
28898
29291
|
for (const key of Object.keys(connection.settings))
|
|
@@ -28907,9 +29300,9 @@ function projectQuestions(files2, connectionFile, flowFiles, method) {
|
|
|
28907
29300
|
const flow = (0, import_yaml13.parse)(String(files2[file]));
|
|
28908
29301
|
for (const [name, table] of Object.entries(flow.tables))
|
|
28909
29302
|
ask(table, "object", `${name}_object`, `Source object for ${name}`);
|
|
28910
|
-
const
|
|
28911
|
-
ask(
|
|
28912
|
-
const options2 =
|
|
29303
|
+
const settings2 = flow.ingest?.with ?? {};
|
|
29304
|
+
ask(settings2, "query", "source_query", "Native source query");
|
|
29305
|
+
const options2 = settings2.source_options;
|
|
28913
29306
|
for (const key of Object.keys(options2 ?? {}))
|
|
28914
29307
|
ask(options2, key, `option_${key}`, `Source option ${key}`);
|
|
28915
29308
|
files2[file] = (0, import_yaml13.stringify)(flow);
|
|
@@ -28920,9 +29313,9 @@ function projectQuestions(files2, connectionFile, flowFiles, method) {
|
|
|
28920
29313
|
function infrastructureQuestions(files2, method, ask) {
|
|
28921
29314
|
if (!files2["platform.yaml"]) return;
|
|
28922
29315
|
const platform = (0, import_yaml13.parse)(String(files2["platform.yaml"]));
|
|
28923
|
-
const
|
|
29316
|
+
const settings2 = platform.settings?.[method] ?? {};
|
|
28924
29317
|
for (const key of ["landing", "key_vault", "integration_runtime"])
|
|
28925
|
-
ask(
|
|
29318
|
+
ask(settings2, key, key, `Platform ${key}`);
|
|
28926
29319
|
files2["platform.yaml"] = (0, import_yaml13.stringify)(platform);
|
|
28927
29320
|
}
|
|
28928
29321
|
|
|
@@ -28994,9 +29387,9 @@ function trimSettings(files2, method, via, flowFiles) {
|
|
|
28994
29387
|
if (!files2["platform.yaml"]) return;
|
|
28995
29388
|
const platform = (0, import_yaml14.parse)(String(files2["platform.yaml"]));
|
|
28996
29389
|
if (platform.settings) platform.settings = { [method]: platform.settings[method] ?? {} };
|
|
28997
|
-
const
|
|
28998
|
-
if (
|
|
28999
|
-
if (
|
|
29390
|
+
const settings2 = platform.settings?.[method];
|
|
29391
|
+
if (settings2) delete settings2.resource_names;
|
|
29392
|
+
if (settings2) moveInfrastructure(files2, flowFiles, settings2);
|
|
29000
29393
|
files2["platform.yaml"] = (0, import_yaml14.stringify)(platform);
|
|
29001
29394
|
}
|
|
29002
29395
|
function trimStandard(files2, method, via) {
|
|
@@ -29006,14 +29399,14 @@ function trimStandard(files2, method, via) {
|
|
|
29006
29399
|
standard.ingest.settings = { [method]: standard.ingest.settings[method] ?? {} };
|
|
29007
29400
|
files2["standard.yaml"] = (0, import_yaml14.stringify)(standard);
|
|
29008
29401
|
}
|
|
29009
|
-
function moveInfrastructure(files2, flowFiles,
|
|
29402
|
+
function moveInfrastructure(files2, flowFiles, settings2) {
|
|
29010
29403
|
for (const file of flowFiles) {
|
|
29011
29404
|
const flow = (0, import_yaml14.parse)(String(files2[file]));
|
|
29012
29405
|
const overrides = flow.ingest?.with;
|
|
29013
29406
|
if (!overrides) continue;
|
|
29014
29407
|
for (const key of ["landing", "integration_runtime"]) {
|
|
29015
29408
|
if (overrides[key] === void 0) continue;
|
|
29016
|
-
|
|
29409
|
+
settings2[key] = overrides[key];
|
|
29017
29410
|
delete overrides[key];
|
|
29018
29411
|
}
|
|
29019
29412
|
files2[file] = (0, import_yaml14.stringify)(flow);
|
|
@@ -29355,7 +29748,8 @@ function tableFacts(table) {
|
|
|
29355
29748
|
}
|
|
29356
29749
|
},
|
|
29357
29750
|
transform: step(table.transform),
|
|
29358
|
-
store: step(table.store)
|
|
29751
|
+
store: step(table.store),
|
|
29752
|
+
serve: step(table.serve)
|
|
29359
29753
|
}),
|
|
29360
29754
|
expectations: table.expectations.map((item) => `${item.name}: ${item.action}`)
|
|
29361
29755
|
};
|
|
@@ -29380,7 +29774,7 @@ function shown(value) {
|
|
|
29380
29774
|
|
|
29381
29775
|
// packages/cli/src/request.ts
|
|
29382
29776
|
import { resolve as resolve5 } from "node:path";
|
|
29383
|
-
function
|
|
29777
|
+
function request4(options2, io) {
|
|
29384
29778
|
const definitions = resolve5(io.cwd, options2.definitions);
|
|
29385
29779
|
const files2 = nodeFiles(definitions);
|
|
29386
29780
|
const texts = environmentsToGenerate(definitions, options2.environments).map(
|
|
@@ -29450,8 +29844,8 @@ function infrastructure(files2, environment2) {
|
|
|
29450
29844
|
envvars: {}
|
|
29451
29845
|
});
|
|
29452
29846
|
const platform = loaded.project?.documents.find((item) => item.kind === "Platform");
|
|
29453
|
-
const
|
|
29454
|
-
const lines = Object.entries(
|
|
29847
|
+
const settings2 = platform?.value?.settings;
|
|
29848
|
+
const lines = Object.entries(settings2 ?? {}).flatMap(
|
|
29455
29849
|
([method, values2]) => Object.entries(values2 ?? {}).map(
|
|
29456
29850
|
([key, value]) => `- \`${method}\` \`${key}\`: ${shown2(value)}`
|
|
29457
29851
|
)
|
|
@@ -29588,7 +29982,7 @@ function draftContract(options2, io) {
|
|
|
29588
29982
|
function contractFromSample(text2, file, id) {
|
|
29589
29983
|
const { columns, rows } = readSample(text2, file);
|
|
29590
29984
|
const properties2 = columns.map(
|
|
29591
|
-
(
|
|
29985
|
+
(column6) => ` - { name: ${scalar(column6)}, logicalType: ${typeOf2(rows.map((row) => row[column6]))} }`
|
|
29592
29986
|
);
|
|
29593
29987
|
return [
|
|
29594
29988
|
`# Drafted from ${file} (${rows.length} ${rows.length === 1 ? "row" : "rows"}): names and types only. Review the`,
|
|
@@ -29619,7 +30013,7 @@ var patterns = [
|
|
|
29619
30013
|
["date", /^\d{4}-\d{2}-\d{2}$/],
|
|
29620
30014
|
["timestamp", /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?$/]
|
|
29621
30015
|
];
|
|
29622
|
-
function
|
|
30016
|
+
function typeOf2(values2) {
|
|
29623
30017
|
const types = new Set(values2.filter((value) => value !== "" && value != null).map(typeOfValue));
|
|
29624
30018
|
if (types.size === 1) return [...types][0];
|
|
29625
30019
|
return types.size === 2 && types.has("integer") && types.has("number") ? "number" : "string";
|
|
@@ -29693,7 +30087,7 @@ ${usage}`);
|
|
|
29693
30087
|
return 2;
|
|
29694
30088
|
}
|
|
29695
30089
|
if (options2.command === "init") return init(options2, io);
|
|
29696
|
-
if (options2.command === "request") return
|
|
30090
|
+
if (options2.command === "request") return request4(options2, io);
|
|
29697
30091
|
if (options2.command === "compare") return compare2(options2, io);
|
|
29698
30092
|
if (options2.command === "contract") return draftContract(options2, io);
|
|
29699
30093
|
return options2.command === "schemas" ? writeSchemas(options2, io) : generate(options2, io);
|