@dataform/core 1.21.1 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +23 -0
- package/bundle.js +119 -6
- package/package.json +1 -1
package/bundle.d.ts
CHANGED
|
@@ -491,6 +491,15 @@ namespace dataform {
|
|
|
491
491
|
|
|
492
492
|
/** BigQueryOptions labels */
|
|
493
493
|
labels?: ({ [k: string]: string }|null);
|
|
494
|
+
|
|
495
|
+
/** BigQueryOptions partitionExpirationDays */
|
|
496
|
+
partitionExpirationDays?: (number|null);
|
|
497
|
+
|
|
498
|
+
/** BigQueryOptions requirePartitionFilter */
|
|
499
|
+
requirePartitionFilter?: (boolean|null);
|
|
500
|
+
|
|
501
|
+
/** BigQueryOptions additionalOptions */
|
|
502
|
+
additionalOptions?: ({ [k: string]: string }|null);
|
|
494
503
|
}
|
|
495
504
|
|
|
496
505
|
/** Represents a BigQueryOptions. */
|
|
@@ -514,6 +523,15 @@ namespace dataform {
|
|
|
514
523
|
/** BigQueryOptions labels. */
|
|
515
524
|
public labels: { [k: string]: string };
|
|
516
525
|
|
|
526
|
+
/** BigQueryOptions partitionExpirationDays. */
|
|
527
|
+
public partitionExpirationDays: number;
|
|
528
|
+
|
|
529
|
+
/** BigQueryOptions requirePartitionFilter. */
|
|
530
|
+
public requirePartitionFilter: boolean;
|
|
531
|
+
|
|
532
|
+
/** BigQueryOptions additionalOptions. */
|
|
533
|
+
public additionalOptions: { [k: string]: string };
|
|
534
|
+
|
|
517
535
|
/**
|
|
518
536
|
* Creates a new BigQueryOptions instance using the specified properties.
|
|
519
537
|
* @param [properties] Properties to set
|
|
@@ -7791,6 +7809,11 @@ interface IBigQueryOptions {
|
|
|
7791
7809
|
labels?: {
|
|
7792
7810
|
[name: string]: string;
|
|
7793
7811
|
};
|
|
7812
|
+
partitionExpirationDays?: number;
|
|
7813
|
+
requirePartitionFilter?: boolean;
|
|
7814
|
+
additionalOptions?: {
|
|
7815
|
+
[name: string]: string;
|
|
7816
|
+
};
|
|
7794
7817
|
}
|
|
7795
7818
|
interface IPrestoOptions {
|
|
7796
7819
|
partitionBy?: string[];
|
package/bundle.js
CHANGED
|
@@ -1303,6 +1303,9 @@ const dataform = $root.dataform = (() => {
|
|
|
1303
1303
|
* @property {Array.<string>|null} [clusterBy] BigQueryOptions clusterBy
|
|
1304
1304
|
* @property {string|null} [updatePartitionFilter] BigQueryOptions updatePartitionFilter
|
|
1305
1305
|
* @property {Object.<string,string>|null} [labels] BigQueryOptions labels
|
|
1306
|
+
* @property {number|null} [partitionExpirationDays] BigQueryOptions partitionExpirationDays
|
|
1307
|
+
* @property {boolean|null} [requirePartitionFilter] BigQueryOptions requirePartitionFilter
|
|
1308
|
+
* @property {Object.<string,string>|null} [additionalOptions] BigQueryOptions additionalOptions
|
|
1306
1309
|
*/
|
|
1307
1310
|
|
|
1308
1311
|
/**
|
|
@@ -1316,6 +1319,7 @@ const dataform = $root.dataform = (() => {
|
|
|
1316
1319
|
function BigQueryOptions(properties) {
|
|
1317
1320
|
this.clusterBy = [];
|
|
1318
1321
|
this.labels = {};
|
|
1322
|
+
this.additionalOptions = {};
|
|
1319
1323
|
if (properties)
|
|
1320
1324
|
for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
|
|
1321
1325
|
if (properties[keys[i]] != null)
|
|
@@ -1354,6 +1358,30 @@ const dataform = $root.dataform = (() => {
|
|
|
1354
1358
|
*/
|
|
1355
1359
|
BigQueryOptions.prototype.labels = $util.emptyObject;
|
|
1356
1360
|
|
|
1361
|
+
/**
|
|
1362
|
+
* BigQueryOptions partitionExpirationDays.
|
|
1363
|
+
* @member {number} partitionExpirationDays
|
|
1364
|
+
* @memberof dataform.BigQueryOptions
|
|
1365
|
+
* @instance
|
|
1366
|
+
*/
|
|
1367
|
+
BigQueryOptions.prototype.partitionExpirationDays = 0;
|
|
1368
|
+
|
|
1369
|
+
/**
|
|
1370
|
+
* BigQueryOptions requirePartitionFilter.
|
|
1371
|
+
* @member {boolean} requirePartitionFilter
|
|
1372
|
+
* @memberof dataform.BigQueryOptions
|
|
1373
|
+
* @instance
|
|
1374
|
+
*/
|
|
1375
|
+
BigQueryOptions.prototype.requirePartitionFilter = false;
|
|
1376
|
+
|
|
1377
|
+
/**
|
|
1378
|
+
* BigQueryOptions additionalOptions.
|
|
1379
|
+
* @member {Object.<string,string>} additionalOptions
|
|
1380
|
+
* @memberof dataform.BigQueryOptions
|
|
1381
|
+
* @instance
|
|
1382
|
+
*/
|
|
1383
|
+
BigQueryOptions.prototype.additionalOptions = $util.emptyObject;
|
|
1384
|
+
|
|
1357
1385
|
/**
|
|
1358
1386
|
* Creates a new BigQueryOptions instance using the specified properties.
|
|
1359
1387
|
* @function create
|
|
@@ -1388,6 +1416,13 @@ const dataform = $root.dataform = (() => {
|
|
|
1388
1416
|
if (message.labels != null && message.hasOwnProperty("labels"))
|
|
1389
1417
|
for (let keys = Object.keys(message.labels), i = 0; i < keys.length; ++i)
|
|
1390
1418
|
writer.uint32(/* id 4, wireType 2 =*/34).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.labels[keys[i]]).ldelim();
|
|
1419
|
+
if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
|
|
1420
|
+
writer.uint32(/* id 5, wireType 0 =*/40).int32(message.partitionExpirationDays);
|
|
1421
|
+
if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
|
|
1422
|
+
writer.uint32(/* id 6, wireType 0 =*/48).bool(message.requirePartitionFilter);
|
|
1423
|
+
if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions"))
|
|
1424
|
+
for (let keys = Object.keys(message.additionalOptions), i = 0; i < keys.length; ++i)
|
|
1425
|
+
writer.uint32(/* id 7, wireType 2 =*/58).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.additionalOptions[keys[i]]).ldelim();
|
|
1391
1426
|
return writer;
|
|
1392
1427
|
};
|
|
1393
1428
|
|
|
@@ -1441,6 +1476,20 @@ const dataform = $root.dataform = (() => {
|
|
|
1441
1476
|
reader.pos++;
|
|
1442
1477
|
message.labels[key] = reader.string();
|
|
1443
1478
|
break;
|
|
1479
|
+
case 5:
|
|
1480
|
+
message.partitionExpirationDays = reader.int32();
|
|
1481
|
+
break;
|
|
1482
|
+
case 6:
|
|
1483
|
+
message.requirePartitionFilter = reader.bool();
|
|
1484
|
+
break;
|
|
1485
|
+
case 7:
|
|
1486
|
+
reader.skip().pos++;
|
|
1487
|
+
if (message.additionalOptions === $util.emptyObject)
|
|
1488
|
+
message.additionalOptions = {};
|
|
1489
|
+
key = reader.string();
|
|
1490
|
+
reader.pos++;
|
|
1491
|
+
message.additionalOptions[key] = reader.string();
|
|
1492
|
+
break;
|
|
1444
1493
|
default:
|
|
1445
1494
|
reader.skipType(tag & 7);
|
|
1446
1495
|
break;
|
|
@@ -1497,6 +1546,20 @@ const dataform = $root.dataform = (() => {
|
|
|
1497
1546
|
if (!$util.isString(message.labels[key[i]]))
|
|
1498
1547
|
return "labels: string{k:string} expected";
|
|
1499
1548
|
}
|
|
1549
|
+
if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
|
|
1550
|
+
if (!$util.isInteger(message.partitionExpirationDays))
|
|
1551
|
+
return "partitionExpirationDays: integer expected";
|
|
1552
|
+
if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
|
|
1553
|
+
if (typeof message.requirePartitionFilter !== "boolean")
|
|
1554
|
+
return "requirePartitionFilter: boolean expected";
|
|
1555
|
+
if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions")) {
|
|
1556
|
+
if (!$util.isObject(message.additionalOptions))
|
|
1557
|
+
return "additionalOptions: object expected";
|
|
1558
|
+
let key = Object.keys(message.additionalOptions);
|
|
1559
|
+
for (let i = 0; i < key.length; ++i)
|
|
1560
|
+
if (!$util.isString(message.additionalOptions[key[i]]))
|
|
1561
|
+
return "additionalOptions: string{k:string} expected";
|
|
1562
|
+
}
|
|
1500
1563
|
return null;
|
|
1501
1564
|
};
|
|
1502
1565
|
|
|
@@ -1530,6 +1593,17 @@ const dataform = $root.dataform = (() => {
|
|
|
1530
1593
|
for (let keys = Object.keys(object.labels), i = 0; i < keys.length; ++i)
|
|
1531
1594
|
message.labels[keys[i]] = String(object.labels[keys[i]]);
|
|
1532
1595
|
}
|
|
1596
|
+
if (object.partitionExpirationDays != null)
|
|
1597
|
+
message.partitionExpirationDays = object.partitionExpirationDays | 0;
|
|
1598
|
+
if (object.requirePartitionFilter != null)
|
|
1599
|
+
message.requirePartitionFilter = Boolean(object.requirePartitionFilter);
|
|
1600
|
+
if (object.additionalOptions) {
|
|
1601
|
+
if (typeof object.additionalOptions !== "object")
|
|
1602
|
+
throw TypeError(".dataform.BigQueryOptions.additionalOptions: object expected");
|
|
1603
|
+
message.additionalOptions = {};
|
|
1604
|
+
for (let keys = Object.keys(object.additionalOptions), i = 0; i < keys.length; ++i)
|
|
1605
|
+
message.additionalOptions[keys[i]] = String(object.additionalOptions[keys[i]]);
|
|
1606
|
+
}
|
|
1533
1607
|
return message;
|
|
1534
1608
|
};
|
|
1535
1609
|
|
|
@@ -1548,11 +1622,15 @@ const dataform = $root.dataform = (() => {
|
|
|
1548
1622
|
let object = {};
|
|
1549
1623
|
if (options.arrays || options.defaults)
|
|
1550
1624
|
object.clusterBy = [];
|
|
1551
|
-
if (options.objects || options.defaults)
|
|
1625
|
+
if (options.objects || options.defaults) {
|
|
1552
1626
|
object.labels = {};
|
|
1627
|
+
object.additionalOptions = {};
|
|
1628
|
+
}
|
|
1553
1629
|
if (options.defaults) {
|
|
1554
1630
|
object.partitionBy = "";
|
|
1555
1631
|
object.updatePartitionFilter = "";
|
|
1632
|
+
object.partitionExpirationDays = 0;
|
|
1633
|
+
object.requirePartitionFilter = false;
|
|
1556
1634
|
}
|
|
1557
1635
|
if (message.partitionBy != null && message.hasOwnProperty("partitionBy"))
|
|
1558
1636
|
object.partitionBy = message.partitionBy;
|
|
@@ -1569,6 +1647,15 @@ const dataform = $root.dataform = (() => {
|
|
|
1569
1647
|
for (let j = 0; j < keys2.length; ++j)
|
|
1570
1648
|
object.labels[keys2[j]] = message.labels[keys2[j]];
|
|
1571
1649
|
}
|
|
1650
|
+
if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
|
|
1651
|
+
object.partitionExpirationDays = message.partitionExpirationDays;
|
|
1652
|
+
if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
|
|
1653
|
+
object.requirePartitionFilter = message.requirePartitionFilter;
|
|
1654
|
+
if (message.additionalOptions && (keys2 = Object.keys(message.additionalOptions)).length) {
|
|
1655
|
+
object.additionalOptions = {};
|
|
1656
|
+
for (let j = 0; j < keys2.length; ++j)
|
|
1657
|
+
object.additionalOptions[keys2[j]] = message.additionalOptions[keys2[j]];
|
|
1658
|
+
}
|
|
1572
1659
|
return object;
|
|
1573
1660
|
};
|
|
1574
1661
|
|
|
@@ -19858,13 +19945,26 @@ class BigQueryAdapter extends Adapter {
|
|
|
19858
19945
|
return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
|
|
19859
19946
|
}
|
|
19860
19947
|
createOrReplace(table) {
|
|
19948
|
+
const options = [];
|
|
19949
|
+
if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) {
|
|
19950
|
+
options.push(`partition_expiration_days=${table.bigquery.partitionExpirationDays}`);
|
|
19951
|
+
}
|
|
19952
|
+
if (table.bigquery && table.bigquery.partitionBy && table.bigquery.requirePartitionFilter) {
|
|
19953
|
+
options.push(`require_partition_filter=${table.bigquery.requirePartitionFilter}`);
|
|
19954
|
+
}
|
|
19955
|
+
if (table.bigquery && table.bigquery.additionalOptions) {
|
|
19956
|
+
for (const [optionName, optionValue] of Object.entries(table.bigquery.additionalOptions)) {
|
|
19957
|
+
options.push(`${optionName}=${optionValue}`);
|
|
19958
|
+
}
|
|
19959
|
+
}
|
|
19861
19960
|
return `create or replace ${table.materialized
|
|
19862
19961
|
? "materialized "
|
|
19863
19962
|
: ""}${this.tableTypeAsSql(this.baseTableType(table.type))} ${this.resolveTarget(table.target)} ${table.bigquery && table.bigquery.partitionBy
|
|
19864
19963
|
? `partition by ${table.bigquery.partitionBy} `
|
|
19865
19964
|
: ""}${table.bigquery && table.bigquery.clusterBy && table.bigquery.clusterBy.length > 0
|
|
19866
19965
|
? `cluster by ${table.bigquery.clusterBy.join(", ")} `
|
|
19867
|
-
: ""}
|
|
19966
|
+
: ""}${options.length > 0 ?
|
|
19967
|
+
`OPTIONS(${options.join(',')})` : ""}as ${table.query}`;
|
|
19868
19968
|
}
|
|
19869
19969
|
createOrReplaceView(target, query) {
|
|
19870
19970
|
return `
|
|
@@ -20865,7 +20965,7 @@ const SortStyleType = ["compound", "interleaved"];
|
|
|
20865
20965
|
const IRedshiftOptionsProperties = () => strictKeysOf()(["distKey", "distStyle", "sortKeys", "sortStyle"]);
|
|
20866
20966
|
const ISnowflakeOptionsProperties = () => strictKeysOf()(["secure", "transient", "clusterBy"]);
|
|
20867
20967
|
const ISQLDataWarehouseOptionsProperties = () => strictKeysOf()(["distribution"]);
|
|
20868
|
-
const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels"]);
|
|
20968
|
+
const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels", "partitionExpirationDays", "requirePartitionFilter", "additionalOptions"]);
|
|
20869
20969
|
const IPrestoOptionsProperties = () => strictKeysOf()(["partitionBy"]);
|
|
20870
20970
|
const ITableAssertionsProperties = () => strictKeysOf()(["uniqueKey", "uniqueKeys", "nonNull", "rowConditions"]);
|
|
20871
20971
|
const ITableConfigProperties = () => strictKeysOf()([
|
|
@@ -22277,7 +22377,7 @@ class RefReplacingContext {
|
|
|
22277
22377
|
}
|
|
22278
22378
|
}
|
|
22279
22379
|
|
|
22280
|
-
const version = "1.
|
|
22380
|
+
const version = "1.22.0";
|
|
22281
22381
|
|
|
22282
22382
|
const SQL_DATA_WAREHOUSE_DIST_HASH_REGEXP = new RegExp("HASH\\s*\\(\\s*\\w*\\s*\\)\\s*");
|
|
22283
22383
|
const DEFAULT_CONFIG = {
|
|
@@ -22669,9 +22769,22 @@ class Session {
|
|
|
22669
22769
|
}
|
|
22670
22770
|
}
|
|
22671
22771
|
if (!!table.bigquery) {
|
|
22672
|
-
if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length)
|
|
22772
|
+
if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length) || table.bigquery.partitionExpirationDays
|
|
22773
|
+
|| table.bigquery.requirePartitionFilter) &&
|
|
22673
22774
|
table.type === "view") {
|
|
22674
|
-
this.compileError(`partitionBy/clusterBy are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
|
|
22775
|
+
this.compileError(`partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
|
|
22776
|
+
}
|
|
22777
|
+
else if ((!table.bigquery.partitionBy && (table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter)) &&
|
|
22778
|
+
table.type === "table") {
|
|
22779
|
+
this.compileError(`requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables`, table.fileName, table.target);
|
|
22780
|
+
}
|
|
22781
|
+
else if (table.bigquery.additionalOptions) {
|
|
22782
|
+
if (table.bigquery.partitionExpirationDays && table.bigquery.additionalOptions.partition_expiration_days) {
|
|
22783
|
+
this.compileError(`partitionExpirationDays has been declared twice`, table.fileName, table.target);
|
|
22784
|
+
}
|
|
22785
|
+
if (table.bigquery.requirePartitionFilter && table.bigquery.additionalOptions.require_partition_filter) {
|
|
22786
|
+
this.compileError(`requirePartitionFilter has been declared twice`, table.fileName, table.target);
|
|
22787
|
+
}
|
|
22675
22788
|
}
|
|
22676
22789
|
}
|
|
22677
22790
|
if (!!Table.IGNORED_PROPS[table.type]) {
|