@dataform/core 1.20.3 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.d.ts +43 -2
- package/bundle.js +229 -29
- package/package.json +1 -1
package/bundle.d.ts
CHANGED
|
@@ -491,6 +491,15 @@ namespace dataform {
|
|
|
491
491
|
|
|
492
492
|
/** BigQueryOptions labels */
|
|
493
493
|
labels?: ({ [k: string]: string }|null);
|
|
494
|
+
|
|
495
|
+
/** BigQueryOptions partitionExpirationDays */
|
|
496
|
+
partitionExpirationDays?: (number|null);
|
|
497
|
+
|
|
498
|
+
/** BigQueryOptions requirePartitionFilter */
|
|
499
|
+
requirePartitionFilter?: (boolean|null);
|
|
500
|
+
|
|
501
|
+
/** BigQueryOptions additionalOptions */
|
|
502
|
+
additionalOptions?: ({ [k: string]: string }|null);
|
|
494
503
|
}
|
|
495
504
|
|
|
496
505
|
/** Represents a BigQueryOptions. */
|
|
@@ -514,6 +523,15 @@ namespace dataform {
|
|
|
514
523
|
/** BigQueryOptions labels. */
|
|
515
524
|
public labels: { [k: string]: string };
|
|
516
525
|
|
|
526
|
+
/** BigQueryOptions partitionExpirationDays. */
|
|
527
|
+
public partitionExpirationDays: number;
|
|
528
|
+
|
|
529
|
+
/** BigQueryOptions requirePartitionFilter. */
|
|
530
|
+
public requirePartitionFilter: boolean;
|
|
531
|
+
|
|
532
|
+
/** BigQueryOptions additionalOptions. */
|
|
533
|
+
public additionalOptions: { [k: string]: string };
|
|
534
|
+
|
|
517
535
|
/**
|
|
518
536
|
* Creates a new BigQueryOptions instance using the specified properties.
|
|
519
537
|
* @param [properties] Properties to set
|
|
@@ -1465,6 +1483,9 @@ namespace dataform {
|
|
|
1465
1483
|
/** Table protected */
|
|
1466
1484
|
"protected"?: (boolean|null);
|
|
1467
1485
|
|
|
1486
|
+
/** Table materialized */
|
|
1487
|
+
materialized?: (boolean|null);
|
|
1488
|
+
|
|
1468
1489
|
/** Table actionDescriptor */
|
|
1469
1490
|
actionDescriptor?: (dataform.IActionDescriptor|null);
|
|
1470
1491
|
|
|
@@ -1544,6 +1565,9 @@ namespace dataform {
|
|
|
1544
1565
|
/** Table protected. */
|
|
1545
1566
|
public protected: boolean;
|
|
1546
1567
|
|
|
1568
|
+
/** Table materialized. */
|
|
1569
|
+
public materialized: boolean;
|
|
1570
|
+
|
|
1547
1571
|
/** Table actionDescriptor. */
|
|
1548
1572
|
public actionDescriptor?: (dataform.IActionDescriptor|null);
|
|
1549
1573
|
|
|
@@ -7785,6 +7809,11 @@ interface IBigQueryOptions {
|
|
|
7785
7809
|
labels?: {
|
|
7786
7810
|
[name: string]: string;
|
|
7787
7811
|
};
|
|
7812
|
+
partitionExpirationDays?: number;
|
|
7813
|
+
requirePartitionFilter?: boolean;
|
|
7814
|
+
additionalOptions?: {
|
|
7815
|
+
[name: string]: string;
|
|
7816
|
+
};
|
|
7788
7817
|
}
|
|
7789
7818
|
interface IPrestoOptions {
|
|
7790
7819
|
partitionBy?: string[];
|
|
@@ -7805,6 +7834,7 @@ interface ITableConfig extends IActionConfig, IDependenciesConfig, IDocumentable
|
|
|
7805
7834
|
presto?: IPrestoOptions;
|
|
7806
7835
|
assertions?: ITableAssertions;
|
|
7807
7836
|
uniqueKey?: string[];
|
|
7837
|
+
materialized?: boolean;
|
|
7808
7838
|
}
|
|
7809
7839
|
interface ITableContext extends ICommonContext {
|
|
7810
7840
|
when: (cond: boolean, trueCase: string, falseCase?: string) => string;
|
|
@@ -7831,6 +7861,7 @@ declare class Table {
|
|
|
7831
7861
|
disabled(): this;
|
|
7832
7862
|
protected(): this;
|
|
7833
7863
|
uniqueKey(uniqueKey: string[]): void;
|
|
7864
|
+
materialized(materialized: boolean): void;
|
|
7834
7865
|
snowflake(snowflake: dataform.ISnowflakeOptions): this;
|
|
7835
7866
|
sqldatawarehouse(sqlDataWarehouse: dataform.ISQLDataWarehouseOptions): this;
|
|
7836
7867
|
redshift(redshift: dataform.IRedshiftOptions): this;
|
|
@@ -7922,11 +7953,13 @@ declare type SqlxConfig = ((ITableConfig & {
|
|
|
7922
7953
|
})) & {
|
|
7923
7954
|
name: string;
|
|
7924
7955
|
};
|
|
7956
|
+
declare type Action = Table | Operation | Assertion | Declaration;
|
|
7925
7957
|
declare class Session {
|
|
7926
7958
|
rootDir: string;
|
|
7927
7959
|
config: dataform.IProjectConfig;
|
|
7928
7960
|
canonicalConfig: dataform.IProjectConfig;
|
|
7929
|
-
actions:
|
|
7961
|
+
actions: Action[];
|
|
7962
|
+
indexedActions: ActionIndex;
|
|
7930
7963
|
tests: {
|
|
7931
7964
|
[name: string]: Test;
|
|
7932
7965
|
};
|
|
@@ -7956,7 +7989,6 @@ declare class Session {
|
|
|
7956
7989
|
compileError(err: Error | string, path?: string, actionTarget?: dataform.ITarget): void;
|
|
7957
7990
|
compile(): dataform.CompiledGraph;
|
|
7958
7991
|
compileToBase64(): string;
|
|
7959
|
-
findActions(target: dataform.ITarget): (Declaration | Table | Assertion | Operation)[];
|
|
7960
7992
|
private getDatabaseSuffixWithUnderscore;
|
|
7961
7993
|
private getSchemaSuffixWithUnderscore;
|
|
7962
7994
|
private getTablePrefixWithUnderscore;
|
|
@@ -7970,6 +8002,15 @@ declare class Session {
|
|
|
7970
8002
|
private checkCircularity;
|
|
7971
8003
|
private checkRunCachingCorrectness;
|
|
7972
8004
|
}
|
|
8005
|
+
declare class ActionIndex {
|
|
8006
|
+
private readonly adapter;
|
|
8007
|
+
private readonly byName;
|
|
8008
|
+
private readonly bySchemaAndName;
|
|
8009
|
+
private readonly byDatabaseAndName;
|
|
8010
|
+
private readonly byDatabaseSchemaAndName;
|
|
8011
|
+
constructor(adapter: IAdapter, actions: Action[]);
|
|
8012
|
+
find(target: dataform.ITarget): Action[];
|
|
8013
|
+
}
|
|
7973
8014
|
|
|
7974
8015
|
declare function main(encodedCoreExecutionRequest: string): string;
|
|
7975
8016
|
|
package/bundle.js
CHANGED
|
@@ -1303,6 +1303,9 @@ const dataform = $root.dataform = (() => {
|
|
|
1303
1303
|
* @property {Array.<string>|null} [clusterBy] BigQueryOptions clusterBy
|
|
1304
1304
|
* @property {string|null} [updatePartitionFilter] BigQueryOptions updatePartitionFilter
|
|
1305
1305
|
* @property {Object.<string,string>|null} [labels] BigQueryOptions labels
|
|
1306
|
+
* @property {number|null} [partitionExpirationDays] BigQueryOptions partitionExpirationDays
|
|
1307
|
+
* @property {boolean|null} [requirePartitionFilter] BigQueryOptions requirePartitionFilter
|
|
1308
|
+
* @property {Object.<string,string>|null} [additionalOptions] BigQueryOptions additionalOptions
|
|
1306
1309
|
*/
|
|
1307
1310
|
|
|
1308
1311
|
/**
|
|
@@ -1316,6 +1319,7 @@ const dataform = $root.dataform = (() => {
|
|
|
1316
1319
|
function BigQueryOptions(properties) {
|
|
1317
1320
|
this.clusterBy = [];
|
|
1318
1321
|
this.labels = {};
|
|
1322
|
+
this.additionalOptions = {};
|
|
1319
1323
|
if (properties)
|
|
1320
1324
|
for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
|
|
1321
1325
|
if (properties[keys[i]] != null)
|
|
@@ -1354,6 +1358,30 @@ const dataform = $root.dataform = (() => {
|
|
|
1354
1358
|
*/
|
|
1355
1359
|
BigQueryOptions.prototype.labels = $util.emptyObject;
|
|
1356
1360
|
|
|
1361
|
+
/**
|
|
1362
|
+
* BigQueryOptions partitionExpirationDays.
|
|
1363
|
+
* @member {number} partitionExpirationDays
|
|
1364
|
+
* @memberof dataform.BigQueryOptions
|
|
1365
|
+
* @instance
|
|
1366
|
+
*/
|
|
1367
|
+
BigQueryOptions.prototype.partitionExpirationDays = 0;
|
|
1368
|
+
|
|
1369
|
+
/**
|
|
1370
|
+
* BigQueryOptions requirePartitionFilter.
|
|
1371
|
+
* @member {boolean} requirePartitionFilter
|
|
1372
|
+
* @memberof dataform.BigQueryOptions
|
|
1373
|
+
* @instance
|
|
1374
|
+
*/
|
|
1375
|
+
BigQueryOptions.prototype.requirePartitionFilter = false;
|
|
1376
|
+
|
|
1377
|
+
/**
|
|
1378
|
+
* BigQueryOptions additionalOptions.
|
|
1379
|
+
* @member {Object.<string,string>} additionalOptions
|
|
1380
|
+
* @memberof dataform.BigQueryOptions
|
|
1381
|
+
* @instance
|
|
1382
|
+
*/
|
|
1383
|
+
BigQueryOptions.prototype.additionalOptions = $util.emptyObject;
|
|
1384
|
+
|
|
1357
1385
|
/**
|
|
1358
1386
|
* Creates a new BigQueryOptions instance using the specified properties.
|
|
1359
1387
|
* @function create
|
|
@@ -1388,6 +1416,13 @@ const dataform = $root.dataform = (() => {
|
|
|
1388
1416
|
if (message.labels != null && message.hasOwnProperty("labels"))
|
|
1389
1417
|
for (let keys = Object.keys(message.labels), i = 0; i < keys.length; ++i)
|
|
1390
1418
|
writer.uint32(/* id 4, wireType 2 =*/34).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.labels[keys[i]]).ldelim();
|
|
1419
|
+
if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
|
|
1420
|
+
writer.uint32(/* id 5, wireType 0 =*/40).int32(message.partitionExpirationDays);
|
|
1421
|
+
if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
|
|
1422
|
+
writer.uint32(/* id 6, wireType 0 =*/48).bool(message.requirePartitionFilter);
|
|
1423
|
+
if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions"))
|
|
1424
|
+
for (let keys = Object.keys(message.additionalOptions), i = 0; i < keys.length; ++i)
|
|
1425
|
+
writer.uint32(/* id 7, wireType 2 =*/58).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.additionalOptions[keys[i]]).ldelim();
|
|
1391
1426
|
return writer;
|
|
1392
1427
|
};
|
|
1393
1428
|
|
|
@@ -1441,6 +1476,20 @@ const dataform = $root.dataform = (() => {
|
|
|
1441
1476
|
reader.pos++;
|
|
1442
1477
|
message.labels[key] = reader.string();
|
|
1443
1478
|
break;
|
|
1479
|
+
case 5:
|
|
1480
|
+
message.partitionExpirationDays = reader.int32();
|
|
1481
|
+
break;
|
|
1482
|
+
case 6:
|
|
1483
|
+
message.requirePartitionFilter = reader.bool();
|
|
1484
|
+
break;
|
|
1485
|
+
case 7:
|
|
1486
|
+
reader.skip().pos++;
|
|
1487
|
+
if (message.additionalOptions === $util.emptyObject)
|
|
1488
|
+
message.additionalOptions = {};
|
|
1489
|
+
key = reader.string();
|
|
1490
|
+
reader.pos++;
|
|
1491
|
+
message.additionalOptions[key] = reader.string();
|
|
1492
|
+
break;
|
|
1444
1493
|
default:
|
|
1445
1494
|
reader.skipType(tag & 7);
|
|
1446
1495
|
break;
|
|
@@ -1497,6 +1546,20 @@ const dataform = $root.dataform = (() => {
|
|
|
1497
1546
|
if (!$util.isString(message.labels[key[i]]))
|
|
1498
1547
|
return "labels: string{k:string} expected";
|
|
1499
1548
|
}
|
|
1549
|
+
if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
|
|
1550
|
+
if (!$util.isInteger(message.partitionExpirationDays))
|
|
1551
|
+
return "partitionExpirationDays: integer expected";
|
|
1552
|
+
if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
|
|
1553
|
+
if (typeof message.requirePartitionFilter !== "boolean")
|
|
1554
|
+
return "requirePartitionFilter: boolean expected";
|
|
1555
|
+
if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions")) {
|
|
1556
|
+
if (!$util.isObject(message.additionalOptions))
|
|
1557
|
+
return "additionalOptions: object expected";
|
|
1558
|
+
let key = Object.keys(message.additionalOptions);
|
|
1559
|
+
for (let i = 0; i < key.length; ++i)
|
|
1560
|
+
if (!$util.isString(message.additionalOptions[key[i]]))
|
|
1561
|
+
return "additionalOptions: string{k:string} expected";
|
|
1562
|
+
}
|
|
1500
1563
|
return null;
|
|
1501
1564
|
};
|
|
1502
1565
|
|
|
@@ -1530,6 +1593,17 @@ const dataform = $root.dataform = (() => {
|
|
|
1530
1593
|
for (let keys = Object.keys(object.labels), i = 0; i < keys.length; ++i)
|
|
1531
1594
|
message.labels[keys[i]] = String(object.labels[keys[i]]);
|
|
1532
1595
|
}
|
|
1596
|
+
if (object.partitionExpirationDays != null)
|
|
1597
|
+
message.partitionExpirationDays = object.partitionExpirationDays | 0;
|
|
1598
|
+
if (object.requirePartitionFilter != null)
|
|
1599
|
+
message.requirePartitionFilter = Boolean(object.requirePartitionFilter);
|
|
1600
|
+
if (object.additionalOptions) {
|
|
1601
|
+
if (typeof object.additionalOptions !== "object")
|
|
1602
|
+
throw TypeError(".dataform.BigQueryOptions.additionalOptions: object expected");
|
|
1603
|
+
message.additionalOptions = {};
|
|
1604
|
+
for (let keys = Object.keys(object.additionalOptions), i = 0; i < keys.length; ++i)
|
|
1605
|
+
message.additionalOptions[keys[i]] = String(object.additionalOptions[keys[i]]);
|
|
1606
|
+
}
|
|
1533
1607
|
return message;
|
|
1534
1608
|
};
|
|
1535
1609
|
|
|
@@ -1548,11 +1622,15 @@ const dataform = $root.dataform = (() => {
|
|
|
1548
1622
|
let object = {};
|
|
1549
1623
|
if (options.arrays || options.defaults)
|
|
1550
1624
|
object.clusterBy = [];
|
|
1551
|
-
if (options.objects || options.defaults)
|
|
1625
|
+
if (options.objects || options.defaults) {
|
|
1552
1626
|
object.labels = {};
|
|
1627
|
+
object.additionalOptions = {};
|
|
1628
|
+
}
|
|
1553
1629
|
if (options.defaults) {
|
|
1554
1630
|
object.partitionBy = "";
|
|
1555
1631
|
object.updatePartitionFilter = "";
|
|
1632
|
+
object.partitionExpirationDays = 0;
|
|
1633
|
+
object.requirePartitionFilter = false;
|
|
1556
1634
|
}
|
|
1557
1635
|
if (message.partitionBy != null && message.hasOwnProperty("partitionBy"))
|
|
1558
1636
|
object.partitionBy = message.partitionBy;
|
|
@@ -1569,6 +1647,15 @@ const dataform = $root.dataform = (() => {
|
|
|
1569
1647
|
for (let j = 0; j < keys2.length; ++j)
|
|
1570
1648
|
object.labels[keys2[j]] = message.labels[keys2[j]];
|
|
1571
1649
|
}
|
|
1650
|
+
if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
|
|
1651
|
+
object.partitionExpirationDays = message.partitionExpirationDays;
|
|
1652
|
+
if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
|
|
1653
|
+
object.requirePartitionFilter = message.requirePartitionFilter;
|
|
1654
|
+
if (message.additionalOptions && (keys2 = Object.keys(message.additionalOptions)).length) {
|
|
1655
|
+
object.additionalOptions = {};
|
|
1656
|
+
for (let j = 0; j < keys2.length; ++j)
|
|
1657
|
+
object.additionalOptions[keys2[j]] = message.additionalOptions[keys2[j]];
|
|
1658
|
+
}
|
|
1572
1659
|
return object;
|
|
1573
1660
|
};
|
|
1574
1661
|
|
|
@@ -3770,6 +3857,7 @@ const dataform = $root.dataform = (() => {
|
|
|
3770
3857
|
* @property {string|null} [type] Table type
|
|
3771
3858
|
* @property {string|null} [query] Table query
|
|
3772
3859
|
* @property {boolean|null} ["protected"] Table protected
|
|
3860
|
+
* @property {boolean|null} [materialized] Table materialized
|
|
3773
3861
|
* @property {dataform.IActionDescriptor|null} [actionDescriptor] Table actionDescriptor
|
|
3774
3862
|
* @property {Array.<string>|null} [tags] Table tags
|
|
3775
3863
|
* @property {string|null} [where] Table where
|
|
@@ -3873,6 +3961,14 @@ const dataform = $root.dataform = (() => {
|
|
|
3873
3961
|
*/
|
|
3874
3962
|
Table.prototype["protected"] = false;
|
|
3875
3963
|
|
|
3964
|
+
/**
|
|
3965
|
+
* Table materialized.
|
|
3966
|
+
* @member {boolean} materialized
|
|
3967
|
+
* @memberof dataform.Table
|
|
3968
|
+
* @instance
|
|
3969
|
+
*/
|
|
3970
|
+
Table.prototype.materialized = false;
|
|
3971
|
+
|
|
3876
3972
|
/**
|
|
3877
3973
|
* Table actionDescriptor.
|
|
3878
3974
|
* @member {dataform.IActionDescriptor|null|undefined} actionDescriptor
|
|
@@ -4070,6 +4166,8 @@ const dataform = $root.dataform = (() => {
|
|
|
4070
4166
|
$root.dataform.SnowflakeOptions.encode(message.snowflake, writer.uint32(/* id 33, wireType 2 =*/266).fork()).ldelim();
|
|
4071
4167
|
if (message.presto != null && message.hasOwnProperty("presto"))
|
|
4072
4168
|
$root.dataform.PrestoOptions.encode(message.presto, writer.uint32(/* id 34, wireType 2 =*/274).fork()).ldelim();
|
|
4169
|
+
if (message.materialized != null && message.hasOwnProperty("materialized"))
|
|
4170
|
+
writer.uint32(/* id 35, wireType 0 =*/280).bool(message.materialized);
|
|
4073
4171
|
return writer;
|
|
4074
4172
|
};
|
|
4075
4173
|
|
|
@@ -4130,6 +4228,9 @@ const dataform = $root.dataform = (() => {
|
|
|
4130
4228
|
case 9:
|
|
4131
4229
|
message["protected"] = reader.bool();
|
|
4132
4230
|
break;
|
|
4231
|
+
case 35:
|
|
4232
|
+
message.materialized = reader.bool();
|
|
4233
|
+
break;
|
|
4133
4234
|
case 24:
|
|
4134
4235
|
message.actionDescriptor = $root.dataform.ActionDescriptor.decode(reader, reader.uint32());
|
|
4135
4236
|
break;
|
|
@@ -4262,6 +4363,9 @@ const dataform = $root.dataform = (() => {
|
|
|
4262
4363
|
if (message["protected"] != null && message.hasOwnProperty("protected"))
|
|
4263
4364
|
if (typeof message["protected"] !== "boolean")
|
|
4264
4365
|
return "protected: boolean expected";
|
|
4366
|
+
if (message.materialized != null && message.hasOwnProperty("materialized"))
|
|
4367
|
+
if (typeof message.materialized !== "boolean")
|
|
4368
|
+
return "materialized: boolean expected";
|
|
4265
4369
|
if (message.actionDescriptor != null && message.hasOwnProperty("actionDescriptor")) {
|
|
4266
4370
|
let error = $root.dataform.ActionDescriptor.verify(message.actionDescriptor);
|
|
4267
4371
|
if (error)
|
|
@@ -4400,6 +4504,8 @@ const dataform = $root.dataform = (() => {
|
|
|
4400
4504
|
message.query = String(object.query);
|
|
4401
4505
|
if (object["protected"] != null)
|
|
4402
4506
|
message["protected"] = Boolean(object["protected"]);
|
|
4507
|
+
if (object.materialized != null)
|
|
4508
|
+
message.materialized = Boolean(object.materialized);
|
|
4403
4509
|
if (object.actionDescriptor != null) {
|
|
4404
4510
|
if (typeof object.actionDescriptor !== "object")
|
|
4405
4511
|
throw TypeError(".dataform.Table.actionDescriptor: object expected");
|
|
@@ -4520,6 +4626,7 @@ const dataform = $root.dataform = (() => {
|
|
|
4520
4626
|
object.canonicalTarget = null;
|
|
4521
4627
|
object.snowflake = null;
|
|
4522
4628
|
object.presto = null;
|
|
4629
|
+
object.materialized = false;
|
|
4523
4630
|
}
|
|
4524
4631
|
if (message.type != null && message.hasOwnProperty("type"))
|
|
4525
4632
|
object.type = message.type;
|
|
@@ -4588,6 +4695,8 @@ const dataform = $root.dataform = (() => {
|
|
|
4588
4695
|
object.snowflake = $root.dataform.SnowflakeOptions.toObject(message.snowflake, options);
|
|
4589
4696
|
if (message.presto != null && message.hasOwnProperty("presto"))
|
|
4590
4697
|
object.presto = $root.dataform.PrestoOptions.toObject(message.presto, options);
|
|
4698
|
+
if (message.materialized != null && message.hasOwnProperty("materialized"))
|
|
4699
|
+
object.materialized = message.materialized;
|
|
4591
4700
|
return object;
|
|
4592
4701
|
};
|
|
4593
4702
|
|
|
@@ -19836,11 +19945,26 @@ class BigQueryAdapter extends Adapter {
|
|
|
19836
19945
|
return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
|
|
19837
19946
|
}
|
|
19838
19947
|
createOrReplace(table) {
|
|
19839
|
-
|
|
19948
|
+
const options = [];
|
|
19949
|
+
if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) {
|
|
19950
|
+
options.push(`partition_expiration_days=${table.bigquery.partitionExpirationDays}`);
|
|
19951
|
+
}
|
|
19952
|
+
if (table.bigquery && table.bigquery.partitionBy && table.bigquery.requirePartitionFilter) {
|
|
19953
|
+
options.push(`require_partition_filter=${table.bigquery.requirePartitionFilter}`);
|
|
19954
|
+
}
|
|
19955
|
+
if (table.bigquery && table.bigquery.additionalOptions) {
|
|
19956
|
+
for (const [optionName, optionValue] of Object.entries(table.bigquery.additionalOptions)) {
|
|
19957
|
+
options.push(`${optionName}=${optionValue}`);
|
|
19958
|
+
}
|
|
19959
|
+
}
|
|
19960
|
+
return `create or replace ${table.materialized
|
|
19961
|
+
? "materialized "
|
|
19962
|
+
: ""}${this.tableTypeAsSql(this.baseTableType(table.type))} ${this.resolveTarget(table.target)} ${table.bigquery && table.bigquery.partitionBy
|
|
19840
19963
|
? `partition by ${table.bigquery.partitionBy} `
|
|
19841
19964
|
: ""}${table.bigquery && table.bigquery.clusterBy && table.bigquery.clusterBy.length > 0
|
|
19842
19965
|
? `cluster by ${table.bigquery.clusterBy.join(", ")} `
|
|
19843
|
-
: ""}
|
|
19966
|
+
: ""}${options.length > 0 ?
|
|
19967
|
+
`OPTIONS(${options.join(',')})` : ""}as ${table.query}`;
|
|
19844
19968
|
}
|
|
19845
19969
|
createOrReplaceView(target, query) {
|
|
19846
19970
|
return `
|
|
@@ -20053,17 +20177,17 @@ class SnowflakeAdapter extends Adapter {
|
|
|
20053
20177
|
assertTasks(assertion, projectConfig) {
|
|
20054
20178
|
const tasks = Tasks.create();
|
|
20055
20179
|
const target = assertion.target;
|
|
20056
|
-
tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query, false)));
|
|
20180
|
+
tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query, false, false)));
|
|
20057
20181
|
tasks.add(Task.assertion(`select sum(1) as row_count from ${this.resolveTarget(target)}`));
|
|
20058
20182
|
return tasks;
|
|
20059
20183
|
}
|
|
20060
|
-
createOrReplaceView(target, query, secure) {
|
|
20061
|
-
return `create or replace ${secure ? "secure " : ""}view ${this.resolveTarget(target)} as ${query}`;
|
|
20184
|
+
createOrReplaceView(target, query, secure, materialized) {
|
|
20185
|
+
return `create or replace ${secure ? "secure " : ""}${materialized ? "materialized " : ""}view ${this.resolveTarget(target)} as ${query}`;
|
|
20062
20186
|
}
|
|
20063
20187
|
createOrReplace(table) {
|
|
20064
20188
|
var _a, _b, _c, _d, _e;
|
|
20065
20189
|
if (table.type === "view") {
|
|
20066
|
-
return this.createOrReplaceView(table.target, table.query, (_a = table.snowflake) === null || _a === void 0 ? void 0 : _a.secure);
|
|
20190
|
+
return this.createOrReplaceView(table.target, table.query, (_a = table.snowflake) === null || _a === void 0 ? void 0 : _a.secure, table.materialized);
|
|
20067
20191
|
}
|
|
20068
20192
|
return `create or replace ${((_b = table.snowflake) === null || _b === void 0 ? void 0 : _b.transient) ? "transient " : ""}table ${this.resolveTarget(table.target)} ${((_d = (_c = table.snowflake) === null || _c === void 0 ? void 0 : _c.clusterBy) === null || _d === void 0 ? void 0 : _d.length) > 0
|
|
20069
20193
|
? `cluster by (${(_e = table.snowflake) === null || _e === void 0 ? void 0 : _e.clusterBy.join(", ")}) `
|
|
@@ -20841,7 +20965,7 @@ const SortStyleType = ["compound", "interleaved"];
|
|
|
20841
20965
|
const IRedshiftOptionsProperties = () => strictKeysOf()(["distKey", "distStyle", "sortKeys", "sortStyle"]);
|
|
20842
20966
|
const ISnowflakeOptionsProperties = () => strictKeysOf()(["secure", "transient", "clusterBy"]);
|
|
20843
20967
|
const ISQLDataWarehouseOptionsProperties = () => strictKeysOf()(["distribution"]);
|
|
20844
|
-
const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels"]);
|
|
20968
|
+
const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels", "partitionExpirationDays", "requirePartitionFilter", "additionalOptions"]);
|
|
20845
20969
|
const IPrestoOptionsProperties = () => strictKeysOf()(["partitionBy"]);
|
|
20846
20970
|
const ITableAssertionsProperties = () => strictKeysOf()(["uniqueKey", "uniqueKeys", "nonNull", "rowConditions"]);
|
|
20847
20971
|
const ITableConfigProperties = () => strictKeysOf()([
|
|
@@ -20862,7 +20986,8 @@ const ITableConfigProperties = () => strictKeysOf()([
|
|
|
20862
20986
|
"assertions",
|
|
20863
20987
|
"database",
|
|
20864
20988
|
"columns",
|
|
20865
|
-
"description"
|
|
20989
|
+
"description",
|
|
20990
|
+
"materialized"
|
|
20866
20991
|
]);
|
|
20867
20992
|
class Table {
|
|
20868
20993
|
constructor() {
|
|
@@ -20928,6 +21053,9 @@ class Table {
|
|
|
20928
21053
|
if (config.uniqueKey) {
|
|
20929
21054
|
this.uniqueKey(config.uniqueKey);
|
|
20930
21055
|
}
|
|
21056
|
+
if (config.materialized) {
|
|
21057
|
+
this.materialized(config.materialized);
|
|
21058
|
+
}
|
|
20931
21059
|
return this;
|
|
20932
21060
|
}
|
|
20933
21061
|
type(type) {
|
|
@@ -20964,6 +21092,9 @@ class Table {
|
|
|
20964
21092
|
uniqueKey(uniqueKey) {
|
|
20965
21093
|
this.proto.uniqueKey = uniqueKey;
|
|
20966
21094
|
}
|
|
21095
|
+
materialized(materialized) {
|
|
21096
|
+
this.proto.materialized = materialized;
|
|
21097
|
+
}
|
|
20967
21098
|
snowflake(snowflake) {
|
|
20968
21099
|
checkExcessProperties((e) => this.session.compileError(e), snowflake, ISnowflakeOptionsProperties(), "snowflake config");
|
|
20969
21100
|
this.proto.snowflake = dataform.SnowflakeOptions.create(snowflake);
|
|
@@ -22146,7 +22277,7 @@ class Test {
|
|
|
22146
22277
|
this.session.compileError(new Error("Tests must operate upon a specified dataset."), this.proto.fileName);
|
|
22147
22278
|
}
|
|
22148
22279
|
else {
|
|
22149
|
-
const allResolved = this.session.
|
|
22280
|
+
const allResolved = this.session.indexedActions.find(resolvableAsTarget(this.datasetToTest));
|
|
22150
22281
|
if (allResolved.length > 1) {
|
|
22151
22282
|
this.session.compileError(new Error(ambiguousActionNameMsg(this.datasetToTest, allResolved)), this.proto.fileName);
|
|
22152
22283
|
}
|
|
@@ -22246,7 +22377,7 @@ class RefReplacingContext {
|
|
|
22246
22377
|
}
|
|
22247
22378
|
}
|
|
22248
22379
|
|
|
22249
|
-
const version = "1.
|
|
22380
|
+
const version = "1.22.0";
|
|
22250
22381
|
|
|
22251
22382
|
const SQL_DATA_WAREHOUSE_DIST_HASH_REGEXP = new RegExp("HASH\\s*\\(\\s*\\w*\\s*\\)\\s*");
|
|
22252
22383
|
const DEFAULT_CONFIG = {
|
|
@@ -22356,7 +22487,7 @@ class Session {
|
|
|
22356
22487
|
}
|
|
22357
22488
|
resolve(ref, ...rest) {
|
|
22358
22489
|
ref = toResolvable(ref, rest);
|
|
22359
|
-
const allResolved = this.
|
|
22490
|
+
const allResolved = this.indexedActions.find(resolvableAsTarget(ref));
|
|
22360
22491
|
if (allResolved.length > 1) {
|
|
22361
22492
|
this.compileError(new Error(ambiguousActionNameMsg(ref, allResolved)));
|
|
22362
22493
|
}
|
|
@@ -22452,6 +22583,7 @@ class Session {
|
|
|
22452
22583
|
this.graphErrors.compilationErrors.push(compileError);
|
|
22453
22584
|
}
|
|
22454
22585
|
compile() {
|
|
22586
|
+
this.indexedActions = new ActionIndex(this.adapter(), this.actions);
|
|
22455
22587
|
const compiledGraph = dataform.CompiledGraph.create({
|
|
22456
22588
|
projectConfig: this.config,
|
|
22457
22589
|
tables: this.compileGraphChunk(this.actions.filter(action => action instanceof Table), dataform.Table.verify),
|
|
@@ -22480,20 +22612,6 @@ class Session {
|
|
|
22480
22612
|
compileToBase64() {
|
|
22481
22613
|
return encode64(dataform.CompiledGraph, this.compile());
|
|
22482
22614
|
}
|
|
22483
|
-
findActions(target) {
|
|
22484
|
-
const adapter = this.adapter();
|
|
22485
|
-
return this.actions.filter(action => {
|
|
22486
|
-
if (!!target.database &&
|
|
22487
|
-
action.proto.target.database !== adapter.normalizeIdentifier(target.database)) {
|
|
22488
|
-
return false;
|
|
22489
|
-
}
|
|
22490
|
-
if (!!target.schema &&
|
|
22491
|
-
action.proto.target.schema !== adapter.normalizeIdentifier(target.schema)) {
|
|
22492
|
-
return false;
|
|
22493
|
-
}
|
|
22494
|
-
return action.proto.target.name === adapter.normalizeIdentifier(target.name);
|
|
22495
|
-
});
|
|
22496
|
-
}
|
|
22497
22615
|
getDatabaseSuffixWithUnderscore() {
|
|
22498
22616
|
return !!this.config.databaseSuffix ? `_${this.config.databaseSuffix}` : "";
|
|
22499
22617
|
}
|
|
@@ -22521,7 +22639,7 @@ class Session {
|
|
|
22521
22639
|
actions.forEach(action => {
|
|
22522
22640
|
const fullyQualifiedDependencies = {};
|
|
22523
22641
|
for (const dependency of action.dependencyTargets) {
|
|
22524
|
-
const possibleDeps = this.
|
|
22642
|
+
const possibleDeps = this.indexedActions.find(dependency);
|
|
22525
22643
|
if (possibleDeps.length === 0) {
|
|
22526
22644
|
this.compileError(new Error(`Missing dependency detected: Action "${targetAsReadableString(action.target)}" depends on "${stringifyResolvable(dependency)}" which does not exist`), action.fileName, action.target);
|
|
22527
22645
|
}
|
|
@@ -22591,6 +22709,11 @@ class Session {
|
|
|
22591
22709
|
if (!!table.type && !TableType.includes(table.type)) {
|
|
22592
22710
|
this.compileError(`Wrong type of table detected. Should only use predefined types: ${joinQuoted(TableType)}`, table.fileName, table.target);
|
|
22593
22711
|
}
|
|
22712
|
+
if (!!table.materialized) {
|
|
22713
|
+
if (table.type !== "view" || (this.config.warehouse !== "snowflake" && this.config.warehouse !== "bigquery")) {
|
|
22714
|
+
this.compileError(new Error(`The 'materialized' option is only valid for Snowflake and BigQuery views`), table.fileName, table.target);
|
|
22715
|
+
}
|
|
22716
|
+
}
|
|
22594
22717
|
if (!!table.snowflake) {
|
|
22595
22718
|
if (table.snowflake.secure && table.type !== "view") {
|
|
22596
22719
|
this.compileError(new Error(`The 'secure' option is only valid for Snowflake views`), table.fileName, table.target);
|
|
@@ -22646,9 +22769,22 @@ class Session {
|
|
|
22646
22769
|
}
|
|
22647
22770
|
}
|
|
22648
22771
|
if (!!table.bigquery) {
|
|
22649
|
-
if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length)
|
|
22772
|
+
if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length) || table.bigquery.partitionExpirationDays
|
|
22773
|
+
|| table.bigquery.requirePartitionFilter) &&
|
|
22650
22774
|
table.type === "view") {
|
|
22651
|
-
this.compileError(`partitionBy/clusterBy are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
|
|
22775
|
+
this.compileError(`partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
|
|
22776
|
+
}
|
|
22777
|
+
else if ((!table.bigquery.partitionBy && (table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter)) &&
|
|
22778
|
+
table.type === "table") {
|
|
22779
|
+
this.compileError(`requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables`, table.fileName, table.target);
|
|
22780
|
+
}
|
|
22781
|
+
else if (table.bigquery.additionalOptions) {
|
|
22782
|
+
if (table.bigquery.partitionExpirationDays && table.bigquery.additionalOptions.partition_expiration_days) {
|
|
22783
|
+
this.compileError(`partitionExpirationDays has been declared twice`, table.fileName, table.target);
|
|
22784
|
+
}
|
|
22785
|
+
if (table.bigquery.requirePartitionFilter && table.bigquery.additionalOptions.require_partition_filter) {
|
|
22786
|
+
this.compileError(`requirePartitionFilter has been declared twice`, table.fileName, table.target);
|
|
22787
|
+
}
|
|
22652
22788
|
}
|
|
22653
22789
|
}
|
|
22654
22790
|
if (!!Table.IGNORED_PROPS[table.type]) {
|
|
@@ -22720,6 +22856,70 @@ function objectExistsOrIsNonEmpty(prop) {
|
|
|
22720
22856
|
(!Array.isArray(prop) && typeof prop === "object" && !!Object.keys(prop).length) ||
|
|
22721
22857
|
typeof prop !== "object");
|
|
22722
22858
|
}
|
|
22859
|
+
class ActionIndex {
|
|
22860
|
+
constructor(adapter, actions) {
|
|
22861
|
+
this.adapter = adapter;
|
|
22862
|
+
this.byName = new Map();
|
|
22863
|
+
this.bySchemaAndName = new Map();
|
|
22864
|
+
this.byDatabaseAndName = new Map();
|
|
22865
|
+
this.byDatabaseSchemaAndName = new Map();
|
|
22866
|
+
for (const action of actions) {
|
|
22867
|
+
if (!this.byName.has(action.proto.target.name)) {
|
|
22868
|
+
this.byName.set(action.proto.target.name, []);
|
|
22869
|
+
}
|
|
22870
|
+
this.byName.get(action.proto.target.name).push(action);
|
|
22871
|
+
if (!this.bySchemaAndName.has(action.proto.target.schema)) {
|
|
22872
|
+
this.bySchemaAndName.set(action.proto.target.schema, new Map());
|
|
22873
|
+
}
|
|
22874
|
+
const forSchema = this.bySchemaAndName.get(action.proto.target.schema);
|
|
22875
|
+
if (!forSchema.has(action.proto.target.name)) {
|
|
22876
|
+
forSchema.set(action.proto.target.name, []);
|
|
22877
|
+
}
|
|
22878
|
+
forSchema.get(action.proto.target.name).push(action);
|
|
22879
|
+
if (!!action.proto.target.database) {
|
|
22880
|
+
if (!this.byDatabaseAndName.has(action.proto.target.database)) {
|
|
22881
|
+
this.byDatabaseAndName.set(action.proto.target.database, new Map());
|
|
22882
|
+
}
|
|
22883
|
+
const forDatabaseNoSchema = this.byDatabaseAndName.get(action.proto.target.database);
|
|
22884
|
+
if (!forDatabaseNoSchema.has(action.proto.target.name)) {
|
|
22885
|
+
forDatabaseNoSchema.set(action.proto.target.name, []);
|
|
22886
|
+
}
|
|
22887
|
+
forDatabaseNoSchema.get(action.proto.target.name).push(action);
|
|
22888
|
+
if (!this.byDatabaseSchemaAndName.has(action.proto.target.database)) {
|
|
22889
|
+
this.byDatabaseSchemaAndName.set(action.proto.target.database, new Map());
|
|
22890
|
+
}
|
|
22891
|
+
const forDatabase = this.byDatabaseSchemaAndName.get(action.proto.target.database);
|
|
22892
|
+
if (!forDatabase.has(action.proto.target.schema)) {
|
|
22893
|
+
forDatabase.set(action.proto.target.schema, new Map());
|
|
22894
|
+
}
|
|
22895
|
+
const forDatabaseAndSchema = forDatabase.get(action.proto.target.schema);
|
|
22896
|
+
if (!forDatabaseAndSchema.has(action.proto.target.name)) {
|
|
22897
|
+
forDatabaseAndSchema.set(action.proto.target.name, []);
|
|
22898
|
+
}
|
|
22899
|
+
forDatabaseAndSchema.get(action.proto.target.name).push(action);
|
|
22900
|
+
}
|
|
22901
|
+
}
|
|
22902
|
+
}
|
|
22903
|
+
find(target) {
|
|
22904
|
+
if (!!target.database) {
|
|
22905
|
+
if (!!target.schema) {
|
|
22906
|
+
return (this.byDatabaseSchemaAndName
|
|
22907
|
+
.get(this.adapter.normalizeIdentifier(target.database))
|
|
22908
|
+
.get(this.adapter.normalizeIdentifier(target.schema))
|
|
22909
|
+
.get(this.adapter.normalizeIdentifier(target.name)) || []);
|
|
22910
|
+
}
|
|
22911
|
+
return (this.byDatabaseAndName
|
|
22912
|
+
.get(this.adapter.normalizeIdentifier(target.database))
|
|
22913
|
+
.get(this.adapter.normalizeIdentifier(target.name)) || []);
|
|
22914
|
+
}
|
|
22915
|
+
if (!!target.schema) {
|
|
22916
|
+
return (this.bySchemaAndName
|
|
22917
|
+
.get(this.adapter.normalizeIdentifier(target.schema))
|
|
22918
|
+
.get(this.adapter.normalizeIdentifier(target.name)) || []);
|
|
22919
|
+
}
|
|
22920
|
+
return this.byName.get(this.adapter.normalizeIdentifier(target.name)) || [];
|
|
22921
|
+
}
|
|
22922
|
+
}
|
|
22723
22923
|
|
|
22724
22924
|
function main(encodedCoreExecutionRequest) {
|
|
22725
22925
|
var _a;
|