@dataform/core 1.21.1 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/bundle.d.ts +23 -0
  2. package/bundle.js +119 -6
  3. package/package.json +1 -1
package/bundle.d.ts CHANGED
@@ -491,6 +491,15 @@ namespace dataform {
491
491
 
492
492
  /** BigQueryOptions labels */
493
493
  labels?: ({ [k: string]: string }|null);
494
+
495
+ /** BigQueryOptions partitionExpirationDays */
496
+ partitionExpirationDays?: (number|null);
497
+
498
+ /** BigQueryOptions requirePartitionFilter */
499
+ requirePartitionFilter?: (boolean|null);
500
+
501
+ /** BigQueryOptions additionalOptions */
502
+ additionalOptions?: ({ [k: string]: string }|null);
494
503
  }
495
504
 
496
505
  /** Represents a BigQueryOptions. */
@@ -514,6 +523,15 @@ namespace dataform {
514
523
  /** BigQueryOptions labels. */
515
524
  public labels: { [k: string]: string };
516
525
 
526
+ /** BigQueryOptions partitionExpirationDays. */
527
+ public partitionExpirationDays: number;
528
+
529
+ /** BigQueryOptions requirePartitionFilter. */
530
+ public requirePartitionFilter: boolean;
531
+
532
+ /** BigQueryOptions additionalOptions. */
533
+ public additionalOptions: { [k: string]: string };
534
+
517
535
  /**
518
536
  * Creates a new BigQueryOptions instance using the specified properties.
519
537
  * @param [properties] Properties to set
@@ -7791,6 +7809,11 @@ interface IBigQueryOptions {
7791
7809
  labels?: {
7792
7810
  [name: string]: string;
7793
7811
  };
7812
+ partitionExpirationDays?: number;
7813
+ requirePartitionFilter?: boolean;
7814
+ additionalOptions?: {
7815
+ [name: string]: string;
7816
+ };
7794
7817
  }
7795
7818
  interface IPrestoOptions {
7796
7819
  partitionBy?: string[];
package/bundle.js CHANGED
@@ -1303,6 +1303,9 @@ const dataform = $root.dataform = (() => {
1303
1303
  * @property {Array.<string>|null} [clusterBy] BigQueryOptions clusterBy
1304
1304
  * @property {string|null} [updatePartitionFilter] BigQueryOptions updatePartitionFilter
1305
1305
  * @property {Object.<string,string>|null} [labels] BigQueryOptions labels
1306
+ * @property {number|null} [partitionExpirationDays] BigQueryOptions partitionExpirationDays
1307
+ * @property {boolean|null} [requirePartitionFilter] BigQueryOptions requirePartitionFilter
1308
+ * @property {Object.<string,string>|null} [additionalOptions] BigQueryOptions additionalOptions
1306
1309
  */
1307
1310
 
1308
1311
  /**
@@ -1316,6 +1319,7 @@ const dataform = $root.dataform = (() => {
1316
1319
  function BigQueryOptions(properties) {
1317
1320
  this.clusterBy = [];
1318
1321
  this.labels = {};
1322
+ this.additionalOptions = {};
1319
1323
  if (properties)
1320
1324
  for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
1321
1325
  if (properties[keys[i]] != null)
@@ -1354,6 +1358,30 @@ const dataform = $root.dataform = (() => {
1354
1358
  */
1355
1359
  BigQueryOptions.prototype.labels = $util.emptyObject;
1356
1360
 
1361
+ /**
1362
+ * BigQueryOptions partitionExpirationDays.
1363
+ * @member {number} partitionExpirationDays
1364
+ * @memberof dataform.BigQueryOptions
1365
+ * @instance
1366
+ */
1367
+ BigQueryOptions.prototype.partitionExpirationDays = 0;
1368
+
1369
+ /**
1370
+ * BigQueryOptions requirePartitionFilter.
1371
+ * @member {boolean} requirePartitionFilter
1372
+ * @memberof dataform.BigQueryOptions
1373
+ * @instance
1374
+ */
1375
+ BigQueryOptions.prototype.requirePartitionFilter = false;
1376
+
1377
+ /**
1378
+ * BigQueryOptions additionalOptions.
1379
+ * @member {Object.<string,string>} additionalOptions
1380
+ * @memberof dataform.BigQueryOptions
1381
+ * @instance
1382
+ */
1383
+ BigQueryOptions.prototype.additionalOptions = $util.emptyObject;
1384
+
1357
1385
  /**
1358
1386
  * Creates a new BigQueryOptions instance using the specified properties.
1359
1387
  * @function create
@@ -1388,6 +1416,13 @@ const dataform = $root.dataform = (() => {
1388
1416
  if (message.labels != null && message.hasOwnProperty("labels"))
1389
1417
  for (let keys = Object.keys(message.labels), i = 0; i < keys.length; ++i)
1390
1418
  writer.uint32(/* id 4, wireType 2 =*/34).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.labels[keys[i]]).ldelim();
1419
+ if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
1420
+ writer.uint32(/* id 5, wireType 0 =*/40).int32(message.partitionExpirationDays);
1421
+ if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
1422
+ writer.uint32(/* id 6, wireType 0 =*/48).bool(message.requirePartitionFilter);
1423
+ if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions"))
1424
+ for (let keys = Object.keys(message.additionalOptions), i = 0; i < keys.length; ++i)
1425
+ writer.uint32(/* id 7, wireType 2 =*/58).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.additionalOptions[keys[i]]).ldelim();
1391
1426
  return writer;
1392
1427
  };
1393
1428
 
@@ -1441,6 +1476,20 @@ const dataform = $root.dataform = (() => {
1441
1476
  reader.pos++;
1442
1477
  message.labels[key] = reader.string();
1443
1478
  break;
1479
+ case 5:
1480
+ message.partitionExpirationDays = reader.int32();
1481
+ break;
1482
+ case 6:
1483
+ message.requirePartitionFilter = reader.bool();
1484
+ break;
1485
+ case 7:
1486
+ reader.skip().pos++;
1487
+ if (message.additionalOptions === $util.emptyObject)
1488
+ message.additionalOptions = {};
1489
+ key = reader.string();
1490
+ reader.pos++;
1491
+ message.additionalOptions[key] = reader.string();
1492
+ break;
1444
1493
  default:
1445
1494
  reader.skipType(tag & 7);
1446
1495
  break;
@@ -1497,6 +1546,20 @@ const dataform = $root.dataform = (() => {
1497
1546
  if (!$util.isString(message.labels[key[i]]))
1498
1547
  return "labels: string{k:string} expected";
1499
1548
  }
1549
+ if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
1550
+ if (!$util.isInteger(message.partitionExpirationDays))
1551
+ return "partitionExpirationDays: integer expected";
1552
+ if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
1553
+ if (typeof message.requirePartitionFilter !== "boolean")
1554
+ return "requirePartitionFilter: boolean expected";
1555
+ if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions")) {
1556
+ if (!$util.isObject(message.additionalOptions))
1557
+ return "additionalOptions: object expected";
1558
+ let key = Object.keys(message.additionalOptions);
1559
+ for (let i = 0; i < key.length; ++i)
1560
+ if (!$util.isString(message.additionalOptions[key[i]]))
1561
+ return "additionalOptions: string{k:string} expected";
1562
+ }
1500
1563
  return null;
1501
1564
  };
1502
1565
 
@@ -1530,6 +1593,17 @@ const dataform = $root.dataform = (() => {
1530
1593
  for (let keys = Object.keys(object.labels), i = 0; i < keys.length; ++i)
1531
1594
  message.labels[keys[i]] = String(object.labels[keys[i]]);
1532
1595
  }
1596
+ if (object.partitionExpirationDays != null)
1597
+ message.partitionExpirationDays = object.partitionExpirationDays | 0;
1598
+ if (object.requirePartitionFilter != null)
1599
+ message.requirePartitionFilter = Boolean(object.requirePartitionFilter);
1600
+ if (object.additionalOptions) {
1601
+ if (typeof object.additionalOptions !== "object")
1602
+ throw TypeError(".dataform.BigQueryOptions.additionalOptions: object expected");
1603
+ message.additionalOptions = {};
1604
+ for (let keys = Object.keys(object.additionalOptions), i = 0; i < keys.length; ++i)
1605
+ message.additionalOptions[keys[i]] = String(object.additionalOptions[keys[i]]);
1606
+ }
1533
1607
  return message;
1534
1608
  };
1535
1609
 
@@ -1548,11 +1622,15 @@ const dataform = $root.dataform = (() => {
1548
1622
  let object = {};
1549
1623
  if (options.arrays || options.defaults)
1550
1624
  object.clusterBy = [];
1551
- if (options.objects || options.defaults)
1625
+ if (options.objects || options.defaults) {
1552
1626
  object.labels = {};
1627
+ object.additionalOptions = {};
1628
+ }
1553
1629
  if (options.defaults) {
1554
1630
  object.partitionBy = "";
1555
1631
  object.updatePartitionFilter = "";
1632
+ object.partitionExpirationDays = 0;
1633
+ object.requirePartitionFilter = false;
1556
1634
  }
1557
1635
  if (message.partitionBy != null && message.hasOwnProperty("partitionBy"))
1558
1636
  object.partitionBy = message.partitionBy;
@@ -1569,6 +1647,15 @@ const dataform = $root.dataform = (() => {
1569
1647
  for (let j = 0; j < keys2.length; ++j)
1570
1648
  object.labels[keys2[j]] = message.labels[keys2[j]];
1571
1649
  }
1650
+ if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
1651
+ object.partitionExpirationDays = message.partitionExpirationDays;
1652
+ if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
1653
+ object.requirePartitionFilter = message.requirePartitionFilter;
1654
+ if (message.additionalOptions && (keys2 = Object.keys(message.additionalOptions)).length) {
1655
+ object.additionalOptions = {};
1656
+ for (let j = 0; j < keys2.length; ++j)
1657
+ object.additionalOptions[keys2[j]] = message.additionalOptions[keys2[j]];
1658
+ }
1572
1659
  return object;
1573
1660
  };
1574
1661
 
@@ -19858,13 +19945,26 @@ class BigQueryAdapter extends Adapter {
19858
19945
  return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
19859
19946
  }
19860
19947
  createOrReplace(table) {
19948
+ const options = [];
19949
+ if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) {
19950
+ options.push(`partition_expiration_days=${table.bigquery.partitionExpirationDays}`);
19951
+ }
19952
+ if (table.bigquery && table.bigquery.partitionBy && table.bigquery.requirePartitionFilter) {
19953
+ options.push(`require_partition_filter=${table.bigquery.requirePartitionFilter}`);
19954
+ }
19955
+ if (table.bigquery && table.bigquery.additionalOptions) {
19956
+ for (const [optionName, optionValue] of Object.entries(table.bigquery.additionalOptions)) {
19957
+ options.push(`${optionName}=${optionValue}`);
19958
+ }
19959
+ }
19861
19960
  return `create or replace ${table.materialized
19862
19961
  ? "materialized "
19863
19962
  : ""}${this.tableTypeAsSql(this.baseTableType(table.type))} ${this.resolveTarget(table.target)} ${table.bigquery && table.bigquery.partitionBy
19864
19963
  ? `partition by ${table.bigquery.partitionBy} `
19865
19964
  : ""}${table.bigquery && table.bigquery.clusterBy && table.bigquery.clusterBy.length > 0
19866
19965
  ? `cluster by ${table.bigquery.clusterBy.join(", ")} `
19867
- : ""}as ${table.query}`;
19966
+ : ""}${options.length > 0 ?
19967
+ `OPTIONS(${options.join(',')})` : ""}as ${table.query}`;
19868
19968
  }
19869
19969
  createOrReplaceView(target, query) {
19870
19970
  return `
@@ -20865,7 +20965,7 @@ const SortStyleType = ["compound", "interleaved"];
20865
20965
  const IRedshiftOptionsProperties = () => strictKeysOf()(["distKey", "distStyle", "sortKeys", "sortStyle"]);
20866
20966
  const ISnowflakeOptionsProperties = () => strictKeysOf()(["secure", "transient", "clusterBy"]);
20867
20967
  const ISQLDataWarehouseOptionsProperties = () => strictKeysOf()(["distribution"]);
20868
- const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels"]);
20968
+ const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels", "partitionExpirationDays", "requirePartitionFilter", "additionalOptions"]);
20869
20969
  const IPrestoOptionsProperties = () => strictKeysOf()(["partitionBy"]);
20870
20970
  const ITableAssertionsProperties = () => strictKeysOf()(["uniqueKey", "uniqueKeys", "nonNull", "rowConditions"]);
20871
20971
  const ITableConfigProperties = () => strictKeysOf()([
@@ -22277,7 +22377,7 @@ class RefReplacingContext {
22277
22377
  }
22278
22378
  }
22279
22379
 
22280
- const version = "1.21.1";
22380
+ const version = "1.22.0";
22281
22381
 
22282
22382
  const SQL_DATA_WAREHOUSE_DIST_HASH_REGEXP = new RegExp("HASH\\s*\\(\\s*\\w*\\s*\\)\\s*");
22283
22383
  const DEFAULT_CONFIG = {
@@ -22669,9 +22769,22 @@ class Session {
22669
22769
  }
22670
22770
  }
22671
22771
  if (!!table.bigquery) {
22672
- if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length)) &&
22772
+ if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length) || table.bigquery.partitionExpirationDays
22773
+ || table.bigquery.requirePartitionFilter) &&
22673
22774
  table.type === "view") {
22674
- this.compileError(`partitionBy/clusterBy are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
22775
+ this.compileError(`partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
22776
+ }
22777
+ else if ((!table.bigquery.partitionBy && (table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter)) &&
22778
+ table.type === "table") {
22779
+ this.compileError(`requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables`, table.fileName, table.target);
22780
+ }
22781
+ else if (table.bigquery.additionalOptions) {
22782
+ if (table.bigquery.partitionExpirationDays && table.bigquery.additionalOptions.partition_expiration_days) {
22783
+ this.compileError(`partitionExpirationDays has been declared twice`, table.fileName, table.target);
22784
+ }
22785
+ if (table.bigquery.requirePartitionFilter && table.bigquery.additionalOptions.require_partition_filter) {
22786
+ this.compileError(`requirePartitionFilter has been declared twice`, table.fileName, table.target);
22787
+ }
22675
22788
  }
22676
22789
  }
22677
22790
  if (!!Table.IGNORED_PROPS[table.type]) {
package/package.json CHANGED
@@ -43,7 +43,7 @@
43
43
  "publishConfig": {
44
44
  "access": "public"
45
45
  },
46
- "version": "1.21.1",
46
+ "version": "1.22.0",
47
47
  "name": "@dataform/core",
48
48
  "description": "Dataform core API.",
49
49
  "main": "bundle.js",