@dataform/core 1.21.0 → 1.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/bundle.d.ts +29 -0
  2. package/bundle.js +141 -6
  3. package/package.json +1 -1
package/bundle.d.ts CHANGED
@@ -15,6 +15,9 @@ namespace dataform {
15
15
  /** ProjectConfig defaultSchema */
16
16
  defaultSchema?: (string|null);
17
17
 
18
+ /** ProjectConfig defaultLocation */
19
+ defaultLocation?: (string|null);
20
+
18
21
  /** ProjectConfig assertionSchema */
19
22
  assertionSchema?: (string|null);
20
23
 
@@ -61,6 +64,9 @@ namespace dataform {
61
64
  /** ProjectConfig defaultSchema. */
62
65
  public defaultSchema: string;
63
66
 
67
+ /** ProjectConfig defaultLocation. */
68
+ public defaultLocation: string;
69
+
64
70
  /** ProjectConfig assertionSchema. */
65
71
  public assertionSchema: string;
66
72
 
@@ -491,6 +497,15 @@ namespace dataform {
491
497
 
492
498
  /** BigQueryOptions labels */
493
499
  labels?: ({ [k: string]: string }|null);
500
+
501
+ /** BigQueryOptions partitionExpirationDays */
502
+ partitionExpirationDays?: (number|null);
503
+
504
+ /** BigQueryOptions requirePartitionFilter */
505
+ requirePartitionFilter?: (boolean|null);
506
+
507
+ /** BigQueryOptions additionalOptions */
508
+ additionalOptions?: ({ [k: string]: string }|null);
494
509
  }
495
510
 
496
511
  /** Represents a BigQueryOptions. */
@@ -514,6 +529,15 @@ namespace dataform {
514
529
  /** BigQueryOptions labels. */
515
530
  public labels: { [k: string]: string };
516
531
 
532
+ /** BigQueryOptions partitionExpirationDays. */
533
+ public partitionExpirationDays: number;
534
+
535
+ /** BigQueryOptions requirePartitionFilter. */
536
+ public requirePartitionFilter: boolean;
537
+
538
+ /** BigQueryOptions additionalOptions. */
539
+ public additionalOptions: { [k: string]: string };
540
+
517
541
  /**
518
542
  * Creates a new BigQueryOptions instance using the specified properties.
519
543
  * @param [properties] Properties to set
@@ -7791,6 +7815,11 @@ interface IBigQueryOptions {
7791
7815
  labels?: {
7792
7816
  [name: string]: string;
7793
7817
  };
7818
+ partitionExpirationDays?: number;
7819
+ requirePartitionFilter?: boolean;
7820
+ additionalOptions?: {
7821
+ [name: string]: string;
7822
+ };
7794
7823
  }
7795
7824
  interface IPrestoOptions {
7796
7825
  partitionBy?: string[];
package/bundle.js CHANGED
@@ -50,6 +50,7 @@ const dataform = $root.dataform = (() => {
50
50
  * @property {string|null} [warehouse] ProjectConfig warehouse
51
51
  * @property {string|null} [defaultDatabase] ProjectConfig defaultDatabase
52
52
  * @property {string|null} [defaultSchema] ProjectConfig defaultSchema
53
+ * @property {string|null} [defaultLocation] ProjectConfig defaultLocation
53
54
  * @property {string|null} [assertionSchema] ProjectConfig assertionSchema
54
55
  * @property {Object.<string,string>|null} [vars] ProjectConfig vars
55
56
  * @property {number|null} [concurrentQueryLimit] ProjectConfig concurrentQueryLimit
@@ -101,6 +102,14 @@ const dataform = $root.dataform = (() => {
101
102
  */
102
103
  ProjectConfig.prototype.defaultSchema = "";
103
104
 
105
+ /**
106
+ * ProjectConfig defaultLocation.
107
+ * @member {string} defaultLocation
108
+ * @memberof dataform.ProjectConfig
109
+ * @instance
110
+ */
111
+ ProjectConfig.prototype.defaultLocation = "";
112
+
104
113
  /**
105
114
  * ProjectConfig assertionSchema.
106
115
  * @member {string} assertionSchema
@@ -222,6 +231,8 @@ const dataform = $root.dataform = (() => {
222
231
  writer.uint32(/* id 14, wireType 2 =*/114).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.vars[keys[i]]).ldelim();
223
232
  if (message.databaseSuffix != null && message.hasOwnProperty("databaseSuffix"))
224
233
  writer.uint32(/* id 15, wireType 2 =*/122).string(message.databaseSuffix);
234
+ if (message.defaultLocation != null && message.hasOwnProperty("defaultLocation"))
235
+ writer.uint32(/* id 16, wireType 2 =*/130).string(message.defaultLocation);
225
236
  return writer;
226
237
  };
227
238
 
@@ -265,6 +276,9 @@ const dataform = $root.dataform = (() => {
265
276
  case 2:
266
277
  message.defaultSchema = reader.string();
267
278
  break;
279
+ case 16:
280
+ message.defaultLocation = reader.string();
281
+ break;
268
282
  case 5:
269
283
  message.assertionSchema = reader.string();
270
284
  break;
@@ -341,6 +355,9 @@ const dataform = $root.dataform = (() => {
341
355
  if (message.defaultSchema != null && message.hasOwnProperty("defaultSchema"))
342
356
  if (!$util.isString(message.defaultSchema))
343
357
  return "defaultSchema: string expected";
358
+ if (message.defaultLocation != null && message.hasOwnProperty("defaultLocation"))
359
+ if (!$util.isString(message.defaultLocation))
360
+ return "defaultLocation: string expected";
344
361
  if (message.assertionSchema != null && message.hasOwnProperty("assertionSchema"))
345
362
  if (!$util.isString(message.assertionSchema))
346
363
  return "assertionSchema: string expected";
@@ -394,6 +411,8 @@ const dataform = $root.dataform = (() => {
394
411
  message.defaultDatabase = String(object.defaultDatabase);
395
412
  if (object.defaultSchema != null)
396
413
  message.defaultSchema = String(object.defaultSchema);
414
+ if (object.defaultLocation != null)
415
+ message.defaultLocation = String(object.defaultLocation);
397
416
  if (object.assertionSchema != null)
398
417
  message.assertionSchema = String(object.assertionSchema);
399
418
  if (object.vars) {
@@ -447,6 +466,7 @@ const dataform = $root.dataform = (() => {
447
466
  object.tablePrefix = "";
448
467
  object.concurrentQueryLimit = 0;
449
468
  object.databaseSuffix = "";
469
+ object.defaultLocation = "";
450
470
  }
451
471
  if (message.warehouse != null && message.hasOwnProperty("warehouse"))
452
472
  object.warehouse = message.warehouse;
@@ -476,6 +496,8 @@ const dataform = $root.dataform = (() => {
476
496
  }
477
497
  if (message.databaseSuffix != null && message.hasOwnProperty("databaseSuffix"))
478
498
  object.databaseSuffix = message.databaseSuffix;
499
+ if (message.defaultLocation != null && message.hasOwnProperty("defaultLocation"))
500
+ object.defaultLocation = message.defaultLocation;
479
501
  return object;
480
502
  };
481
503
 
@@ -1303,6 +1325,9 @@ const dataform = $root.dataform = (() => {
1303
1325
  * @property {Array.<string>|null} [clusterBy] BigQueryOptions clusterBy
1304
1326
  * @property {string|null} [updatePartitionFilter] BigQueryOptions updatePartitionFilter
1305
1327
  * @property {Object.<string,string>|null} [labels] BigQueryOptions labels
1328
+ * @property {number|null} [partitionExpirationDays] BigQueryOptions partitionExpirationDays
1329
+ * @property {boolean|null} [requirePartitionFilter] BigQueryOptions requirePartitionFilter
1330
+ * @property {Object.<string,string>|null} [additionalOptions] BigQueryOptions additionalOptions
1306
1331
  */
1307
1332
 
1308
1333
  /**
@@ -1316,6 +1341,7 @@ const dataform = $root.dataform = (() => {
1316
1341
  function BigQueryOptions(properties) {
1317
1342
  this.clusterBy = [];
1318
1343
  this.labels = {};
1344
+ this.additionalOptions = {};
1319
1345
  if (properties)
1320
1346
  for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
1321
1347
  if (properties[keys[i]] != null)
@@ -1354,6 +1380,30 @@ const dataform = $root.dataform = (() => {
1354
1380
  */
1355
1381
  BigQueryOptions.prototype.labels = $util.emptyObject;
1356
1382
 
1383
+ /**
1384
+ * BigQueryOptions partitionExpirationDays.
1385
+ * @member {number} partitionExpirationDays
1386
+ * @memberof dataform.BigQueryOptions
1387
+ * @instance
1388
+ */
1389
+ BigQueryOptions.prototype.partitionExpirationDays = 0;
1390
+
1391
+ /**
1392
+ * BigQueryOptions requirePartitionFilter.
1393
+ * @member {boolean} requirePartitionFilter
1394
+ * @memberof dataform.BigQueryOptions
1395
+ * @instance
1396
+ */
1397
+ BigQueryOptions.prototype.requirePartitionFilter = false;
1398
+
1399
+ /**
1400
+ * BigQueryOptions additionalOptions.
1401
+ * @member {Object.<string,string>} additionalOptions
1402
+ * @memberof dataform.BigQueryOptions
1403
+ * @instance
1404
+ */
1405
+ BigQueryOptions.prototype.additionalOptions = $util.emptyObject;
1406
+
1357
1407
  /**
1358
1408
  * Creates a new BigQueryOptions instance using the specified properties.
1359
1409
  * @function create
@@ -1388,6 +1438,13 @@ const dataform = $root.dataform = (() => {
1388
1438
  if (message.labels != null && message.hasOwnProperty("labels"))
1389
1439
  for (let keys = Object.keys(message.labels), i = 0; i < keys.length; ++i)
1390
1440
  writer.uint32(/* id 4, wireType 2 =*/34).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.labels[keys[i]]).ldelim();
1441
+ if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
1442
+ writer.uint32(/* id 5, wireType 0 =*/40).int32(message.partitionExpirationDays);
1443
+ if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
1444
+ writer.uint32(/* id 6, wireType 0 =*/48).bool(message.requirePartitionFilter);
1445
+ if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions"))
1446
+ for (let keys = Object.keys(message.additionalOptions), i = 0; i < keys.length; ++i)
1447
+ writer.uint32(/* id 7, wireType 2 =*/58).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.additionalOptions[keys[i]]).ldelim();
1391
1448
  return writer;
1392
1449
  };
1393
1450
 
@@ -1441,6 +1498,20 @@ const dataform = $root.dataform = (() => {
1441
1498
  reader.pos++;
1442
1499
  message.labels[key] = reader.string();
1443
1500
  break;
1501
+ case 5:
1502
+ message.partitionExpirationDays = reader.int32();
1503
+ break;
1504
+ case 6:
1505
+ message.requirePartitionFilter = reader.bool();
1506
+ break;
1507
+ case 7:
1508
+ reader.skip().pos++;
1509
+ if (message.additionalOptions === $util.emptyObject)
1510
+ message.additionalOptions = {};
1511
+ key = reader.string();
1512
+ reader.pos++;
1513
+ message.additionalOptions[key] = reader.string();
1514
+ break;
1444
1515
  default:
1445
1516
  reader.skipType(tag & 7);
1446
1517
  break;
@@ -1497,6 +1568,20 @@ const dataform = $root.dataform = (() => {
1497
1568
  if (!$util.isString(message.labels[key[i]]))
1498
1569
  return "labels: string{k:string} expected";
1499
1570
  }
1571
+ if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
1572
+ if (!$util.isInteger(message.partitionExpirationDays))
1573
+ return "partitionExpirationDays: integer expected";
1574
+ if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
1575
+ if (typeof message.requirePartitionFilter !== "boolean")
1576
+ return "requirePartitionFilter: boolean expected";
1577
+ if (message.additionalOptions != null && message.hasOwnProperty("additionalOptions")) {
1578
+ if (!$util.isObject(message.additionalOptions))
1579
+ return "additionalOptions: object expected";
1580
+ let key = Object.keys(message.additionalOptions);
1581
+ for (let i = 0; i < key.length; ++i)
1582
+ if (!$util.isString(message.additionalOptions[key[i]]))
1583
+ return "additionalOptions: string{k:string} expected";
1584
+ }
1500
1585
  return null;
1501
1586
  };
1502
1587
 
@@ -1530,6 +1615,17 @@ const dataform = $root.dataform = (() => {
1530
1615
  for (let keys = Object.keys(object.labels), i = 0; i < keys.length; ++i)
1531
1616
  message.labels[keys[i]] = String(object.labels[keys[i]]);
1532
1617
  }
1618
+ if (object.partitionExpirationDays != null)
1619
+ message.partitionExpirationDays = object.partitionExpirationDays | 0;
1620
+ if (object.requirePartitionFilter != null)
1621
+ message.requirePartitionFilter = Boolean(object.requirePartitionFilter);
1622
+ if (object.additionalOptions) {
1623
+ if (typeof object.additionalOptions !== "object")
1624
+ throw TypeError(".dataform.BigQueryOptions.additionalOptions: object expected");
1625
+ message.additionalOptions = {};
1626
+ for (let keys = Object.keys(object.additionalOptions), i = 0; i < keys.length; ++i)
1627
+ message.additionalOptions[keys[i]] = String(object.additionalOptions[keys[i]]);
1628
+ }
1533
1629
  return message;
1534
1630
  };
1535
1631
 
@@ -1548,11 +1644,15 @@ const dataform = $root.dataform = (() => {
1548
1644
  let object = {};
1549
1645
  if (options.arrays || options.defaults)
1550
1646
  object.clusterBy = [];
1551
- if (options.objects || options.defaults)
1647
+ if (options.objects || options.defaults) {
1552
1648
  object.labels = {};
1649
+ object.additionalOptions = {};
1650
+ }
1553
1651
  if (options.defaults) {
1554
1652
  object.partitionBy = "";
1555
1653
  object.updatePartitionFilter = "";
1654
+ object.partitionExpirationDays = 0;
1655
+ object.requirePartitionFilter = false;
1556
1656
  }
1557
1657
  if (message.partitionBy != null && message.hasOwnProperty("partitionBy"))
1558
1658
  object.partitionBy = message.partitionBy;
@@ -1569,6 +1669,15 @@ const dataform = $root.dataform = (() => {
1569
1669
  for (let j = 0; j < keys2.length; ++j)
1570
1670
  object.labels[keys2[j]] = message.labels[keys2[j]];
1571
1671
  }
1672
+ if (message.partitionExpirationDays != null && message.hasOwnProperty("partitionExpirationDays"))
1673
+ object.partitionExpirationDays = message.partitionExpirationDays;
1674
+ if (message.requirePartitionFilter != null && message.hasOwnProperty("requirePartitionFilter"))
1675
+ object.requirePartitionFilter = message.requirePartitionFilter;
1676
+ if (message.additionalOptions && (keys2 = Object.keys(message.additionalOptions)).length) {
1677
+ object.additionalOptions = {};
1678
+ for (let j = 0; j < keys2.length; ++j)
1679
+ object.additionalOptions[keys2[j]] = message.additionalOptions[keys2[j]];
1680
+ }
1572
1681
  return object;
1573
1682
  };
1574
1683
 
@@ -19858,13 +19967,26 @@ class BigQueryAdapter extends Adapter {
19858
19967
  return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
19859
19968
  }
19860
19969
  createOrReplace(table) {
19970
+ const options = [];
19971
+ if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) {
19972
+ options.push(`partition_expiration_days=${table.bigquery.partitionExpirationDays}`);
19973
+ }
19974
+ if (table.bigquery && table.bigquery.partitionBy && table.bigquery.requirePartitionFilter) {
19975
+ options.push(`require_partition_filter=${table.bigquery.requirePartitionFilter}`);
19976
+ }
19977
+ if (table.bigquery && table.bigquery.additionalOptions) {
19978
+ for (const [optionName, optionValue] of Object.entries(table.bigquery.additionalOptions)) {
19979
+ options.push(`${optionName}=${optionValue}`);
19980
+ }
19981
+ }
19861
19982
  return `create or replace ${table.materialized
19862
19983
  ? "materialized "
19863
19984
  : ""}${this.tableTypeAsSql(this.baseTableType(table.type))} ${this.resolveTarget(table.target)} ${table.bigquery && table.bigquery.partitionBy
19864
19985
  ? `partition by ${table.bigquery.partitionBy} `
19865
19986
  : ""}${table.bigquery && table.bigquery.clusterBy && table.bigquery.clusterBy.length > 0
19866
19987
  ? `cluster by ${table.bigquery.clusterBy.join(", ")} `
19867
- : ""}as ${table.query}`;
19988
+ : ""}${options.length > 0 ?
19989
+ `OPTIONS(${options.join(',')})` : ""}as ${table.query}`;
19868
19990
  }
19869
19991
  createOrReplaceView(target, query) {
19870
19992
  return `
@@ -20865,7 +20987,7 @@ const SortStyleType = ["compound", "interleaved"];
20865
20987
  const IRedshiftOptionsProperties = () => strictKeysOf()(["distKey", "distStyle", "sortKeys", "sortStyle"]);
20866
20988
  const ISnowflakeOptionsProperties = () => strictKeysOf()(["secure", "transient", "clusterBy"]);
20867
20989
  const ISQLDataWarehouseOptionsProperties = () => strictKeysOf()(["distribution"]);
20868
- const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels"]);
20990
+ const IBigQueryOptionsProperties = () => strictKeysOf()(["partitionBy", "clusterBy", "updatePartitionFilter", "labels", "partitionExpirationDays", "requirePartitionFilter", "additionalOptions"]);
20869
20991
  const IPrestoOptionsProperties = () => strictKeysOf()(["partitionBy"]);
20870
20992
  const ITableAssertionsProperties = () => strictKeysOf()(["uniqueKey", "uniqueKeys", "nonNull", "rowConditions"]);
20871
20993
  const ITableConfigProperties = () => strictKeysOf()([
@@ -22277,7 +22399,7 @@ class RefReplacingContext {
22277
22399
  }
22278
22400
  }
22279
22401
 
22280
- const version = "1.21.0";
22402
+ const version = "1.22.1";
22281
22403
 
22282
22404
  const SQL_DATA_WAREHOUSE_DIST_HASH_REGEXP = new RegExp("HASH\\s*\\(\\s*\\w*\\s*\\)\\s*");
22283
22405
  const DEFAULT_CONFIG = {
@@ -22669,9 +22791,22 @@ class Session {
22669
22791
  }
22670
22792
  }
22671
22793
  if (!!table.bigquery) {
22672
- if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length)) &&
22794
+ if ((table.bigquery.partitionBy || ((_b = table.bigquery.clusterBy) === null || _b === void 0 ? void 0 : _b.length) || table.bigquery.partitionExpirationDays
22795
+ || table.bigquery.requirePartitionFilter) &&
22673
22796
  table.type === "view") {
22674
- this.compileError(`partitionBy/clusterBy are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
22797
+ this.compileError(`partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables`, table.fileName, table.target);
22798
+ }
22799
+ else if ((!table.bigquery.partitionBy && (table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter)) &&
22800
+ table.type === "table") {
22801
+ this.compileError(`requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables`, table.fileName, table.target);
22802
+ }
22803
+ else if (table.bigquery.additionalOptions) {
22804
+ if (table.bigquery.partitionExpirationDays && table.bigquery.additionalOptions.partition_expiration_days) {
22805
+ this.compileError(`partitionExpirationDays has been declared twice`, table.fileName, table.target);
22806
+ }
22807
+ if (table.bigquery.requirePartitionFilter && table.bigquery.additionalOptions.require_partition_filter) {
22808
+ this.compileError(`requirePartitionFilter has been declared twice`, table.fileName, table.target);
22809
+ }
22675
22810
  }
22676
22811
  }
22677
22812
  if (!!Table.IGNORED_PROPS[table.type]) {
package/package.json CHANGED
@@ -43,7 +43,7 @@
43
43
  "publishConfig": {
44
44
  "access": "public"
45
45
  },
46
- "version": "1.21.0",
46
+ "version": "1.22.1",
47
47
  "name": "@dataform/core",
48
48
  "description": "Dataform core API.",
49
49
  "main": "bundle.js",