@dataform/core 1.20.3 → 1.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/bundle.d.ts +20 -2
  2. package/bundle.js +111 -24
  3. package/package.json +1 -1
package/bundle.d.ts CHANGED
@@ -1465,6 +1465,9 @@ namespace dataform {
1465
1465
  /** Table protected */
1466
1466
  "protected"?: (boolean|null);
1467
1467
 
1468
+ /** Table materialized */
1469
+ materialized?: (boolean|null);
1470
+
1468
1471
  /** Table actionDescriptor */
1469
1472
  actionDescriptor?: (dataform.IActionDescriptor|null);
1470
1473
 
@@ -1544,6 +1547,9 @@ namespace dataform {
1544
1547
  /** Table protected. */
1545
1548
  public protected: boolean;
1546
1549
 
1550
+ /** Table materialized. */
1551
+ public materialized: boolean;
1552
+
1547
1553
  /** Table actionDescriptor. */
1548
1554
  public actionDescriptor?: (dataform.IActionDescriptor|null);
1549
1555
 
@@ -7805,6 +7811,7 @@ interface ITableConfig extends IActionConfig, IDependenciesConfig, IDocumentable
7805
7811
  presto?: IPrestoOptions;
7806
7812
  assertions?: ITableAssertions;
7807
7813
  uniqueKey?: string[];
7814
+ materialized?: boolean;
7808
7815
  }
7809
7816
  interface ITableContext extends ICommonContext {
7810
7817
  when: (cond: boolean, trueCase: string, falseCase?: string) => string;
@@ -7831,6 +7838,7 @@ declare class Table {
7831
7838
  disabled(): this;
7832
7839
  protected(): this;
7833
7840
  uniqueKey(uniqueKey: string[]): void;
7841
+ materialized(materialized: boolean): void;
7834
7842
  snowflake(snowflake: dataform.ISnowflakeOptions): this;
7835
7843
  sqldatawarehouse(sqlDataWarehouse: dataform.ISQLDataWarehouseOptions): this;
7836
7844
  redshift(redshift: dataform.IRedshiftOptions): this;
@@ -7922,11 +7930,13 @@ declare type SqlxConfig = ((ITableConfig & {
7922
7930
  })) & {
7923
7931
  name: string;
7924
7932
  };
7933
+ declare type Action = Table | Operation | Assertion | Declaration;
7925
7934
  declare class Session {
7926
7935
  rootDir: string;
7927
7936
  config: dataform.IProjectConfig;
7928
7937
  canonicalConfig: dataform.IProjectConfig;
7929
- actions: Array<Table | Operation | Assertion | Declaration>;
7938
+ actions: Action[];
7939
+ indexedActions: ActionIndex;
7930
7940
  tests: {
7931
7941
  [name: string]: Test;
7932
7942
  };
@@ -7956,7 +7966,6 @@ declare class Session {
7956
7966
  compileError(err: Error | string, path?: string, actionTarget?: dataform.ITarget): void;
7957
7967
  compile(): dataform.CompiledGraph;
7958
7968
  compileToBase64(): string;
7959
- findActions(target: dataform.ITarget): (Declaration | Table | Assertion | Operation)[];
7960
7969
  private getDatabaseSuffixWithUnderscore;
7961
7970
  private getSchemaSuffixWithUnderscore;
7962
7971
  private getTablePrefixWithUnderscore;
@@ -7970,6 +7979,15 @@ declare class Session {
7970
7979
  private checkCircularity;
7971
7980
  private checkRunCachingCorrectness;
7972
7981
  }
7982
+ declare class ActionIndex {
7983
+ private readonly adapter;
7984
+ private readonly byName;
7985
+ private readonly bySchemaAndName;
7986
+ private readonly byDatabaseAndName;
7987
+ private readonly byDatabaseSchemaAndName;
7988
+ constructor(adapter: IAdapter, actions: Action[]);
7989
+ find(target: dataform.ITarget): Action[];
7990
+ }
7973
7991
 
7974
7992
  declare function main(encodedCoreExecutionRequest: string): string;
7975
7993
 
package/bundle.js CHANGED
@@ -3770,6 +3770,7 @@ const dataform = $root.dataform = (() => {
3770
3770
  * @property {string|null} [type] Table type
3771
3771
  * @property {string|null} [query] Table query
3772
3772
  * @property {boolean|null} ["protected"] Table protected
3773
+ * @property {boolean|null} [materialized] Table materialized
3773
3774
  * @property {dataform.IActionDescriptor|null} [actionDescriptor] Table actionDescriptor
3774
3775
  * @property {Array.<string>|null} [tags] Table tags
3775
3776
  * @property {string|null} [where] Table where
@@ -3873,6 +3874,14 @@ const dataform = $root.dataform = (() => {
3873
3874
  */
3874
3875
  Table.prototype["protected"] = false;
3875
3876
 
3877
+ /**
3878
+ * Table materialized.
3879
+ * @member {boolean} materialized
3880
+ * @memberof dataform.Table
3881
+ * @instance
3882
+ */
3883
+ Table.prototype.materialized = false;
3884
+
3876
3885
  /**
3877
3886
  * Table actionDescriptor.
3878
3887
  * @member {dataform.IActionDescriptor|null|undefined} actionDescriptor
@@ -4070,6 +4079,8 @@ const dataform = $root.dataform = (() => {
4070
4079
  $root.dataform.SnowflakeOptions.encode(message.snowflake, writer.uint32(/* id 33, wireType 2 =*/266).fork()).ldelim();
4071
4080
  if (message.presto != null && message.hasOwnProperty("presto"))
4072
4081
  $root.dataform.PrestoOptions.encode(message.presto, writer.uint32(/* id 34, wireType 2 =*/274).fork()).ldelim();
4082
+ if (message.materialized != null && message.hasOwnProperty("materialized"))
4083
+ writer.uint32(/* id 35, wireType 0 =*/280).bool(message.materialized);
4073
4084
  return writer;
4074
4085
  };
4075
4086
 
@@ -4130,6 +4141,9 @@ const dataform = $root.dataform = (() => {
4130
4141
  case 9:
4131
4142
  message["protected"] = reader.bool();
4132
4143
  break;
4144
+ case 35:
4145
+ message.materialized = reader.bool();
4146
+ break;
4133
4147
  case 24:
4134
4148
  message.actionDescriptor = $root.dataform.ActionDescriptor.decode(reader, reader.uint32());
4135
4149
  break;
@@ -4262,6 +4276,9 @@ const dataform = $root.dataform = (() => {
4262
4276
  if (message["protected"] != null && message.hasOwnProperty("protected"))
4263
4277
  if (typeof message["protected"] !== "boolean")
4264
4278
  return "protected: boolean expected";
4279
+ if (message.materialized != null && message.hasOwnProperty("materialized"))
4280
+ if (typeof message.materialized !== "boolean")
4281
+ return "materialized: boolean expected";
4265
4282
  if (message.actionDescriptor != null && message.hasOwnProperty("actionDescriptor")) {
4266
4283
  let error = $root.dataform.ActionDescriptor.verify(message.actionDescriptor);
4267
4284
  if (error)
@@ -4400,6 +4417,8 @@ const dataform = $root.dataform = (() => {
4400
4417
  message.query = String(object.query);
4401
4418
  if (object["protected"] != null)
4402
4419
  message["protected"] = Boolean(object["protected"]);
4420
+ if (object.materialized != null)
4421
+ message.materialized = Boolean(object.materialized);
4403
4422
  if (object.actionDescriptor != null) {
4404
4423
  if (typeof object.actionDescriptor !== "object")
4405
4424
  throw TypeError(".dataform.Table.actionDescriptor: object expected");
@@ -4520,6 +4539,7 @@ const dataform = $root.dataform = (() => {
4520
4539
  object.canonicalTarget = null;
4521
4540
  object.snowflake = null;
4522
4541
  object.presto = null;
4542
+ object.materialized = false;
4523
4543
  }
4524
4544
  if (message.type != null && message.hasOwnProperty("type"))
4525
4545
  object.type = message.type;
@@ -4588,6 +4608,8 @@ const dataform = $root.dataform = (() => {
4588
4608
  object.snowflake = $root.dataform.SnowflakeOptions.toObject(message.snowflake, options);
4589
4609
  if (message.presto != null && message.hasOwnProperty("presto"))
4590
4610
  object.presto = $root.dataform.PrestoOptions.toObject(message.presto, options);
4611
+ if (message.materialized != null && message.hasOwnProperty("materialized"))
4612
+ object.materialized = message.materialized;
4591
4613
  return object;
4592
4614
  };
4593
4615
 
@@ -19836,7 +19858,9 @@ class BigQueryAdapter extends Adapter {
19836
19858
  return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
19837
19859
  }
19838
19860
  createOrReplace(table) {
19839
- return `create or replace ${this.tableTypeAsSql(this.baseTableType(table.type))} ${this.resolveTarget(table.target)} ${table.bigquery && table.bigquery.partitionBy
19861
+ return `create or replace ${table.materialized
19862
+ ? "materialized "
19863
+ : ""}${this.tableTypeAsSql(this.baseTableType(table.type))} ${this.resolveTarget(table.target)} ${table.bigquery && table.bigquery.partitionBy
19840
19864
  ? `partition by ${table.bigquery.partitionBy} `
19841
19865
  : ""}${table.bigquery && table.bigquery.clusterBy && table.bigquery.clusterBy.length > 0
19842
19866
  ? `cluster by ${table.bigquery.clusterBy.join(", ")} `
@@ -20053,17 +20077,17 @@ class SnowflakeAdapter extends Adapter {
20053
20077
  assertTasks(assertion, projectConfig) {
20054
20078
  const tasks = Tasks.create();
20055
20079
  const target = assertion.target;
20056
- tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query, false)));
20080
+ tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query, false, false)));
20057
20081
  tasks.add(Task.assertion(`select sum(1) as row_count from ${this.resolveTarget(target)}`));
20058
20082
  return tasks;
20059
20083
  }
20060
- createOrReplaceView(target, query, secure) {
20061
- return `create or replace ${secure ? "secure " : ""}view ${this.resolveTarget(target)} as ${query}`;
20084
+ createOrReplaceView(target, query, secure, materialized) {
20085
+ return `create or replace ${secure ? "secure " : ""}${materialized ? "materialized " : ""}view ${this.resolveTarget(target)} as ${query}`;
20062
20086
  }
20063
20087
  createOrReplace(table) {
20064
20088
  var _a, _b, _c, _d, _e;
20065
20089
  if (table.type === "view") {
20066
- return this.createOrReplaceView(table.target, table.query, (_a = table.snowflake) === null || _a === void 0 ? void 0 : _a.secure);
20090
+ return this.createOrReplaceView(table.target, table.query, (_a = table.snowflake) === null || _a === void 0 ? void 0 : _a.secure, table.materialized);
20067
20091
  }
20068
20092
  return `create or replace ${((_b = table.snowflake) === null || _b === void 0 ? void 0 : _b.transient) ? "transient " : ""}table ${this.resolveTarget(table.target)} ${((_d = (_c = table.snowflake) === null || _c === void 0 ? void 0 : _c.clusterBy) === null || _d === void 0 ? void 0 : _d.length) > 0
20069
20093
  ? `cluster by (${(_e = table.snowflake) === null || _e === void 0 ? void 0 : _e.clusterBy.join(", ")}) `
@@ -20862,7 +20886,8 @@ const ITableConfigProperties = () => strictKeysOf()([
20862
20886
  "assertions",
20863
20887
  "database",
20864
20888
  "columns",
20865
- "description"
20889
+ "description",
20890
+ "materialized"
20866
20891
  ]);
20867
20892
  class Table {
20868
20893
  constructor() {
@@ -20928,6 +20953,9 @@ class Table {
20928
20953
  if (config.uniqueKey) {
20929
20954
  this.uniqueKey(config.uniqueKey);
20930
20955
  }
20956
+ if (config.materialized) {
20957
+ this.materialized(config.materialized);
20958
+ }
20931
20959
  return this;
20932
20960
  }
20933
20961
  type(type) {
@@ -20964,6 +20992,9 @@ class Table {
20964
20992
  uniqueKey(uniqueKey) {
20965
20993
  this.proto.uniqueKey = uniqueKey;
20966
20994
  }
20995
+ materialized(materialized) {
20996
+ this.proto.materialized = materialized;
20997
+ }
20967
20998
  snowflake(snowflake) {
20968
20999
  checkExcessProperties((e) => this.session.compileError(e), snowflake, ISnowflakeOptionsProperties(), "snowflake config");
20969
21000
  this.proto.snowflake = dataform.SnowflakeOptions.create(snowflake);
@@ -22146,7 +22177,7 @@ class Test {
22146
22177
  this.session.compileError(new Error("Tests must operate upon a specified dataset."), this.proto.fileName);
22147
22178
  }
22148
22179
  else {
22149
- const allResolved = this.session.findActions(resolvableAsTarget(this.datasetToTest));
22180
+ const allResolved = this.session.indexedActions.find(resolvableAsTarget(this.datasetToTest));
22150
22181
  if (allResolved.length > 1) {
22151
22182
  this.session.compileError(new Error(ambiguousActionNameMsg(this.datasetToTest, allResolved)), this.proto.fileName);
22152
22183
  }
@@ -22246,7 +22277,7 @@ class RefReplacingContext {
22246
22277
  }
22247
22278
  }
22248
22279
 
22249
- const version = "1.20.3";
22280
+ const version = "1.21.0";
22250
22281
 
22251
22282
  const SQL_DATA_WAREHOUSE_DIST_HASH_REGEXP = new RegExp("HASH\\s*\\(\\s*\\w*\\s*\\)\\s*");
22252
22283
  const DEFAULT_CONFIG = {
@@ -22356,7 +22387,7 @@ class Session {
22356
22387
  }
22357
22388
  resolve(ref, ...rest) {
22358
22389
  ref = toResolvable(ref, rest);
22359
- const allResolved = this.findActions(resolvableAsTarget(ref));
22390
+ const allResolved = this.indexedActions.find(resolvableAsTarget(ref));
22360
22391
  if (allResolved.length > 1) {
22361
22392
  this.compileError(new Error(ambiguousActionNameMsg(ref, allResolved)));
22362
22393
  }
@@ -22452,6 +22483,7 @@ class Session {
22452
22483
  this.graphErrors.compilationErrors.push(compileError);
22453
22484
  }
22454
22485
  compile() {
22486
+ this.indexedActions = new ActionIndex(this.adapter(), this.actions);
22455
22487
  const compiledGraph = dataform.CompiledGraph.create({
22456
22488
  projectConfig: this.config,
22457
22489
  tables: this.compileGraphChunk(this.actions.filter(action => action instanceof Table), dataform.Table.verify),
@@ -22480,20 +22512,6 @@ class Session {
22480
22512
  compileToBase64() {
22481
22513
  return encode64(dataform.CompiledGraph, this.compile());
22482
22514
  }
22483
- findActions(target) {
22484
- const adapter = this.adapter();
22485
- return this.actions.filter(action => {
22486
- if (!!target.database &&
22487
- action.proto.target.database !== adapter.normalizeIdentifier(target.database)) {
22488
- return false;
22489
- }
22490
- if (!!target.schema &&
22491
- action.proto.target.schema !== adapter.normalizeIdentifier(target.schema)) {
22492
- return false;
22493
- }
22494
- return action.proto.target.name === adapter.normalizeIdentifier(target.name);
22495
- });
22496
- }
22497
22515
  getDatabaseSuffixWithUnderscore() {
22498
22516
  return !!this.config.databaseSuffix ? `_${this.config.databaseSuffix}` : "";
22499
22517
  }
@@ -22521,7 +22539,7 @@ class Session {
22521
22539
  actions.forEach(action => {
22522
22540
  const fullyQualifiedDependencies = {};
22523
22541
  for (const dependency of action.dependencyTargets) {
22524
- const possibleDeps = this.findActions(dependency);
22542
+ const possibleDeps = this.indexedActions.find(dependency);
22525
22543
  if (possibleDeps.length === 0) {
22526
22544
  this.compileError(new Error(`Missing dependency detected: Action "${targetAsReadableString(action.target)}" depends on "${stringifyResolvable(dependency)}" which does not exist`), action.fileName, action.target);
22527
22545
  }
@@ -22591,6 +22609,11 @@ class Session {
22591
22609
  if (!!table.type && !TableType.includes(table.type)) {
22592
22610
  this.compileError(`Wrong type of table detected. Should only use predefined types: ${joinQuoted(TableType)}`, table.fileName, table.target);
22593
22611
  }
22612
+ if (!!table.materialized) {
22613
+ if (table.type !== "view" || (this.config.warehouse !== "snowflake" && this.config.warehouse !== "bigquery")) {
22614
+ this.compileError(new Error(`The 'materialized' option is only valid for Snowflake and BigQuery views`), table.fileName, table.target);
22615
+ }
22616
+ }
22594
22617
  if (!!table.snowflake) {
22595
22618
  if (table.snowflake.secure && table.type !== "view") {
22596
22619
  this.compileError(new Error(`The 'secure' option is only valid for Snowflake views`), table.fileName, table.target);
@@ -22720,6 +22743,70 @@ function objectExistsOrIsNonEmpty(prop) {
22720
22743
  (!Array.isArray(prop) && typeof prop === "object" && !!Object.keys(prop).length) ||
22721
22744
  typeof prop !== "object");
22722
22745
  }
22746
+ class ActionIndex {
22747
+ constructor(adapter, actions) {
22748
+ this.adapter = adapter;
22749
+ this.byName = new Map();
22750
+ this.bySchemaAndName = new Map();
22751
+ this.byDatabaseAndName = new Map();
22752
+ this.byDatabaseSchemaAndName = new Map();
22753
+ for (const action of actions) {
22754
+ if (!this.byName.has(action.proto.target.name)) {
22755
+ this.byName.set(action.proto.target.name, []);
22756
+ }
22757
+ this.byName.get(action.proto.target.name).push(action);
22758
+ if (!this.bySchemaAndName.has(action.proto.target.schema)) {
22759
+ this.bySchemaAndName.set(action.proto.target.schema, new Map());
22760
+ }
22761
+ const forSchema = this.bySchemaAndName.get(action.proto.target.schema);
22762
+ if (!forSchema.has(action.proto.target.name)) {
22763
+ forSchema.set(action.proto.target.name, []);
22764
+ }
22765
+ forSchema.get(action.proto.target.name).push(action);
22766
+ if (!!action.proto.target.database) {
22767
+ if (!this.byDatabaseAndName.has(action.proto.target.database)) {
22768
+ this.byDatabaseAndName.set(action.proto.target.database, new Map());
22769
+ }
22770
+ const forDatabaseNoSchema = this.byDatabaseAndName.get(action.proto.target.database);
22771
+ if (!forDatabaseNoSchema.has(action.proto.target.name)) {
22772
+ forDatabaseNoSchema.set(action.proto.target.name, []);
22773
+ }
22774
+ forDatabaseNoSchema.get(action.proto.target.name).push(action);
22775
+ if (!this.byDatabaseSchemaAndName.has(action.proto.target.database)) {
22776
+ this.byDatabaseSchemaAndName.set(action.proto.target.database, new Map());
22777
+ }
22778
+ const forDatabase = this.byDatabaseSchemaAndName.get(action.proto.target.database);
22779
+ if (!forDatabase.has(action.proto.target.schema)) {
22780
+ forDatabase.set(action.proto.target.schema, new Map());
22781
+ }
22782
+ const forDatabaseAndSchema = forDatabase.get(action.proto.target.schema);
22783
+ if (!forDatabaseAndSchema.has(action.proto.target.name)) {
22784
+ forDatabaseAndSchema.set(action.proto.target.name, []);
22785
+ }
22786
+ forDatabaseAndSchema.get(action.proto.target.name).push(action);
22787
+ }
22788
+ }
22789
+ }
22790
+ find(target) {
22791
+ if (!!target.database) {
22792
+ if (!!target.schema) {
22793
+ return (this.byDatabaseSchemaAndName
22794
+ .get(this.adapter.normalizeIdentifier(target.database))
22795
+ .get(this.adapter.normalizeIdentifier(target.schema))
22796
+ .get(this.adapter.normalizeIdentifier(target.name)) || []);
22797
+ }
22798
+ return (this.byDatabaseAndName
22799
+ .get(this.adapter.normalizeIdentifier(target.database))
22800
+ .get(this.adapter.normalizeIdentifier(target.name)) || []);
22801
+ }
22802
+ if (!!target.schema) {
22803
+ return (this.bySchemaAndName
22804
+ .get(this.adapter.normalizeIdentifier(target.schema))
22805
+ .get(this.adapter.normalizeIdentifier(target.name)) || []);
22806
+ }
22807
+ return this.byName.get(this.adapter.normalizeIdentifier(target.name)) || [];
22808
+ }
22809
+ }
22723
22810
 
22724
22811
  function main(encodedCoreExecutionRequest) {
22725
22812
  var _a;
package/package.json CHANGED
@@ -43,7 +43,7 @@
43
43
  "publishConfig": {
44
44
  "access": "public"
45
45
  },
46
- "version": "1.20.3",
46
+ "version": "1.21.0",
47
47
  "name": "@dataform/core",
48
48
  "description": "Dataform core API.",
49
49
  "main": "bundle.js",