@sqlanvil/core 0.0.1 → 1.0.1

package/core.proto

@@ -0,0 +1,467 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package sqlanvil;
+
+import "configs.proto";
+import "extension.proto";
+import "google/protobuf/struct.proto";
+
+option java_package = "com.sqlanvil.protos";
+option java_outer_classname = "CoreMeta";
+option java_multiple_files = true;
+
+option go_package = "github.com/sqlanvil/sqlanvil/protos/sqlanvil";
+
+message ProjectConfig {
+  string warehouse = 1;
+
+  string default_database = 9;
+  string default_schema = 2;
+  string default_location = 16;
+
+  string assertion_schema = 5;
+
+  map<string, string> vars = 14;
+
+  string database_suffix = 15;
+  string schema_suffix = 7;
+  string table_prefix = 11;
+  string builtin_assertion_name_prefix = 18;
+
+  NotebookRuntimeOptions default_notebook_runtime_options = 17;
+
+  DefaultIcebergConfig default_iceberg_config = 19;
+
+  bool disable_assertions = 20;
+
+  string default_reservation = 21;
+
+  bool include_tests_in_compiled_graph = 22;
+
+  reserved 3, 4, 6, 8, 10, 12, 13;
+}
+
+message CompileConfig {
+  // Required.
+  string project_dir = 1;
+
+  // A list of all files in the project directory.
+  repeated string file_paths = 8;
+
+  // Project configuration overrides.
+  ProjectConfig project_config_override = 3;
+
+  // Override compilation timeout settings.
+  int32 timeout_millis = 6;
+
+  Extension extension = 10;
+
+  // Whether to emit compilation debug logs.
+  bool verbose = 11;
+
+  reserved 2, 4, 5, 7, 9, 12;
+}
+
+message Target {
+  string database = 3;
+  string schema = 1;
+  string name = 2;
+  bool include_dependent_assertions = 4;
+}
+
+enum FileFormat {
+  FILE_FORMAT_UNSPECIFIED = 0;
+  PARQUET = 1;
+}
+
+enum TableFormat {
+  TABLE_FORMAT_UNSPECIFIED = 0;
+  ICEBERG = 1;
+}
+
+message BigQueryOptions {
+  string partition_by = 1;
+  repeated string cluster_by = 2;
+  string update_partition_filter = 3;
+  map<string, string> labels = 4;
+  int32 partition_expiration_days = 5;
+  bool require_partition_filter = 6;
+  map<string, string> additional_options = 7;
+  string connection = 8;
+  TableFormat table_format = 9;
+  FileFormat file_format = 10;
+  string storage_uri = 11;
+}
+
+message GraphErrors {
+  repeated CompilationError compilation_errors = 1;
+
+  reserved 2;
+}
+
+message CompilationError {
+  string file_name = 1;
+  // TODO: Deprecate this and replace with action_target.
+  string action_name = 4;
+  Target action_target = 5;
+  string message = 2;
+  string stack = 3;
+}
+
+// Compilation mode, unspecified is interpreted to AoT.
+enum ActionCompilationMode {
+  ACTION_COMPILATION_MODE_UNSPECIFIED = 0;
+  // Ahead-of-time compilation (regular sqlanvil compilation).
+  ACTION_COMPILATION_MODE_AOT = 1;
+  // Just-in-time compilation.
+  // Will only populate jit_code fields in compiled graph,
+  // to be re-compiled at execution time.
+  ACTION_COMPILATION_MODE_JIT = 2;
+}
+
+message ActionDescriptor {
+  string description = 1;
+  // For Operations, 'columns' may be set iff has_output == true.
+  // For Assertions, 'columns' will always be empty.
+  repeated ColumnDescriptor columns = 2;
+  map<string, string> bigquery_labels = 3;
+  Metadata metadata = 4;
+  ActionCompilationMode compilation_mode = 5;
+  string reservation = 6;
+}
+
+message Metadata {
+  string overview = 1;
+  google.protobuf.Struct extra_properties = 2;
+}
+
+message ColumnDescriptor {
+  string description = 1;
+
+  // For nested records, the path might look like e.g. ["record", "column"].
+  // For simple columns, this will always contain a single entry e.g.
+  // ["column"].
+  repeated string path = 2;
+
+  string display_name = 3;
+
+  repeated string tags = 7;
+
+  // BigQuery specific settings.
+
+  repeated string bigquery_policy_tags = 8;
+
+  reserved 4, 5, 6;
+}
+
+enum TableType {
+  UNKNOWN_TYPE = 0;
+  TABLE = 1;
+  INCREMENTAL = 2;
+  VIEW = 3;
+
+  reserved 4;
+}
+
+enum OnSchemaChange {
+  // Ignore any schema changes (default).
+  IGNORE = 0;
+  // Fails if the query would result in a new column(s) being added, deleted,
+  // or renamed.
+  FAIL = 1;
+  // Does not block any new column(s) from being added.
+  EXTEND = 2;
+  // Does not block any new column(s) from being added, deleted or renamed.
+  SYNCHRONIZE = 3;
+}
+
+message Table {
+  // For legacy compatability reasons, both incremental tables and views are
+  // configured in compiled graphs via the Table proto.
+  TableType enum_type = 36;
+
+  Target target = 4;
+  Target canonical_target = 32;
+
+  repeated Target dependency_targets = 27;
+  ActionHermeticity hermeticity = 31;
+
+  bool disabled = 6;
+
+  string type = 3 [deprecated = true];
+  string query = 5;
+  bool protected = 9;
+  bool materialized = 35;
+
+  ActionDescriptor action_descriptor = 24;
+
+  repeated string tags = 23;
+
+  // Incremental only.
+  string where = 8 [deprecated = true];
+  string incremental_query = 26;
+  repeated string unique_key = 30;
+  OnSchemaChange on_schema_change = 37;
+
+  // Pre/post operations.
+  repeated string pre_ops = 13;
+  repeated string post_ops = 14;
+  repeated string incremental_pre_ops = 28;
+  repeated string incremental_post_ops = 29;
+
+  string jit_code = 38;
+
+  // Warehouse specific features.
+  BigQueryOptions bigquery = 22;
+  PostgresOptions postgres = 39;
+  SupabaseOptions supabase = 40;
+
+  // Generated.
+  string file_name = 18;
+
+  reserved 1, 2, 7, 12, 16;
+}
+
+message Operation {
+  Target target = 3;
+  Target canonical_target = 13;
+  repeated Target dependency_targets = 11;
+
+  ActionHermeticity hermeticity = 12;
+
+  bool disabled = 14;
+
+  repeated string queries = 6;
+  string jit_code = 15;
+  bool has_output = 8;
+  repeated string tags = 9;
+
+  ActionDescriptor action_descriptor = 10;
+
+  // Generated.
+  string file_name = 7;
+
+  reserved 1, 2, 4, 5;
+}
+
+message Assertion {
+  Target target = 8;
+  Target canonical_target = 13;
+
+  repeated Target dependency_targets = 11;
+  ActionHermeticity hermeticity = 12;
+
+  bool disabled = 14;
+
+  string query = 3;
+
+  repeated string tags = 9;
+
+  ActionDescriptor action_descriptor = 10;
+
+  // Only present for auto assertions.
+  Target parent_action = 15;
+
+  string jit_code = 16;
+
+  // Generated.
+  string file_name = 7;
+
+  reserved 1, 2, 4, 5, 6;
+}
+
+enum ActionHermeticity {
+  UNKNOWN = 0;
+  HERMETIC = 1;
+  NON_HERMETIC = 2;
+}
+
+message Declaration {
+  Target target = 2;
+  Target canonical_target = 5;
+
+  ActionDescriptor action_descriptor = 3;
+
+  // Generated.
+  string file_name = 4;
+
+  repeated string tags = 6;
+}
+
+message Test {
+  string name = 1;
+
+  string test_query = 2;
+  string expected_output_query = 3;
+
+  string jit_code = 10;
+
+  // Generated.
+  string file_name = 4;
+
+  bool disabled = 5;
+
+  repeated string tags = 6;
+
+  Target target = 7;
+  Target canonical_target = 8;
+
+  repeated Target dependency_targets = 9;
+}
+
+message Notebook {
+  Target target = 1;
+
+  Target canonical_target = 2;
+
+  repeated string tags = 3;
+
+  repeated Target dependency_targets = 4;
+
+  string file_name = 5;
+
+  bool disabled = 6;
+
+  string notebook_contents = 7;
+}
+
+message RepositorySnapshotDestination {
+  string repository_snapshot_uri = 1;
+}
+
+message NotebookRuntimeOptions {
+  oneof output_sink {
+    string output_bucket = 1;
+  }
+
+  string runtime_template_name = 2;
+
+  oneof repository_snapshot_storage {
+    RepositorySnapshotDestination repository_snapshot_destination = 3;
+  }
+}
+
+// Data Preparation Related entries
+message DataPreparation {
+  // Used ONLY as an identifiers. All outputs should be listed in the targets
+  // section.
+  Target target = 8;
+  Target canonical_target = 9;
+
+  // Data preparations can have more than 1 output
+  repeated Target targets = 1;
+
+  repeated Target canonical_targets = 2;
+
+  repeated string tags = 3;
+
+  repeated Target dependency_targets = 4;
+
+  string file_name = 5;
+
+  bool disabled = 6;
+
+  oneof definition {
+    // YAML file containing the data preparation YAML with resolved
+    // dependencies, as well as error table and load configuration information.
+    string data_preparation_yaml = 11;
+
+    // SQL representing a compiled SQL string
+    string query = 15;
+  }
+
+  // Error table configuration
+  Target error_table = 12;
+  int32 error_table_retention_days = 13;
+
+  LoadConfiguration load = 16;
+
+  reserved 7, 10;
+}
+
+message LoadConfiguration {
+  oneof mode {
+    SimpleLoadMode replace = 1;
+    SimpleLoadMode append = 2;
+    IncrementalLoadMode maximum = 3;
+    IncrementalLoadMode unique = 4;
+    IncrementalLoadMode automatic = 5;
+    MergeLoadMode merge = 6;
+  }
+}
+
+message SimpleLoadMode {}
+
+message IncrementalLoadMode {
+  string column_name = 1;
+}
+
+message MergeLoadMode {
+  repeated string unique_key = 1;
+}
+
+message CompiledGraph {
+  ProjectConfig project_config = 4;
+
+  repeated Table tables = 1;
+  repeated Operation operations = 2;
+  repeated Assertion assertions = 3;
+  repeated Declaration declarations = 9;
+  // Note that the Test action doesn't really belong in the compiled graph
+  // because it is not used at runtime. Consider moving it if useful.
+  repeated Test tests = 8;
+  repeated Notebook notebooks = 12;
+  repeated DataPreparation data_preparations = 13;
+
+  GraphErrors graph_errors = 7;
+
+  string sqlanvil_core_version = 10;
+
+  repeated Target targets = 11;
+
+  google.protobuf.Struct jit_data = 15;
+
+  reserved 5, 6;
+}
+
+message CoreExecutionRequest {
+  oneof request {
+    CompileExecutionRequest compile = 1;
+  }
+}
+
+message CoreExecutionResponse {
+  oneof response {
+    CompileExecutionResponse compile = 1;
+  }
+}
+
+message CompileExecutionRequest {
+  CompileConfig compile_config = 1;
+}
+
+message CompileExecutionResponse {
+  CompiledGraph compiled_graph = 1;
+}
+
+// This feature list is added to when making potentilly backwards breaking
+// changes. It lets the caller of sqlanvil Core know whether it supports the
+// change.
+enum SupportedFeatures {
+  UNKNOWN_FEATURE = 0;
+  ARRAY_BUFFER_IPC = 1;
+}

package/db_adapter.proto

@@ -0,0 +1,96 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package sqlanvil;
+
+import "core.proto";
+
+// Common structures for Evaluation, execution and compilation.
+
+message Field {
+  enum Primitive {
+    UNKNOWN = 0;
+    INTEGER = 1;
+    FLOAT = 2;
+    NUMERIC = 5;
+    BOOLEAN = 3;
+    STRING = 4;
+    DATE = 6;
+    DATETIME = 7;
+    TIMESTAMP = 8;
+    TIME = 9;
+    BYTES = 10;
+    ANY = 11;
+    GEOGRAPHY = 12;
+    BIGNUMERIC = 13;
+    JSON = 14;
+    INTERVAL = 15;
+  }
+
+  enum Flag {
+    UNKNOWN_FLAG = 0;
+    REPEATED = 1;
+  }
+
+  string name = 1;
+
+  repeated Flag flags = 6;
+
+  oneof type {
+    Primitive primitive = 7;
+    Fields struct = 3;
+  }
+
+  string description = 5;
+
+  reserved 2, 4;
+}
+
+message Fields {
+  repeated Field fields = 1;
+}
+
+message TableMetadata {
+  Target target = 1;
+  enum Type {
+    UNKNOWN = 0;
+    TABLE = 1;
+    VIEW = 2;
+    MATERIALIZED_VIEW = 3;
+  }
+  Type type = 6;
+
+  repeated Field fields = 3;
+  string description = 5;
+  map<string, string> labels = 7;
+  int64 last_updated_millis = 4;
+
+  message BigQuery {
+    bool has_streaming_buffer = 1;
+  }
+  BigQuery bigquery = 8;
+
+  reserved 2;
+}
+
+message ExecutionMetadata {
+  message BigqueryMetadata {
+    string job_id = 1;
+    int64 total_bytes_processed = 2;
+    int64 total_bytes_billed = 3;
+  }
+  BigqueryMetadata bigquery = 1;
+}

package/extension.proto

@@ -0,0 +1,46 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package sqlanvil;
+
+option java_package = "com.sqlanvil.protos";
+option java_outer_classname = "ExtensionMeta";
+option java_multiple_files = true;
+
+option go_package = "github.com/sqlanvil/sqlanvil/protos/sqlanvil";
+
+// Execution mode of compilation extension.
+enum ExtensionCompilationMode {
+  // Unspecified compiled mode - extension will be skipped.
+  COMPILATION_MODE_UNSPECIFIED = 0;
+  // Execute before regular sqlanvil compilation.
+  PROLOGUE = 1;
+  // Replace regular sqlanvil compilation.
+  APPLICATION_CODE = 2;
+}
+
+message Extension {
+  // Name of the extension (JS module) to delegate the compilation to.
+  string name = 1;
+
+  // Compilation mode of the extension, required.
+  ExtensionCompilationMode compilation_mode = 2;
+
+  // Supplementary key-value options for the extension execution.
+  // This can be used to pass environment-specific parameters or
+  // auxiliary configuration paths to the underlying extension engine.
+  map<string, string> options = 3;
+}