manifold-cli 0.0.15 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 465e6236b9114a9a9170502e55646ae094a392ee25bb9ce29af81046fdb1b386
-  data.tar.gz: b8dc973ee3caf57fcde8919b93580c2278850b90aaa1df82425f9a819fc66a2d
+  metadata.gz: 61582d1f6df6c8a674f9ef9b2374ae231fc98da47435ea5893d5077cb4dad34e
+  data.tar.gz: ab91d3938b7a382ac003a467cd04d19cc6059804fe21e099e7e2aa60675eed3a
 SHA512:
-  metadata.gz: 69d18cf361bdbf945685aab0264c3222c7c692e45141e3e10352238523f180df0b35f97689bff3bb71712897ddb52fb1f037c8a951762fa1818d6d28a560944b
-  data.tar.gz: f8e29a241c6f76a6632e5e47c421ac03d0736c1af3d7e8e2a374026d30f76aded46c02359aaa1a0d29b720c9bdc65355ed4e44a761cfe1af549d730988982e77
+  metadata.gz: 5cc06d2a611b1b29edbb466ab20cf8b82d4b99d609f57b71bd5e7f6098e7fcbd636c46a662306b8318c429148fbba3b4b6d6ddc3c603cddcb53ff81c9fe50f4c
+  data.tar.gz: b8e44688d8e47628dc718bf992eb28114fde80a7cd60975e7a49cf91ce05fdf3ed2b6bc349c73332801ed5934b0a56a6f71ca100ad4b02df763c2f67cd2a3f8b

data/lib/manifold/api/schema_generator.rb

@@ -4,8 +4,6 @@ module Manifold
   module API
     # Handles schema generation for Manifold tables
     class SchemaGenerator
-      VALID_OPERATORS = %w[AND OR NOT NAND NOR XOR XNOR].freeze
-
       def initialize(dimensions_fields, manifold_yaml)
         @dimensions_fields = dimensions_fields
         @manifold_yaml = manifold_yaml
@@ -33,39 +31,52 @@ module Manifold
       private
 
       def metrics_fields
-        return [] unless @manifold_yaml["contexts"] && @manifold_yaml["metrics"]
+        return [] unless @manifold_yaml["metrics"]
+
+        @manifold_yaml["metrics"].map do |group_name, group_config|
+          {
+            "name" => group_name,
+            "type" => "RECORD",
+            "mode" => "NULLABLE",
+            "fields" => group_metrics_fields(group_config)
+          }
+        end
+      end
+
+      def group_metrics_fields(group_config)
+        return [] unless group_config["breakouts"] && group_config["aggregations"]
 
-        @manifold_yaml["contexts"].map do |context_name, _context_config|
+        group_config["breakouts"].map do |breakout_name, _breakout_config|
           {
-            "name" => context_name,
+            "name" => breakout_name,
             "type" => "RECORD",
             "mode" => "NULLABLE",
-            "fields" => context_metrics_fields
+            "fields" => breakout_metrics_fields(group_config)
           }
         end
       end
 
-      def context_metrics_fields
+      def breakout_metrics_fields(group_config)
         [
-          *countif_fields,
-          *sumif_fields
+          *countif_fields(group_config),
+          *sumif_fields(group_config)
         ]
       end
 
-      def countif_fields
-        return [] unless @manifold_yaml.dig("metrics", "countif")
+      def countif_fields(group_config)
+        return [] unless group_config.dig("aggregations", "countif")
 
         [{
-          "name" => @manifold_yaml["metrics"]["countif"],
+          "name" => group_config["aggregations"]["countif"],
           "type" => "INTEGER",
           "mode" => "NULLABLE"
         }]
       end
 
-      def sumif_fields
-        return [] unless @manifold_yaml.dig("metrics", "sumif")
+      def sumif_fields(group_config)
+        return [] unless group_config.dig("aggregations", "sumif")
 
-        @manifold_yaml["metrics"]["sumif"].keys.map do |metric_name|
+        group_config["aggregations"]["sumif"].keys.map do |metric_name|
           {
             "name" => metric_name,
             "type" => "INTEGER",
@@ -73,12 +84,6 @@ module Manifold
           }
         end
       end
-
-      def validate_operator!(operator)
-        return if VALID_OPERATORS.include?(operator)
-
-        raise ArgumentError, "Invalid operator: #{operator}. Valid operators are: #{VALID_OPERATORS.join(", ")}"
-      end
     end
   end
 end

data/lib/manifold/api/workspace.rb

@@ -4,6 +4,8 @@ module Manifold
   module API
     # Handles terraform configuration generation
     class TerraformGenerator
+      attr_accessor :manifold_config
+
       def initialize(name, vectors, vector_service, manifold_yaml)
         @name = name
         @vectors = vectors
@@ -17,11 +19,91 @@ module Manifold
           vector_config = @vector_service.load_vector_config(vector)
           config.add_vector(vector_config)
         end
-        config.merge_config = @manifold_yaml["dimensions"]&.fetch("merge", nil) if @manifold_yaml["dimensions"]
+        config.dimensions_config = @manifold_yaml["dimensions"]&.fetch("merge", nil) if @manifold_yaml["dimensions"]
+        config.manifold_config = @manifold_yaml
         config.write(path)
       end
     end
 
+    # Handles SQL generation for manifold workspaces
+    class SqlGenerator
+      def initialize(name, manifold_yaml)
+        @name = name
+        @manifold_yaml = manifold_yaml
+      end
+
+      def generate_dimensions_merge_sql(source_sql)
+        return unless valid_dimensions_config?
+
+        sql_builder = Terraform::SQLBuilder.new(@name, @manifold_yaml)
+        sql_builder.build_dimensions_merge_sql(source_sql)
+      end
+
+      private
+
+      def valid_dimensions_config?
+        return false unless @manifold_yaml
+
+        !@manifold_yaml["dimensions"]&.dig("merge", "source").nil?
+      end
+    end
+
+    # Handles schema file generation for manifold workspaces
+    class SchemaWriter
+      def initialize(name, vectors, vector_service, manifold_yaml, logger)
+        @name = name
+        @vectors = vectors
+        @vector_service = vector_service
+        @manifold_yaml = manifold_yaml
+        @logger = logger
+      end
+
+      def write_schemas(tables_directory)
+        tables_directory.mkpath
+        write_dimensions_schema(tables_directory)
+        write_manifold_schema(tables_directory)
+      end
+
+      private
+
+      def write_dimensions_schema(tables_directory)
+        dimensions_path = tables_directory.join("dimensions.json")
+        dimensions_path.write(dimensions_schema_json.concat("\n"))
+      end
+
+      def write_manifold_schema(tables_directory)
+        manifold_path = tables_directory.join("manifold.json")
+        manifold_path.write(manifold_schema_json.concat("\n"))
+      end
+
+      def schema_generator
+        @schema_generator ||= SchemaGenerator.new(dimensions_fields, @manifold_yaml)
+      end
+
+      def manifold_schema
+        schema_generator.manifold_schema
+      end
+
+      def dimensions_schema
+        schema_generator.dimensions_schema
+      end
+
+      def dimensions_fields
+        @dimensions_fields ||= @vectors.filter_map do |vector|
+          @logger.info("Loading vector schema for '#{vector}'.")
+          @vector_service.load_vector_schema(vector)
+        end
+      end
+
+      def dimensions_schema_json
+        JSON.pretty_generate(dimensions_schema)
+      end
+
+      def manifold_schema_json
+        JSON.pretty_generate(manifold_schema)
+      end
+    end
+
     # Encapsulates a single manifold.
     class Workspace
       attr_reader :name, :template_path, :logger
@@ -49,13 +131,13 @@ module Manifold
       def generate(with_terraform: false)
         return nil unless manifold_exists? && any_vectors?
 
-        tables_directory.mkpath
-        generate_dimensions
-        generate_manifold
+        generate_schemas
         logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
 
         return unless with_terraform
 
+        write_manifold_merge_sql
+        write_dimensions_merge_sql
         generate_terraform
         logger.info("Generated Terraform configuration for workspace '#{name}'.")
       end
@@ -86,53 +168,58 @@ module Manifold
         directory.join("main.tf.json")
       end
 
-      private
+      def write_manifold_merge_sql
+        return unless manifold_file
 
-      def directory
-        Pathname.pwd.join("workspaces", name)
-      end
+        sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml)
+        metrics_builder = Terraform::MetricsBuilder.new(manifold_yaml)
+        sql = sql_builder.build_manifold_merge_sql(metrics_builder) do
+          metrics_builder.build_metrics_struct
+        end
 
-      def manifold_yaml
-        @manifold_yaml ||= YAML.safe_load_file(manifold_path)
+        routines_directory.join("merge_manifold.sql").write(sql)
       end
 
-      def generate_dimensions
-        dimensions_path.write(dimensions_schema_json.concat("\n"))
-      end
+      def write_dimensions_merge_sql
+        return unless dimensions_merge_source_exists?
 
-      def generate_manifold
-        manifold_schema_path.write(manifold_schema_json.concat("\n"))
+        sql = generate_dimensions_merge_sql
+        return unless sql
+
+        write_dimensions_merge_sql_file(sql)
       end
 
-      def manifold_schema_path
-        tables_directory.join("manifold.json")
+      def dimensions_merge_source_exists?
+        manifold_yaml["dimensions"]&.dig("merge", "source")
       end
 
-      def schema_generator
-        @schema_generator ||= SchemaGenerator.new(dimensions_fields, manifold_yaml)
+      def generate_dimensions_merge_sql
+        source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
+        SqlGenerator.new(name, manifold_yaml).generate_dimensions_merge_sql(source_sql)
       end
 
-      def manifold_schema
-        schema_generator.manifold_schema
+      def write_dimensions_merge_sql_file(sql)
+        routines_directory.mkpath
+        dimensions_merge_sql_path.write(sql)
       end
 
-      def dimensions_schema
-        schema_generator.dimensions_schema
+      def dimensions_merge_sql_path
+        routines_directory.join("merge_dimensions.sql")
       end
 
-      def dimensions_fields
-        @dimensions_fields ||= vectors.filter_map do |vector|
-          logger.info("Loading vector schema for '#{vector}'.")
-          @vector_service.load_vector_schema(vector)
-        end
+      private
+
+      def directory
+        Pathname.pwd.join("workspaces", name)
       end
 
-      def dimensions_schema_json
-        JSON.pretty_generate(dimensions_schema)
+      def manifold_yaml
+        @manifold_yaml ||= YAML.safe_load_file(manifold_path)
       end
 
-      def dimensions_path
-        tables_directory.join("dimensions.json")
+      def generate_schemas
+        SchemaWriter.new(name, vectors, @vector_service, manifold_yaml, logger)
+                    .write_schemas(tables_directory)
       end
 
       def any_vectors?
@@ -145,12 +232,9 @@ module Manifold
 
       def generate_terraform
         terraform_generator = TerraformGenerator.new(name, vectors, @vector_service, manifold_yaml)
+        terraform_generator.manifold_config = manifold_yaml
         terraform_generator.generate(terraform_main_path)
       end
-
-      def manifold_schema_json
-        JSON.pretty_generate(manifold_schema)
-      end
     end
   end
 end

data/lib/manifold/cli.rb

@@ -4,7 +4,7 @@ module Manifold
   # CLI provides command line interface functionality
   # for creating and managing umbrella projects for data management.
   class CLI < Thor
-    attr_accessor :logger, :bq_service
+    attr_accessor :logger
 
     def initialize(*args, logger: Logger.new($stdout))
       super(*args)

data/lib/manifold/templates/workspace_template.yml

@@ -12,20 +12,31 @@ timestamp:
   interval: HOUR
   field: timestamp
 
-contexts:
-  paid: IS_PAID(context.location)
-  organic: IS_ORGANIC(context.location)
-  paidOrganic:
-    fields:
-      - paid
-      - organic
-    operator: AND
-
 metrics:
-  countif: tapCount
-  sumif:
-    sequenceSum:
-      field: context.sequence
+  renders:
+    breakouts:
+      paid: IS_PAID(context.location)
+      organic: IS_ORGANIC(context.location)
+
+    aggregations:
+      countif: renderCount
+      sumif:
+        sequenceSum:
+          field: context.sequence
+
+    source: my_project.my_dataset.my_table
+    filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
+
+  taps:
+    breakouts:
+      paid: IS_PAID(context.location)
+      organic: IS_ORGANIC(context.location)
+
+    aggregations:
+      countif: tapCount
+      sumif:
+        sequenceSum:
+          field: context.sequence
 
-source: my_project.my_dataset.my_table
-filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
+    source: my_project.my_dataset.my_table
+    filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)

data/lib/manifold/terraform/workspace_configuration.rb

@@ -2,29 +2,274 @@
 
 module Manifold
   module Terraform
+    # Handles building metrics SQL for manifold routines
+    class MetricsBuilder
+      def initialize(manifold_config)
+        @manifold_config = manifold_config
+      end
+
+      def build_metrics_struct
+        return "" unless @manifold_config["metrics"]
+
+        metric_groups = @manifold_config["metrics"].map do |group_name, group_config|
+          build_group_struct(group_name, group_config)
+        end
+
+        metric_groups.join(",\n")
+      end
+
+      private
+
+      def build_group_struct(group_name, group_config)
+        return "" unless valid_group_config?(group_config)
+
+        breakout_structs = build_breakout_structs(group_config)
+        return "" if breakout_structs.empty?
+
+        "\tSTRUCT(\n#{breakout_structs.join(",\n")}\n\t) AS #{group_name}"
+      end
+
+      def valid_group_config?(group_config)
+        group_config["breakouts"] &&
+          group_config["aggregations"] &&
+          !group_config["breakouts"].empty? &&
+          !group_config["aggregations"].empty?
+      end
+
+      def build_breakout_structs(group_config)
+        group_config["breakouts"].map do |name, config|
+          build_breakout_struct(name, config, group_config)
+        end.compact
+      end
+
+      def build_breakout_struct(name, config, group_config)
+        condition = build_breakout_condition(name, config, group_config)
+        metrics = build_breakout_metrics(group_config, condition)
+        return if metrics.empty?
+
+        "\t\tSTRUCT(\n\t\t\t#{metrics}\n\t\t) AS #{name}"
+      end
+
+      def build_breakout_metrics(group_config, condition)
+        metrics = []
+        add_count_metrics(metrics, group_config, condition)
+        add_sum_metrics(metrics, group_config, condition)
+        metrics.join(",\n\t\t\t")
+      end
+
+      def add_count_metrics(metrics, group_config, condition)
+        return unless group_config.dig("aggregations", "countif")
+
+        metrics << "COUNTIF(#{condition}) AS #{group_config["aggregations"]["countif"]}"
+      end
+
+      def add_sum_metrics(metrics, group_config, condition)
+        group_config.dig("aggregations", "sumif")&.each do |name, config|
+          metrics << "SUM(IF(#{condition}, #{config["field"]}, 0)) AS #{name}"
+        end
+      end
+
+      def build_breakout_condition(_name, config, group_config)
+        return config unless config.is_a?(Hash)
+
+        operator = config["operator"]
+        fields = config["fields"]
+        build_operator_condition(operator, fields, group_config)
+      end
+
+      def build_operator_condition(operator, fields, group_config)
+        conditions = fields.map { |f| group_config["breakouts"][f] }
+        case operator
+        when "AND", "OR" then join_conditions(conditions, operator)
+        when "NOT" then negate_condition(conditions.first)
+        when "NAND", "NOR" then negate_joined_conditions(conditions, operator[1..])
+        when "XOR" then build_xor_condition(conditions)
+        when "XNOR" then build_xnor_condition(conditions)
+        else config
+        end
+      end
+
+      def join_conditions(conditions, operator)
+        conditions.join(" #{operator} ")
+      end
+
+      def negate_condition(condition)
+        "NOT (#{condition})"
+      end
+
+      def negate_joined_conditions(conditions, operator)
+        "NOT (#{join_conditions(conditions, operator)})"
+      end
+
+      def build_xor_condition(conditions)
+        "(#{conditions[0]} AND NOT #{conditions[1]}) OR (NOT #{conditions[0]} AND #{conditions[1]})"
+      end
+
+      def build_xnor_condition(conditions)
+        "(#{conditions[0]} AND #{conditions[1]}) OR (NOT #{conditions[0]} AND NOT #{conditions[1]})"
+      end
+    end
+
+    # Handles building SQL for manifold routines
+    class SQLBuilder
+      def initialize(name, manifold_config)
+        @name = name
+        @manifold_config = manifold_config
+      end
+
+      def build_manifold_merge_sql(_metrics_builder, &)
+        return "" unless valid_config?
+
+        <<~SQL
+          MERGE #{@name}.Manifold AS target USING (
+            #{build_metrics_cte(&)}
+            #{build_final_select}
+          ) AS source
+          ON source.id = target.id AND source.timestamp = target.timestamp
+          #{build_merge_actions}
+        SQL
+      end
+
+      def build_dimensions_merge_sql(source_sql)
+        <<~SQL
+          MERGE #{@name}.Dimensions AS TARGET
+          USING (
+            #{source_sql}
+          ) AS source
+          ON source.id = target.id
+          WHEN MATCHED THEN UPDATE SET target.dimensions = source.dimensions
+          WHEN NOT MATCHED THEN INSERT ROW;
+        SQL
+      end
+
+      private
+
+      def valid_config?
+        source_table && timestamp_field && @manifold_config["metrics"]
+      end
+
+      def source_table
+        first_group = @manifold_config["metrics"]&.values&.first
+        first_group&.dig("source")
+      end
+
+      def interval
+        @manifold_config&.dig("timestamp", "interval") || "DAY"
+      end
+
+      def where_clause
+        first_group = @manifold_config["metrics"]&.values&.first
+        return "" unless first_group&.dig("filter")
+
+        "WHERE #{first_group["filter"]}"
+      end
+
+      def timestamp_field
+        @manifold_config&.dig("timestamp", "field")
+      end
+
+      def build_metrics_cte(&)
+        <<~SQL
+          WITH Metrics AS (
+            #{build_metrics_select(&)}
+          )
+        SQL
+      end
+
+      def build_metrics_select(&block)
+        <<~SQL
+          SELECT
+            id,
+            TIMESTAMP_TRUNC(#{timestamp_field}, #{interval}) timestamp,
+            STRUCT(
+              #{block.call}
+            ) AS metrics
+          FROM #{source_table}
+          #{where_clause}
+          GROUP BY 1, 2
+        SQL
+      end
+
+      def build_final_select
+        <<~SQL
+          SELECT
+            id,
+            timestamp,
+            Dimensions.dimensions,
+            Metrics.metrics
+          FROM Metrics
+          LEFT JOIN #{@name}.Dimensions USING (id)
+        SQL
+      end
+
+      def build_merge_actions
+        <<~SQL
+          WHEN MATCHED THEN
+            UPDATE SET
+              metrics = source.metrics,
+              dimensions = source.dimensions
+          WHEN NOT MATCHED THEN
+            INSERT ROW;
+        SQL
+      end
+    end
+
+    # Handles building table configurations
+    class TableConfigBuilder
+      def initialize(name)
+        @name = name
+      end
+
+      def build_table_configs
+        {
+          "dimensions" => dimensions_table_config,
+          "manifold" => manifold_table_config
+        }
+      end
+
+      private
+
+      def dimensions_table_config
+        build_table_config("Dimensions")
+      end
+
+      def manifold_table_config
+        build_table_config("Manifold")
+      end
+
+      def build_table_config(table_id)
+        {
+          "dataset_id" => @name,
+          "project" => "${var.project_id}",
+          "table_id" => table_id,
+          "schema" => "${file(\"${path.module}/tables/#{table_id.downcase}.json\")}",
+          "depends_on" => ["google_bigquery_dataset.#{@name}"]
+        }
+      end
+    end
+
     # Represents a Terraform configuration for a Manifold workspace.
     class WorkspaceConfiguration < Configuration
       attr_reader :name
+      attr_writer :dimensions_config, :manifold_config
 
       def initialize(name)
         super()
         @name = name
         @vectors = []
-        @merge_config = nil
+        @dimensions_config = nil
       end
 
       def add_vector(vector_config)
         @vectors << vector_config
       end
 
-      attr_writer :merge_config
-
       def as_json
         {
           "variable" => variables_block,
           "resource" => {
             "google_bigquery_dataset" => dataset_config,
-            "google_bigquery_table" => table_config,
+            "google_bigquery_table" => TableConfigBuilder.new(name).build_table_configs,
             "google_bigquery_routine" => routine_config
           }.compact
         }
@@ -51,69 +296,40 @@ module Manifold
         }
       end
 
-      def table_config
-        {
-          "dimensions" => dimensions_table_config,
-          "manifold" => manifold_table_config
-        }
-      end
+      def routine_config
+        routines = {
+          "merge_dimensions" => dimensions_routine_attributes
+          # "merge_manifold" => manifold_routine_attributes
+        }.compact
 
-      def dimensions_table_config
-        {
-          "dataset_id" => name,
-          "project" => "${var.project_id}",
-          "table_id" => "Dimensions",
-          "schema" => "${file(\"${path.module}/tables/dimensions.json\")}",
-          "depends_on" => ["google_bigquery_dataset.#{name}"]
-        }
+        routines.empty? ? nil : routines
       end
 
-      def manifold_table_config
+      def dimensions_routine_attributes
+        return nil if @vectors.empty? || @dimensions_config.nil?
+
         {
           "dataset_id" => name,
           "project" => "${var.project_id}",
-          "table_id" => "Manifold",
-          "schema" => "${file(\"${path.module}/tables/manifold.json\")}",
+          "routine_id" => "merge_dimensions",
+          "routine_type" => "PROCEDURE",
+          "language" => "SQL",
+          "definition_body" => "${file(\"${path.module}/routines/merge_dimensions.sql\")}",
           "depends_on" => ["google_bigquery_dataset.#{name}"]
         }
       end
 
-      def routine_config
-        return nil if @vectors.empty? || @merge_config.nil?
-
-        {
-          "merge_dimensions" => routine_attributes
-        }
-      end
-
-      def routine_attributes
+      def manifold_routine_attributes
         {
           "dataset_id" => name,
           "project" => "${var.project_id}",
-          "routine_id" => "merge_dimensions",
+          "routine_id" => "merge_manifold",
           "routine_type" => "PROCEDURE",
           "language" => "SQL",
-          "definition_body" => merge_routine_definition,
+          "definition_body" => "${file(\"${path.module}/routines/merge_manifold.sql\")}",
           "depends_on" => ["google_bigquery_dataset.#{name}"]
         }
       end
-
-      def merge_routine_definition
-        source_sql = read_source_sql(@merge_config["source"])
-        <<~SQL
-          MERGE #{name}.Dimensions AS TARGET
-          USING (
-            #{source_sql}
-          ) AS source
-          ON source.id = target.id
-          WHEN MATCHED THEN UPDATE SET target.dimensions = source.dimensions
-          WHEN NOT MATCHED THEN INSERT ROW;
-        SQL
-      end
-
-      def read_source_sql(source_path)
-        File.read(Pathname.pwd.join(source_path))
-      end
     end
   end
 end

data/lib/manifold/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Manifold
-  VERSION = "0.0.15"
+  VERSION = "0.0.17"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: manifold-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.17
 platform: ruby
 authors:
 - claytongentry
 bindir: bin
 cert_chain: []
-date: 2025-02-09 00:00:00.000000000 Z
+date: 2025-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor