manifold-cli 0.0.18 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26f9112132a14f5bbb3cd123dc4d399ed9d618ec41638eaf43b5bbb0f870b6dd
-  data.tar.gz: cc8bc857ca5163f2e86ffe8656fe48b8fb46c3295bfa7f3ffad531597e0b15df
+  metadata.gz: 72cd86649418feb395a957902e3e966e3df9fdc2931729beeb6719b0a98e831b
+  data.tar.gz: 399938be40f27388f0d9c742b922d5ab5909cc132e76c9f0a793e89ed2f4277b
 SHA512:
-  metadata.gz: dba055e83e3ef141fd49c8af5d7079d33a81ac6a54539c32270e86ecbd73840ee183f611e99f7c985c1b7c5cbb7bd32b2ac650abc8faa85946600a2ddf10af1b
-  data.tar.gz: d44010ad67f25f6ee9bd4762ee34a319062ebb82d68af81192bdba8becebad6850fd65e3e5e12f628114dd19ba17378bbecf196320e226613bb29e4062b3a80d
+  metadata.gz: a6b9919146b5e9cce7e32b423cc9119c6371a95fa5b4733d72ac217c23fc91ba95f95c472225afbb577a5bc8a359c01f26cbe2dcbc705b9792f402bcb88d6d0a
+  data.tar.gz: 691783ca633a17ae13f0488fcd69c64521dae189cbc139d1f37a011fb65dd2d1c9fadfc43903d998b82acbbb61d52c94336962229f3b69739d5050b9e1dc51be

data/lib/manifold/api/schema_manager.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Manifold
+  module API
+    # Handles schema generation and writing for Manifold tables
+    # rubocop:disable Metrics/ClassLength
+    class SchemaManager
+      def initialize(name, vectors, vector_service, manifold_yaml, logger)
+        @name = name
+        @vectors = vectors
+        @vector_service = vector_service
+        @manifold_yaml = manifold_yaml
+        @logger = logger
+      end
+
+      # Generates and writes schemas to the specified directory
+      def write_schemas(tables_directory)
+        tables_directory.mkpath
+        write_dimensions_schema(tables_directory)
+        write_manifold_schema(tables_directory)
+        write_metrics_schemas(tables_directory)
+      end
+
+      # Returns the dimensions schema structure
+      def dimensions_schema
+        [
+          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
+          { "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
+            "fields" => dimensions_fields }
+        ]
+      end
+
+      # Returns the manifold schema structure
+      def manifold_schema
+        [
+          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
+          { "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
+          { "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
+            "fields" => dimensions_fields },
+          { "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
+            "fields" => metrics_fields }
+        ]
+      end
+
+      private
+
+      def write_dimensions_schema(tables_directory)
+        dimensions_path = tables_directory.join("dimensions.json")
+        dimensions_path.write(dimensions_schema_json.concat("\n"))
+      end
+
+      def write_manifold_schema(tables_directory)
+        manifold_path = tables_directory.join("manifold.json")
+        manifold_path.write(manifold_schema_json.concat("\n"))
+      end
+
+      def write_metrics_schemas(tables_directory)
+        return unless @manifold_yaml["metrics"]
+
+        # Create metrics subdirectory
+        metrics_directory = tables_directory.join("metrics")
+        metrics_directory.mkpath
+
+        @manifold_yaml["metrics"].each do |group_name, group_config|
+          metrics_table_path = metrics_directory.join("#{group_name}.json")
+          metrics_table_schema = metrics_table_schema(group_name, group_config)
+          metrics_table_path.write(JSON.pretty_generate(metrics_table_schema).concat("\n"))
+          @logger.info("Generated metrics table schema for '#{group_name}'.")
+        end
+      end
+
+      def metrics_table_schema(group_name, group_config)
+        [
+          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
+          { "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
+          { "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
+            "fields" => [metrics_group_field(group_name, group_config)] }
+        ]
+      end
+
+      def metrics_group_field(group_name, group_config)
+        {
+          "name" => group_name,
+          "type" => "RECORD",
+          "mode" => "NULLABLE",
+          "fields" => group_metrics_fields(group_config)
+        }
+      end
+
+      def dimensions_fields
+        @dimensions_fields ||= @vectors.filter_map do |vector|
+          @logger.info("Loading vector schema for '#{vector}'.")
+          @vector_service.load_vector_schema(vector)
+        end
+      end
+
+      def dimensions_schema_json
+        JSON.pretty_generate(dimensions_schema)
+      end
+
+      def manifold_schema_json
+        JSON.pretty_generate(manifold_schema)
+      end
+
+      def metrics_fields
+        return [] unless @manifold_yaml["metrics"]
+
+        @manifold_yaml["metrics"].map do |group_name, group_config|
+          {
+            "name" => group_name,
+            "type" => "RECORD",
+            "mode" => "NULLABLE",
+            "fields" => group_metrics_fields(group_config)
+          }
+        end
+      end
+
+      def group_metrics_fields(group_config)
+        return [] unless group_config["breakouts"] && group_config["aggregations"]
+
+        group_config["breakouts"].map do |breakout_name, _breakout_config|
+          {
+            "name" => breakout_name,
+            "type" => "RECORD",
+            "mode" => "NULLABLE",
+            "fields" => breakout_metrics_fields(group_config)
+          }
+        end
+      end
+
+      def breakout_metrics_fields(group_config)
+        [
+          *countif_fields(group_config),
+          *sumif_fields(group_config)
+        ]
+      end
+
+      def countif_fields(group_config)
+        return [] unless group_config.dig("aggregations", "countif")
+
+        [{
+          "name" => group_config["aggregations"]["countif"],
+          "type" => "INTEGER",
+          "mode" => "NULLABLE"
+        }]
+      end
+
+      def sumif_fields(group_config)
+        return [] unless group_config.dig("aggregations", "sumif")
+
+        group_config["aggregations"]["sumif"].keys.map do |metric_name|
+          {
+            "name" => metric_name,
+            "type" => "INTEGER",
+            "mode" => "NULLABLE"
+          }
+        end
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/manifold/api/workspace.rb

@@ -25,85 +25,6 @@ module Manifold
       end
     end
 
-    # Handles SQL generation for manifold workspaces
-    class SqlGenerator
-      def initialize(name, manifold_yaml)
-        @name = name
-        @manifold_yaml = manifold_yaml
-      end
-
-      def generate_dimensions_merge_sql(source_sql)
-        return unless valid_dimensions_config?
-
-        sql_builder = Terraform::SQLBuilder.new(@name, @manifold_yaml)
-        sql_builder.build_dimensions_merge_sql(source_sql)
-      end
-
-      private
-
-      def valid_dimensions_config?
-        return false unless @manifold_yaml
-
-        !@manifold_yaml["dimensions"]&.dig("merge", "source").nil?
-      end
-    end
-
-    # Handles schema file generation for manifold workspaces
-    class SchemaWriter
-      def initialize(name, vectors, vector_service, manifold_yaml, logger)
-        @name = name
-        @vectors = vectors
-        @vector_service = vector_service
-        @manifold_yaml = manifold_yaml
-        @logger = logger
-      end
-
-      def write_schemas(tables_directory)
-        tables_directory.mkpath
-        write_dimensions_schema(tables_directory)
-        write_manifold_schema(tables_directory)
-      end
-
-      private
-
-      def write_dimensions_schema(tables_directory)
-        dimensions_path = tables_directory.join("dimensions.json")
-        dimensions_path.write(dimensions_schema_json.concat("\n"))
-      end
-
-      def write_manifold_schema(tables_directory)
-        manifold_path = tables_directory.join("manifold.json")
-        manifold_path.write(manifold_schema_json.concat("\n"))
-      end
-
-      def schema_generator
-        @schema_generator ||= SchemaGenerator.new(dimensions_fields, @manifold_yaml)
-      end
-
-      def manifold_schema
-        schema_generator.manifold_schema
-      end
-
-      def dimensions_schema
-        schema_generator.dimensions_schema
-      end
-
-      def dimensions_fields
-        @dimensions_fields ||= @vectors.filter_map do |vector|
-          @logger.info("Loading vector schema for '#{vector}'.")
-          @vector_service.load_vector_schema(vector)
-        end
-      end
-
-      def dimensions_schema_json
-        JSON.pretty_generate(dimensions_schema)
-      end
-
-      def manifold_schema_json
-        JSON.pretty_generate(manifold_schema)
-      end
-    end
-
     # Encapsulates a single manifold.
     class Workspace
       attr_reader :name, :template_path, :logger
@@ -131,7 +52,7 @@ module Manifold
       def generate(with_terraform: false)
         return nil unless manifold_exists? && any_vectors?
 
-        generate_schemas
+        write_schemas
         logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
 
         return unless with_terraform
@@ -177,21 +98,20 @@ module Manifold
       end
 
       def write_dimensions_merge_sql
-        return unless dimensions_merge_source_exists?
+        return unless valid_dimensions_config?
 
-        sql = generate_dimensions_merge_sql
+        source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
+        sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml)
+        sql = sql_builder.build_dimensions_merge_sql(source_sql)
         return unless sql
 
         write_dimensions_merge_sql_file(sql)
       end
 
-      def dimensions_merge_source_exists?
-        manifold_yaml["dimensions"]&.dig("merge", "source")
-      end
+      def valid_dimensions_config?
+        return false unless manifold_yaml
 
-      def generate_dimensions_merge_sql
-        source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
-        SqlGenerator.new(name, manifold_yaml).generate_dimensions_merge_sql(source_sql)
+        !manifold_yaml["dimensions"]&.dig("merge", "source").nil?
       end
 
       def write_dimensions_merge_sql_file(sql)
@@ -213,9 +133,9 @@ module Manifold
         @manifold_yaml ||= YAML.safe_load_file(manifold_path)
       end
 
-      def generate_schemas
-        SchemaWriter.new(name, vectors, @vector_service, manifold_yaml, logger)
-                    .write_schemas(tables_directory)
+      def write_schemas
+        SchemaManager.new(name, vectors, @vector_service, manifold_yaml, logger)
+                     .write_schemas(tables_directory)
       end
 
       def any_vectors?

data/lib/manifold/templates/workspace_template.yml

@@ -24,5 +24,4 @@ metrics:
         sequenceSum:
           field: context.sequence
 
-    source: my_project.render_metrics
     filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)

data/lib/manifold/terraform/project_configuration.rb

@@ -6,7 +6,7 @@ module Manifold
     class ProjectConfiguration < Configuration
       attr_reader :workspaces, :provider_version, :skip_provider_config
 
-      DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION = "6.18.1"
+      DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION = "6.24.0"
 
       def initialize(workspaces, provider_version: DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION,
                      skip_provider_config: false)

data/lib/manifold/terraform/workspace_configuration.rb

@@ -2,50 +2,11 @@
 
 module Manifold
   module Terraform
-    # Handles building metrics SQL for manifold routines
-    class MetricsSQLBuilder
-      def initialize(name, manifold_config)
-        @name = name
-        @manifold_config = manifold_config
-      end
-
-      def build_metrics_select
-        <<~SQL
-          SELECT
-              id,
-              timestamp,
-              #{build_metrics_struct}
-            FROM #{build_metric_joins}
-        SQL
-      end
-
-      private
-
-      def build_metrics_struct
-        metric_groups = @manifold_config["metrics"].keys
-        metric_groups.map { |group| "#{group}.metrics #{group}" }.join(",\n    ")
-      end
-
-      def build_metric_joins
-        metric_groups = @manifold_config["metrics"]
-        joins = metric_groups.map { |group, config| "#{config["source"]} AS #{group}" }
-        first = joins.shift
-        return first if joins.empty?
-
-        "#{first}\n  #{joins.map { |table| "FULL OUTER JOIN #{table} USING (id, timestamp)" }.join("\n  ")}"
-      end
-
-      def timestamp_field
-        @manifold_config&.dig("timestamp", "field")
-      end
-    end
-
     # Handles building SQL for manifold routines
     class SQLBuilder
       def initialize(name, manifold_config)
         @name = name
         @manifold_config = manifold_config
-        @metrics_builder = MetricsSQLBuilder.new(name, manifold_config)
       end
 
       def build_manifold_merge_sql
@@ -75,22 +36,21 @@ module Manifold
       private
 
       def valid_config?
-        source_table && timestamp_field && @manifold_config["metrics"]
-      end
-
-      def source_table
-        first_group = @manifold_config["metrics"]&.values&.first
-        first_group&.dig("source")
+        timestamp_field && @manifold_config["metrics"] && !@manifold_config["metrics"].empty?
       end
 
       def timestamp_field
         @manifold_config&.dig("timestamp", "field")
       end
 
+      def metrics_table_name(group_name)
+        "#{group_name.capitalize}Metrics"
+      end
+
       def build_source_query
         <<~SQL
           WITH Metrics AS (
-            #{@metrics_builder.build_metrics_select}
+            #{build_metrics_select}
           )
 
           SELECT
@@ -117,23 +77,65 @@ module Manifold
             INSERT ROW;
         SQL
       end
+
+      # Metrics SQL building methods
+      def build_metrics_select
+        <<~SQL
+          SELECT
+              id,
+              timestamp,
+              #{build_metrics_struct}
+            FROM #{build_metric_joins}
+        SQL
+      end
+
+      def build_metrics_struct
+        metric_groups = @manifold_config["metrics"].keys
+        metric_groups.map { |group| "#{group}.metrics #{group}" }.join(",\n    ")
+      end
+
+      def build_metric_joins
+        metric_groups = @manifold_config["metrics"]
+        joins = metric_groups.map do |group, config|
+          table = "#{@name}.#{metrics_table_name(group)}"
+          filter = config["filter"] ? " WHERE #{config["filter"]}" : ""
+          "(SELECT * FROM #{table}#{filter}) AS #{group}"
+        end
+        first = joins.shift
+        return first if joins.empty?
+
+        "#{first}\n  #{joins.map { |table| "FULL OUTER JOIN #{table} USING (id, timestamp)" }.join("\n  ")}"
+      end
     end
 
     # Handles building table configurations
     class TableConfigBuilder
-      def initialize(name)
+      def initialize(name, manifold_config = nil)
         @name = name
+        @manifold_config = manifold_config
       end
 
       def build_table_configs
-        {
+        configs = {
           "dimensions" => dimensions_table_config,
           "manifold" => manifold_table_config
         }
+
+        if @manifold_config&.dig("metrics")
+          @manifold_config["metrics"].each_key do |group_name|
+            configs[metrics_table_name(group_name).downcase] = metrics_table_config(group_name)
+          end
+        end
+
+        configs
       end
 
       private
 
+      def metrics_table_name(group_name)
+        "#{group_name.capitalize}Metrics"
+      end
+
       def dimensions_table_config
         build_table_config("Dimensions")
       end
@@ -142,12 +144,18 @@ module Manifold
         build_table_config("Manifold")
       end
 
-      def build_table_config(table_id)
+      def metrics_table_config(group_name)
+        titlecased_name = metrics_table_name(group_name)
+        build_table_config(titlecased_name, "metrics/#{group_name}.json")
+      end
+
+      def build_table_config(table_id, schema_path = nil)
+        schema_path ||= "#{table_id.downcase}.json"
         {
           "dataset_id" => @name,
           "project" => "${var.project_id}",
           "table_id" => table_id,
-          "schema" => "${file(\"${path.module}/tables/#{table_id.downcase}.json\")}",
+          "schema" => "${file(\"${path.module}/tables/#{schema_path}\")}",
           "depends_on" => ["google_bigquery_dataset.#{@name}"]
         }
       end
@@ -174,7 +182,7 @@ module Manifold
           "variable" => variables_block,
           "resource" => {
             "google_bigquery_dataset" => dataset_config,
-            "google_bigquery_table" => TableConfigBuilder.new(name).build_table_configs,
+            "google_bigquery_table" => TableConfigBuilder.new(name, @manifold_config).build_table_configs,
             "google_bigquery_routine" => routine_config
           }.compact
         }

data/lib/manifold/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Manifold
-  VERSION = "0.0.18"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: manifold-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.1.0
 platform: ruby
 authors:
 - claytongentry
 bindir: bin
 cert_chain: []
-date: 2025-02-19 00:00:00.000000000 Z
+date: 2025-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -43,7 +43,7 @@ files:
 - lib/manifold.rb
 - lib/manifold/api.rb
 - lib/manifold/api/project.rb
-- lib/manifold/api/schema_generator.rb
+- lib/manifold/api/schema_manager.rb
 - lib/manifold/api/vector.rb
 - lib/manifold/api/workspace.rb
 - lib/manifold/cli.rb

data/lib/manifold/api/schema_generator.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-module Manifold
-  module API
-    # Handles schema generation for Manifold tables
-    class SchemaGenerator
-      def initialize(dimensions_fields, manifold_yaml)
-        @dimensions_fields = dimensions_fields
-        @manifold_yaml = manifold_yaml
-      end
-
-      def dimensions_schema
-        [
-          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
-          { "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
-            "fields" => @dimensions_fields }
-        ]
-      end
-
-      def manifold_schema
-        [
-          { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
-          { "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
-          { "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
-            "fields" => @dimensions_fields },
-          { "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
-            "fields" => metrics_fields }
-        ]
-      end
-
-      private
-
-      def metrics_fields
-        return [] unless @manifold_yaml["metrics"]
-
-        @manifold_yaml["metrics"].map do |group_name, group_config|
-          {
-            "name" => group_name,
-            "type" => "RECORD",
-            "mode" => "NULLABLE",
-            "fields" => group_metrics_fields(group_config)
-          }
-        end
-      end
-
-      def group_metrics_fields(group_config)
-        return [] unless group_config["breakouts"] && group_config["aggregations"]
-
-        group_config["breakouts"].map do |breakout_name, _breakout_config|
-          {
-            "name" => breakout_name,
-            "type" => "RECORD",
-            "mode" => "NULLABLE",
-            "fields" => breakout_metrics_fields(group_config)
-          }
-        end
-      end
-
-      def breakout_metrics_fields(group_config)
-        [
-          *countif_fields(group_config),
-          *sumif_fields(group_config)
-        ]
-      end
-
-      def countif_fields(group_config)
-        return [] unless group_config.dig("aggregations", "countif")
-
-        [{
-          "name" => group_config["aggregations"]["countif"],
-          "type" => "INTEGER",
-          "mode" => "NULLABLE"
-        }]
-      end
-
-      def sumif_fields(group_config)
-        return [] unless group_config.dig("aggregations", "sumif")
-
-        group_config["aggregations"]["sumif"].keys.map do |metric_name|
-          {
-            "name" => metric_name,
-            "type" => "INTEGER",
-            "mode" => "NULLABLE"
-          }
-        end
-      end
-    end
-  end
-end