manifold-cli 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0478f04fda1c307a346fb2e0a58f93949e73fadbf533b10d1bc374d3c9dd6b3
-  data.tar.gz: b6b8fcd1cdc6291af4a01b2014d61257f7b4ac585f54e1f93beaabfc31b5a511
+  metadata.gz: 1eb276f49eed54c682bf10a9edc53c567a4e61c1dc2c7ee1b27e98596c91d81f
+  data.tar.gz: 3e9374eeb7c36b5a5cd99b17a1d168b363ee161c23ffcae3ec54a170401ae34f
 SHA512:
-  metadata.gz: fb8477bd304089eb1d48c5b2f276333b5ef97d15c2eb10d154ae7ae52aee0531930ae4f5871a699abf4cefdcb1ff14cc70f3b6bf4bbfd619a6a4d8e8ab05c044
-  data.tar.gz: 5dc53ac8f096d39b85f1bfd39a47b1c6d1ed2f28d4d2a8bb0e14e9c0e687e18ab3960161fe774e47c21f0322a253b561dee46d826b554e058bbb8cdd0265dab1
+  metadata.gz: d1760a9ee8d1f2a8c78521761168696e50d87d6ca1194feb68f6a4c8a552d112624ba9c5512226f042bacedee1ae490a55f2e8ec33179e2cc588e26eb8f8e302
+  data.tar.gz: 57ff9863d1ce37d21ac58cdc8b634449754106a2472a3d1bfbebd6332d302b1ec39377ffb5dad882091df5b500bbb654c0ea385e2a60190486f903701a4d8035

data/.rubocop.yml

@@ -4,7 +4,7 @@ require: rubocop-rspec
 
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 3.1
+  TargetRubyVersion: 3.2
 
 Style/StringLiterals:
   EnforcedStyle: double_quotes

data/README.md

@@ -61,7 +61,29 @@ manifold add <data_project_name>
 After you fill out the manifold.yml file, this command generates the necessary BigQuery schema files based on the specified dimensions and metrics.
 
 ```bash
-manifold generate <data_project_name> bq
+manifold generate
+```
+
+4. **Generate Terraform Configuration (Optional)**
+
+Manifold can optionally generate Terraform configurations for managing your BigQuery resources. To generate both BigQuery schemas and Terraform configurations, use the `--tf` flag:
+
+```bash
+manifold generate --tf
+```
+
+This will create:
+
+- A root `main.tf.json` file that sets up the Google Cloud provider and workspace modules
+- Individual workspace configurations in `workspaces/<workspace_name>/main.tf.json`
+- Dataset and table definitions that reference your generated BigQuery schemas
+
+The generated Terraform configurations use the Google Cloud provider and expect a `project_id` variable to be set. You can apply these configurations using standard Terraform commands:
+
+```bash
+terraform init
+terraform plan -var="project_id=your-project-id"
+terraform apply -var="project_id=your-project-id"
 ```
 
 ## Manifold Configuration

data/lib/manifold/api/project.rb

@@ -22,8 +22,9 @@ module Manifold
         @workspaces ||= workspace_directories.map { |dir| Workspace.from_directory(dir, logger:) }
       end
 
-      def generate
-        workspaces.each(&:generate)
+      def generate(with_terraform: false)
+        workspaces.each { |w| w.generate(with_terraform:) }
+        generate_terraform_entrypoint if with_terraform
       end
 
       def workspaces_directory
@@ -39,6 +40,11 @@ module Manifold
       def workspace_directories
         workspaces_directory.children.select(&:directory?)
       end
+
+      def generate_terraform_entrypoint
+        config = Terraform::ProjectConfiguration.new(workspaces)
+        config.write(directory.join("main.tf.json"))
+      end
     end
   end
 end

data/lib/manifold/api/workspace.rb

@@ -26,10 +26,11 @@ module Manifold
         FileUtils.cp(template_path, manifold_path)
       end
 
-      def generate
+      def generate(with_terraform: false)
         return unless manifold_exists? && any_vectors?
 
         generate_dimensions
+        generate_terraform if with_terraform
         logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
       end
 
@@ -55,6 +56,10 @@ module Manifold
         directory.join("manifold.yml")
       end
 
+      def terraform_main_path
+        directory.join("main.tf.json")
+      end
+
       private
 
       def directory
@@ -99,6 +104,15 @@ module Manifold
       def vectors
         manifold_yaml["vectors"]
       end
+
+      def generate_terraform
+        config = Terraform::WorkspaceConfiguration.new(name)
+        vectors.each do |vector|
+          vector_config = @vector_service.load_vector_config(vector)
+          config.add_vector(vector_config)
+        end
+        config.write(terraform_main_path)
+      end
     end
   end
 end

data/lib/manifold/cli.rb

@@ -46,10 +46,12 @@ module Manifold
     end
 
     desc "generate", "Generate BigQuery schema for all workspaces in the project"
+    method_option :tf, type: :boolean, desc: "Generate Terraform configurations"
     def generate
-      name = Pathname.pwd.basename.to_s
-      project = API::Project.new(name, directory: Pathname.pwd, logger:)
-      project.generate
+      path = Pathname.pwd
+      name = path.basename.to_s
+      project = API::Project.new(name, directory: path, logger:)
+      project.generate(with_terraform: options[:tf])
       logger.info "Generated BigQuery schema for all workspaces in the project."
     end
   end

data/lib/manifold/services/vector_service.rb

@@ -19,6 +19,16 @@ module Manifold
         raise "Invalid YAML in vector configuration #{path}: #{e.message}"
       end
 
+      def load_vector_config(vector_name)
+        path = config_path(vector_name)
+        config = YAML.safe_load_file(path)
+        config.merge("name" => vector_name.downcase)
+      rescue Errno::ENOENT, Errno::EISDIR
+        raise "Vector configuration not found: #{path}"
+      rescue Psych::Exception => e
+        raise "Invalid YAML in vector configuration #{path}: #{e.message}"
+      end
+
       private
 
       def transform_attributes_to_schema(attributes)

data/lib/manifold/templates/vector_template.yml

@@ -5,3 +5,7 @@ attributes:
   # id: STRING
   # created_at: TIMESTAMP
   # status: STRING
+
+# Optionally, reference a view specifying how to select vector dimensions
+# merge:
+#   source: lib/views/select_my_vector.sql

data/lib/manifold/terraform/configuration.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Manifold
+  module Terraform
+    # Provides a base class for Terraform configuration files.
+    class Configuration
+      def as_json
+        raise NotImplementedError, "#{self.class} must implement #as_json"
+      end
+
+      def write(path)
+        path.write("#{JSON.pretty_generate(as_json)}\n")
+      end
+    end
+  end
+end

data/lib/manifold/terraform/project_configuration.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Manifold
+  module Terraform
+    # Represents a Terraform configuration for a Manifold project.
+    class ProjectConfiguration < Configuration
+      attr_reader :workspaces, :provider_version
+
+      DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION = "6.18.1"
+
+      def initialize(workspaces, provider_version: DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION)
+        super()
+        @workspaces = workspaces
+        @provider_version = provider_version
+      end
+
+      def as_json
+        {
+          "terraform" => terraform_block,
+          "provider" => provider_block,
+          "variable" => variables_block,
+          "module" => workspace_modules
+        }
+      end
+
+      private
+
+      def terraform_block
+        {
+          "required_providers" => {
+            "google" => {
+              "source" => "hashicorp/google",
+              "version" => provider_version
+            }
+          }
+        }
+      end
+
+      def provider_block
+        {
+          "google" => {
+            "project" => "${var.project_id}"
+          }
+        }
+      end
+
+      def variables_block
+        {
+          "project_id" => {
+            "description" => "The GCP project ID where resources will be created",
+            "type" => "string"
+          }
+        }
+      end
+
+      def workspace_modules
+        workspaces.each_with_object({}) do |workspace, modules|
+          modules[workspace.name] = {
+            "source" => "./workspaces/#{workspace.name}",
+            "project_id" => "${var.project_id}"
+          }
+        end
+      end
+    end
+  end
+end

data/lib/manifold/terraform/workspace_configuration.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Manifold
+  module Terraform
+    # Represents a Terraform configuration for a Manifold workspace.
+    class WorkspaceConfiguration < Configuration
+      attr_reader :name
+
+      def initialize(name)
+        super()
+        @name = name
+        @vectors = []
+      end
+
+      def add_vector(vector_config)
+        @vectors << vector_config
+      end
+
+      def as_json
+        {
+          "variable" => variables_block,
+          "resource" => {
+            "google_bigquery_dataset" => dataset_config,
+            "google_bigquery_table" => table_config,
+            "google_bigquery_routine" => routine_config
+          }.compact
+        }
+      end
+
+      private
+
+      def variables_block
+        {
+          "project_id" => {
+            "description" => "The GCP project ID where resources will be created",
+            "type" => "string"
+          }
+        }
+      end
+
+      def dataset_config
+        {
+          name => {
+            "dataset_id" => name,
+            "project" => "${var.project_id}",
+            "location" => "US"
+          }
+        }
+      end
+
+      def table_config
+        {
+          "dimensions" => {
+            "dataset_id" => name,
+            "project" => "${var.project_id}",
+            "table_id" => "Dimensions",
+            "schema" => "${file(\"${path.module}/tables/dimensions.json\")}",
+            "depends_on" => ["google_bigquery_dataset.#{name}"]
+          }
+        }
+      end
+
+      def routine_config
+        return nil if @vectors.empty?
+
+        routines = @vectors.filter_map { |vector| build_routine(vector) }
+        routines.empty? ? nil : routines.to_h
+      end
+
+      def build_routine(vector)
+        return nil unless vector["merge"]&.fetch("source", nil)
+
+        routine_name = "merge_#{vector["name"].downcase}_dimensions"
+        [routine_name, routine_attributes(routine_name, vector)]
+      end
+
+      def routine_attributes(routine_name, vector)
+        {
+          "dataset_id" => name,
+          "project" => "${var.project_id}",
+          "routine_id" => routine_name,
+          "routine_type" => "PROCEDURE",
+          "language" => "SQL",
+          "definition_body" => merge_routine_definition(vector),
+          "depends_on" => ["google_bigquery_dataset.#{name}"]
+        }
+      end
+
+      def merge_routine_definition(vector)
+        source_sql = read_source_sql(vector["merge"]["source"])
+        <<~SQL
+          MERGE #{name}.Dimensions AS TARGET
+          USING (
+            #{source_sql}
+          ) AS source
+          ON source.id = target.id
+          WHEN MATCHED THEN UPDATE SET target.#{vector["name"].downcase} = source.dimensions
+          WHEN NOT MATCHED THEN INSERT ROW;
+        SQL
+      end
+
+      def read_source_sql(source_path)
+        File.read(Pathname.pwd.join(source_path))
+      end
+    end
+  end
+end

data/lib/manifold/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Manifold
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: manifold-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - claytongentry
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-22 00:00:00.000000000 Z
+date: 2025-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -24,7 +23,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email:
 - clayton@bustle.com
 executables:
@@ -51,6 +49,9 @@ files:
 - lib/manifold/services/vector_service.rb
 - lib/manifold/templates/vector_template.yml
 - lib/manifold/templates/workspace_template.yml
+- lib/manifold/terraform/configuration.rb
+- lib/manifold/terraform/project_configuration.rb
+- lib/manifold/terraform/workspace_configuration.rb
 - lib/manifold/version.rb
 - sig/manifold.rbs
 homepage: https://github.com/bustle/manifold
@@ -61,7 +62,6 @@ metadata:
   source_code_uri: https://github.com/bustle/manifold
   changelog_uri: https://github.com/bustle/manifold/CHANGELOG.md
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -69,15 +69,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: A CLI for managing data infrastructures in BigQuery
 test_files: []