RubyGems - rdt - Versions diffs - 0.0.9 - Mend

rdt 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

checksums.yaml +7 -0
data/lib/dbt/mermaid.rb +57 -0
data/lib/dbt/model.rb +188 -0
data/lib/dbt/runner.rb +41 -0
data/lib/dbt/sql_template_helpers.rb +54 -0
data/lib/dbt.rb +28 -0
metadata +86 -0

checksums.yaml ADDED Viewed

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e699a3cc4e7b1bcfb20fceb193ee0d7b1e805d62b70f79bfbc2d528a77e7d4b7
+  data.tar.gz: 46d2140d1b30ce7ff8533c85e941cde449d92432a77c147cb59709aaed2492af
+SHA512:
+  metadata.gz: e77b10bca5337de9917267ee5197f1452a4a24bfde5aa9999b0e3e187f37eb15bee77defd9a5f4285b20a30ab1c030835aacf10dd0d9d08b72006f5b07eb71ce
+  data.tar.gz: 43ddb567994a3c91e95d0f22f3d444074b8d889b5ac1ed2b6285c0d2c78a67512024a880b2c9f7b61869f0bd19b59c555babea878367a946fe4e592236716296

data/lib/dbt/mermaid.rb ADDED Viewed

@@ -0,0 +1,57 @@
+require "base64"
+module Dbt
+  class Mermaid
+    class << self
+      def markdown_for(dag)
+        mermaid = "flowchart LR\n"
+        dag.each do |model, dependencies|
+          mermaid += "#{model}\n"
+          dependencies.each do |dependency|
+            mermaid += "#{dependency} --> #{model}\n"
+          end
+        end
+        mermaid
+      end
+      # does not work
+      def encode_to_editor_url(md)
+        json = {
+          code: md,
+          mermaid: {
+            theme: "default"
+          },
+          updateEditor: false,
+          autoSync: true,
+          updateDiagram: false
+        }
+        encoded = Base64.urlsafe_encode64(json.to_json.force_encoding("ASCII"))
+        "https://mermaid.ink/img/#{encoded}"
+        #url = encode_mermaid(diagram)
+        encoded
+      end
+      def generate_file(chart)
+        html = <<~HTML
+          <!DOCTYPE html>
+            <html lang="en">
+              <body>
+                <style>svg { max-width: none; width: 2000px; }</style>
+                <pre class="mermaid">
+                #{chart}
+                </pre>
+                <script type="module">
+                  import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs';
+                </script>
+              </body>
+            </html>
+            HTML
+        begin
+          File.write("dependencies.html", html)
+        rescue Errno::EACCES => e
+          puts "Failed to write to file: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/dbt/model.rb ADDED Viewed

@@ -0,0 +1,188 @@
+module Dbt
+  class Model
+    include SqlTemplateHelpers
+    attr_reader :name,
+                :code,
+                :materialize_as,
+                :sources,
+                :refs,
+                :built,
+                :filepath,
+                :skip,
+                :is_incremental,
+                :unique_by_column
+    def initialize(filepath, schema = SCHEMA)
+      @filepath = filepath
+      @name = File.basename(filepath, ".sql")
+      @original_code = File.read(filepath)
+      @sources = []
+      @refs = []
+      @built = false
+      @materialize_as = "VIEW"
+      @schema = schema
+      def source(table)
+        @sources << table.to_s
+        table.to_s
+      end
+      def ref(model)
+        @refs << model.to_s
+        "#{@schema}.#{model}"
+      end
+      def this
+        "#{@schema}.#{@name}"
+      end
+      def build_as kind, unique_by: "unique_by"
+        case kind.to_s.downcase
+          when "view"
+            @materialize_as = "VIEW"
+          when "table"
+            @materialize_as = "TABLE"
+          when "incremental"
+            if get_relation_type(this) != "TABLE"
+              @materialize_as = "TABLE"
+              @is_incremental = false
+            else
+              @is_incremental = true
+              @unique_by_column = unique_by
+            end
+        else
+          raise "Invalid build_as materialization: #{kind}"
+        end
+        ""
+      end
+      def materialize
+        # legacy, use build_as :table
+        # will add warning in the future
+        build_as :table
+        ""
+      end
+      def skip
+        @skip = true
+        ""
+      end
+      @code = ERB.new(@original_code).result(binding)
+    end
+    def build
+      if @skip
+        puts "SKIPPING #{@name}"
+      elsif @is_incremental
+        puts "INCREMENTAL #{@name}"
+        assert_column_uniqueness(unique_by_column, this)
+        temp_table = "#{@schema}.#{@name}_incremental_build_temp_table"
+        # drop the temp table if it exists
+        ActiveRecord::Base.connection.execute <<~SQL
+        DROP TABLE IF EXISTS #{temp_table};
+        SQL
+        # create a temp table with the same schema as the source
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE TABLE #{temp_table} AS (
+            #{code}
+          );
+        SQL
+        assert_column_uniqueness(unique_by_column, temp_table)
+        # delete rows from the table that are in the source
+        ActiveRecord::Base.connection.execute <<~SQL
+          DELETE FROM #{this}
+          USING #{temp_table}
+          WHERE #{this}.#{unique_by_column} = #{temp_table}.#{unique_by_column};
+        SQL
+        # insert rows from the source into the table
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO #{this}
+          SELECT * FROM #{temp_table};
+        SQL
+        # drop the temp table
+        ActiveRecord::Base.connection.execute <<~SQL
+          DROP TABLE #{temp_table};
+        SQL
+      else
+        puts "BUILDING #{@name}"
+        curent_relation_type = get_relation_type(this)
+        case @materialize_as
+          when "VIEW"
+            ActiveRecord::Base.connection.execute <<~SQL
+              BEGIN;
+              #{drop_relation(this)}
+              CREATE VIEW #{this} AS (
+                #{@code}
+              );
+              COMMIT;
+            SQL
+          when "TABLE"
+            temp_table = "#{@schema}.#{@name}_build_step_temp_table"
+            ActiveRecord::Base.connection.execute <<~SQL
+              DROP TABLE IF EXISTS #{temp_table};
+              CREATE TABLE #{temp_table} AS (
+                #{@code}
+              );
+              BEGIN;
+              #{drop_relation(this)}
+              ALTER TABLE #{temp_table} RENAME TO #{@name};
+              DROP TABLE IF EXISTS #{temp_table};
+              COMMIT;
+            SQL
+          else
+            raise "Invalid materialize_as: #{@materialize_as}"
+        end
+        @built = true
+      end
+    end
+    def drop_relation(relation)
+      type = get_relation_type(relation)
+      if type.present?
+        "DROP #{type} #{relation} CASCADE;"
+      else
+        ""
+      end
+    end
+    def get_relation_type(relation)
+      relnamespace, relname = relation.split(".")
+      type =
+        ActiveRecord::Base
+          .connection
+          .execute(
+            "
+      SELECT
+        CASE c.relkind
+          WHEN 'r' THEN 'TABLE'
+          WHEN 'v' THEN 'VIEW'
+          WHEN 'm' THEN 'MATERIALIZED VIEW'
+        END AS relation_type
+      FROM pg_class c
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE c.relname = '#{relname}' AND n.nspname = '#{relnamespace}';"
+          )
+          .values
+          .first
+          &.first
+    end
+    def assert_column_uniqueness(column, relation)
+      result = ActiveRecord::Base.connection.execute <<~SQL
+        SELECT COUNT(*) = COUNT(DISTINCT #{column}) FROM #{relation};
+      SQL
+      if result.values.first.first == false
+        raise "Column #{column} is not unique in #{relation}"
+      end
+      true
+    end
+  end
+end

data/lib/dbt/runner.rb ADDED Viewed

@@ -0,0 +1,41 @@
+module Dbt
+  class Runner
+    class << self
+      def run(custom_schema = nil, glob_path = "app/sql/**/*.sql")
+        schema = custom_schema || Dbt.settings["schema"] || Dbt::SCHEMA
+        ActiveRecord::Base.connection.execute "CREATE SCHEMA IF NOT EXISTS #{schema}"
+        file_paths = Dir.glob(glob_path)
+        models = file_paths.map { |fp| Model.new(fp, schema) }
+        dependencies =
+          models.map { |m| { m.name => m.refs } }.reduce({}, :merge!)
+        check_if_all_refs_have_sql_files dependencies
+        graph = Dagwood::DependencyGraph.new dependencies
+        md = Mermaid.markdown_for dependencies
+        Mermaid.generate_file md
+        graph.order.each do |model_name|
+          models.find { |m| m.name == model_name }.build
+        end
+      end
+      def test
+        puts "Running tests..."
+        schema = Dbt.settings["schema"] || Dbt::SCHEMA
+        tables = run(schema, "app/sql_test/**/*.sql")
+        tables.each do |table|
+          puts "TEST #{table}"
+          raise "Table #{table} is not empty" unless ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{schema}.#{table}").to_a[0]["count"] == 0
+        end
+        puts "All tests passed!"
+      end
+      def check_if_all_refs_have_sql_files(dependencies)
+        dependencies.each do |key, value|
+          sem_arquivo = (value || []) - dependencies.keys
+          unless sem_arquivo.empty?
+            raise "Missing .sql model files for ref #{sem_arquivo} in model #{key}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dbt/sql_template_helpers.rb ADDED Viewed

@@ -0,0 +1,54 @@
+module Dbt
+  module SqlTemplateHelpers
+    def star relation, *exclued_columns
+      columns = ActiveRecord::Base.connection.execute("select * from #{relation} limit 0").fields - exclued_columns.map(&:to_s)
+      columns.map { |column| "#{relation}.#{column}" }.join(', ')
+    end
+    ### JSON SQL helpers
+    def j(key)
+      "body ->> '#{key}' #{key.underscore}"
+    end
+    def j_numeric(key)
+      "(body ->> '#{key}')::numeric #{key.underscore}"
+    end
+    def j_numeric_comma(key)
+      "REPLACE(REPLACE((body ->> '#{key}'),'.',''),',','.')::numeric #{key.underscore}"
+    end
+    def j_except *keys
+      "body - '#{keys.join("','")}'"
+    end
+    ### XML SQL helpers
+    def x(key)
+      "(xpath('//cmd[@t=''#{key}'']/text()', body))[1]::text AS #{key.underscore}"
+    end
+    def x_numeric(key)
+      # x_numeric was replacing '.' and ',' to '' and '.' to convert to numeric
+      # which should be done by x_numeric_comma.
+      puts "WARNING: x_numeric will not change , to . in a future version. Use x_numeric_comma instead."
+      #"(xpath('//cmd[@t=''#{key}'']/text()', body))[1]::numeric #{key.underscore}"
+      x_numeric_comma(key)
+    end
+    def x_numeric_comma(key)
+      "REPLACE(REPLACE((xpath('/xjx/cmd[@t=''#{key}'']/text()', body))[1]::text, '.', ''), ',', '.')::numeric #{key.underscore}"
+    end
+    def x_date(key)
+      "TO_DATE((xpath('/xjx/cmd[@t=''#{key}'']/text()', body))[1]::text, 'DD/MM/YYYY') #{key.underscore}"
+    end
+    def x_except *keys
+      not_in_clause = keys.map { |key| "''#{key}''" }
+      .join('or @t = ')
+      "xpath('//cmd[not(@t = #{not_in_clause})]', body)"
+    end
+  end
+end

data/lib/dbt.rb ADDED Viewed

@@ -0,0 +1,28 @@
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+require "dagwood"
+module Dbt
+  SCHEMA = "felipe_dbt"
+  def self.settings
+    @settings ||= begin
+      path = Rails.root.join("config", "dbt.yml").to_s
+      if File.exist?(path)
+        YAML.safe_load(ERB.new(File.read(path)).result, aliases: true)
+      else
+        {}
+      end
+    end
+  end
+  def self.run(...)
+    Runner.run(...)
+  end
+  def self.test(...)
+    Runner.test(...)
+  end
+end

metadata ADDED Viewed

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: rdt
+version: !ruby/object:Gem::Version
+  version: 0.0.9
+platform: ruby
+authors:
+- Felipe Mesquita
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dagwood
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+description: SQL-based data modeling for Rails applications and Postgres
+email: felipemesquita@hey.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/dbt.rb
+- lib/dbt/mermaid.rb
+- lib/dbt/model.rb
+- lib/dbt/runner.rb
+- lib/dbt/sql_template_helpers.rb
+homepage: https://github.com/felipedmesquita/dbt
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Ruby Data Tool
+test_files: []