sequelizer 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cfd3663b92545f1bffb3c811297f81f26a2a1c237bc621c31d22d905be4bc0b
-  data.tar.gz: 1865666d4f264967f909820d9c7572dfff2b74bc3c80219c589c8197f18e5e59
+  metadata.gz: 734691cabf0f5799023e6f921c222e771ef04285462903ae109376a364970573
+  data.tar.gz: 86ea46b200a4cfa33d2adf3fb8ddbbc770b1f0d5a64041d61c19a139735e1406
 SHA512:
-  metadata.gz: c02b029eb0bf30bfb6498d75fbde32aa7a1576074c74dd55993a91a3f4d6458c53e0e1ed45c16d3177767ddde7b44ce706cbafce0269681f025acbe80f46ea35
-  data.tar.gz: f123e5d48c8178d9b5609f6f44d9744fb00f31085b11364aa839b902784f9de60beedabc7b90a7c6e4ae6b84a62d9d9a1b338190a0597fec40107518632b94f8
+  metadata.gz: 7ff33a0b6c7722d1cd5ff704916af17e70a819936c308b44fbb8031883d6e50cfe2399f42484d392d09070086ecf3dcf8db4e8678a1b9bb10a9f213dc163bf22
+  data.tar.gz: 180e2a43051c492d36f4be93802b0be3fb0b3eea65bc6457f354b96ad93b724862cccec83e7ebead01f4d2206ddeb329e1e1401f37bc217bf6c1a61240f6072f

data/.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: Run Tests
+on: [push, pull_request]
+jobs:
+  Run-Tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.2'
+        bundler-cache: true
+    - 
+      name: Run Tests
+      run: bundle exec rake test

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in sequelizer.gemspec
 gemspec
+
+group :test do
+  gem "sequel-hexspace", github: "outcomesinsights/sequel-hexspace", branch: "master"
+end

data/lib/sequel/extensions/make_readyable.rb

@@ -0,0 +1,108 @@
+module Sequel
+  module MakeReadyable
+    ##
+    # This method is primarily geared towards Spark SQL-based databases.
+    #
+    # Given some options, prepares a set of views to represent a set
+    # of tables across a collection of different schemas and external,
+    # unmanaged tables.
+    #
+    #   DB.make_ready(use_schema: :schema)
+    #   # => USE `schema`
+    #
+    # When using search_path, tables from previous schema override tables
+    # from the next schema.  This is analogous to the way Unix searches 
+    # the PATH variable for programs.
+    #
+    # Assuming the following tables: schema1.a, schema2.a, schema2.b
+    #
+    #   DB.make_ready(search_path: [:schema1, :schema2])
+    #   # => CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a;`
+    #   # => CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b;`
+    #
+    # When using Pathnames, the extension on the file becomes the format
+    # to try to read from the file.  
+    #
+    #   DB.make_ready(search_path: [Pathname.new("c.parquet"), Pathname.new("d.orc")])
+    #   # => CREATE TEMPORARY VIEW `c` USING parquet OPTIONS ('path'='c.parquet')
+    #   # => CREATE TEMPORARY VIEW `d` USING orc OPTIONS ('path'='d.orc')
+    #
+    # @param [Hash] opts the options used to prepare the database
+    # @option opts [String] :use_schema The schema to be used as the primary schema
+    # @option opts [Array] :search_path A set of sympbols (to represent schemas) or Pathnames (to represent externally managed data files)
+    def make_ready(opts = {})
+      ReadyMaker.new(self, opts).run
+    end
+  end
+
+  private 
+  class ReadyMaker
+    attr_reader :db, :opts
+
+    def initialize(db, opts)
+      @db = db
+      @opts = opts
+    end
+  
+    def run
+      if opts[:use_schema]
+        db.extension :usable
+        db.use(opts[:use_schema])
+      end
+      only_tables = Array(opts[:only])
+      created_views = (Array(opts[:except]) || [])
+      (opts[:search_path] || []).each do |schema|
+        schema = schema.is_a?(Pathname) ? schema : schema.to_sym
+        source = get_source(db, schema)
+        tables = source.tables(schema: schema) - created_views
+        tables &= only_tables unless only_tables.empty?
+        tables.each do |table|
+          create_view(source, table, schema)
+          created_views << table
+        end
+      end
+    end
+
+    def create_view(source, table, schema)
+      if schema.to_s =~ %r{/}
+        source.create_view(table, temp: true)
+      else
+        source.create_view(table, db[Sequel.qualify(schema, table)], temp: true)
+      end
+    end
+
+    def get_source(db, schema)
+      if schema.to_s =~ %r{/}
+        FileSourcerer.new(db, Pathname.new(schema))
+      else
+        db
+      end
+    end
+
+    class FileSourcerer
+      attr_reader :db, :schema
+      def initialize(db, schema)
+        @db = db
+        @schema = schema
+      end
+
+      def tables(opts = {})
+        [schema.basename(".*").to_s.to_sym]
+      end
+
+      def create_view(table, opts = {})
+        db.create_view(table, {
+          temp: true,
+          using: format,
+          options: { path: schema.expand_path }
+        }.merge(opts))
+      end
+
+      def format
+        schema.extname[1..-1]
+      end
+    end
+  end
+
+  Database.register_extension(:make_readyable, MakeReadyable)
+end

data/lib/sequel/extensions/sqls.rb

@@ -0,0 +1,31 @@
+# frozen-string-literal: true
+#
+# The sqls extension will record each SQL statement sent to the
+# database
+#
+#   DB.extension :sqls
+#   DB[:table]
+#   DB.sqls # =>  ["SELECT * FROM table LIMIT 1"]
+#
+# Related module: Sequel::Sqls
+
+module Sequel
+  module Sqls
+    attr_reader :sqls
+
+    # Record SQL statements when logging query.
+    def log_connection_yield(sql, conn, args=nil)
+      @sqls_mutex.synchronize{sqls.push(sql)}
+      super
+    end
+
+    def self.extended(db)
+      db.instance_exec do
+        @sqls_mutex ||= Mutex.new
+        @sqls ||= []
+      end
+    end
+  end
+
+  Database.register_extension(:sqls, Sqls)
+end

data/lib/sequel/extensions/usable.rb

@@ -0,0 +1,15 @@
+module Sequel
+  module Usable
+    def use(schema_name)
+      run(use_sql(schema_name))
+    end
+
+    private
+
+    def use_sql(schema_name)
+      "USE #{quote_identifier(schema_name)}"
+    end
+  end
+
+  Database.register_extension(:usable, Usable)
+end

data/lib/sequelizer/version.rb

@@ -1,4 +1,4 @@
 module Sequelizer
   # Version for the gem
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/sequelizer.rb

@@ -30,11 +30,11 @@ module Sequelizer
   def new_db(options = {})
     cached = find_cached(options)
     return cached if cached && !options[:force_new]
-    @cache[options] = ConnectionMaker.new(options).connection
+    @_sequelizer_cache[options] = ConnectionMaker.new(options).connection
   end
 
   def find_cached(options)
-    @cache ||= {}
-    @cache[options]
+    @_sequelizer_cache ||= {}
+    @_sequelizer_cache[options]
   end
 end

data/test/lib/sequel/extensions/test_make_readyable.rb

@@ -0,0 +1,88 @@
+require_relative "../../../test_helper"
+require "fileutils"
+require "pathname"
+require "sequel"
+require "sequel/extensions/make_readyable"
+
+class TestUsable < Minitest::Test
+  def setup
+    # These features are mostly intended for Spark, but sqlite is a close enough
+    # mock that we'll just roll with it
+    @db = Sequel.mock(host: :spark)
+    @db.extension :make_readyable
+    def @db.tables(opts = {})
+      case opts[:schema]
+      when :schema1
+        [:a]
+      when :schema2
+        [:a, :b]
+      when :schema3
+        [:a, :b]
+      end
+    end
+  end
+
+  def test_should_call_use_schema
+    @db.make_ready(use_schema: :some_schema)
+    assert_equal(["USE `some_schema`"], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths
+    @db.make_ready(search_path: [:schema1, :schema2, :schema3])
+    assert_equal([
+      "CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`",
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths_passed_as_strings
+    @db.make_ready(search_path: ["schema1", "schema2", "schema3"])
+    assert_equal([
+      "CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`",
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths_accepts_except
+    @db.make_ready(search_path: [:schema1, :schema2, :schema3], except: :a)
+    assert_equal([
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths_accepts_only
+    @db.make_ready(search_path: [:schema1, :schema2, :schema3], only: :b)
+    assert_equal([
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_path
+    dir = Pathname.new(Dir.mktmpdir)
+    a_file = dir + "a.parquet"
+    b_file = dir + "b.parquet"
+    FileUtils.touch(a_file.to_s)
+    FileUtils.touch(b_file.to_s)
+
+    @db.make_ready(search_path: [:schema1, a_file, b_file, :schema2])
+    sqls = @db.sqls.dup
+    assert_equal("CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`", sqls[0])
+    assert_match(%r{CREATE TEMPORARY VIEW `b` USING parquet OPTIONS \('path'='/tmp/[^/]+/b.parquet'\)}, sqls[1])
+  end
+
+  def test_should_create_views_format_based_on_path
+    dir = Pathname.new(Dir.mktmpdir)
+    a_file = dir + "a.parquet"
+    b_file = dir + "b.delta"
+    c_file = dir + "c.csv"
+    FileUtils.touch(a_file.to_s)
+    FileUtils.touch(b_file.to_s)
+    FileUtils.touch(c_file.to_s)
+
+    @db.make_ready(search_path: [a_file, b_file, c_file])
+    sqls = @db.sqls.dup
+    assert_match(%r{CREATE TEMPORARY VIEW `a` USING parquet OPTIONS \('path'='/tmp/[^/]+/a.parquet'\)}, sqls[0])
+    assert_match(%r{CREATE TEMPORARY VIEW `b` USING delta OPTIONS \('path'='/tmp/[^/]+/b.delta'\)}, sqls[1])
+    assert_match(%r{CREATE TEMPORARY VIEW `c` USING csv OPTIONS \('path'='/tmp/[^/]+/c.csv'\)}, sqls[2])
+  end
+end

data/test/lib/sequel/extensions/test_usable.rb

@@ -0,0 +1,12 @@
+require_relative '../../../test_helper'
+require 'sequel'
+require 'sequel/extensions/usable'
+
+class TestUsable < Minitest::Test
+  def test_should_call_use
+    db = Sequel.mock(host: :sqlite)
+    db.extension :usable
+    db.use(:some_schema)
+    assert_equal(db.sqls, ["USE `some_schema`"])
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequelizer
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Ryan Duryea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-09 00:00:00.000000000 Z
+date: 2024-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -145,8 +145,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Guardfile
@@ -158,7 +158,10 @@ files:
 - examples/database.yml
 - examples/dot_env.txt
 - lib/sequel/extensions/db_opts.rb
+- lib/sequel/extensions/make_readyable.rb
 - lib/sequel/extensions/settable.rb
+- lib/sequel/extensions/sqls.rb
+- lib/sequel/extensions/usable.rb
 - lib/sequelizer.rb
 - lib/sequelizer/cli.rb
 - lib/sequelizer/connection_maker.rb
@@ -171,6 +174,8 @@ files:
 - lib/sequelizer/yaml_config.rb
 - sequelizer.gemspec
 - test/lib/sequel/extensions/test_db_opts.rb
+- test/lib/sequel/extensions/test_make_readyable.rb
+- test/lib/sequel/extensions/test_usable.rb
 - test/lib/sequelizer/test_connection_maker.rb
 - test/lib/sequelizer/test_env_config.rb
 - test/lib/sequelizer/test_gemfile_modifier.rb
@@ -196,12 +201,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.4.6
 signing_key: 
 specification_version: 4
 summary: Sequel database connections via config/database.yml or .env
 test_files:
 - test/lib/sequel/extensions/test_db_opts.rb
+- test/lib/sequel/extensions/test_make_readyable.rb
+- test/lib/sequel/extensions/test_usable.rb
 - test/lib/sequelizer/test_connection_maker.rb
 - test/lib/sequelizer/test_env_config.rb
 - test/lib/sequelizer/test_gemfile_modifier.rb

data/.travis.yml

@@ -1,21 +0,0 @@
-language: ruby
-rvm:
-- 2.4
-sudo: false
-env:
-  secure: xFZ00J+axPQADZ2Dqc61Ljo56vT8/uwVzOFw9zxKQs3ThMfI8+OvlI7LV8sf0XPC5Cp+BObYCxzsqIOY8M8yo+NfknlNmlrQaPqi3lf+3tKvEFeFiQZ/jvbGReIdxRdPViVls1W3zEDdRwe9zUAiz7C+xBYCRfZRoGjfm7gx/4c=
-before_install:
-- gem install bundler
-- bundle
-script: bundle exec rake test
-jobs:
-  include:
-  - stage: deploy
-    before_install: gem install tping
-    install: true
-    script:
-      - tping --token $TRAVIS_PRO_TOKEN --user outcomesinsights --repo t_shank --pro --branch $TRAVIS_BRANCH
-      - tping --token $TRAVIS_PRO_TOKEN --user outcomesinsights --repo jigsaw-diagram-editor --pro --branch $TRAVIS_BRANCH
-notifications:
-  slack:
-    secure: b+ao+3BuBtM5nj/m1gP5AbrrTIdQiV/HkT1deXvY4gg5xZQDheDeLmOI7wSFP1o67BrwrAY7rpcwIP7S/99fudrU3rKI3+GLn8KoefdAv78Z4tsMs9rodJJ3Z3ZmnEdMK2i2+hCLJ1pzZ9Ae3e+GDHsBPkTz4+TNE1lrOPxDIUo=