sequelizer 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96e14a988922e4e961273049b4ec5137332a6276702fbc7ff4c7ece29dbf10da
-  data.tar.gz: e8eb82b04cbacfebdc8abd98ac4a85614d366097a455f0b6b9bf7ed7df8d41bc
+  metadata.gz: 734691cabf0f5799023e6f921c222e771ef04285462903ae109376a364970573
+  data.tar.gz: 86ea46b200a4cfa33d2adf3fb8ddbbc770b1f0d5a64041d61c19a139735e1406
 SHA512:
-  metadata.gz: 98e32334ee46938dc39b5ae853ec54e56f6577fdefb86637f50ea24a77f33730f6059ba7ee75138b150ae46191a9f6957f86c44183b26564c934b56bdc3b286e
-  data.tar.gz: d56db775cf833b2935afa22d9308d91820d980150a6e0d7c8ef058d3d2727cabc7b3e949444fe195ddf1109e62b46671e0396629b81d58ae4bb7e561d2d93a73
+  metadata.gz: 7ff33a0b6c7722d1cd5ff704916af17e70a819936c308b44fbb8031883d6e50cfe2399f42484d392d09070086ecf3dcf8db4e8678a1b9bb10a9f213dc163bf22
+  data.tar.gz: 180e2a43051c492d36f4be93802b0be3fb0b3eea65bc6457f354b96ad93b724862cccec83e7ebead01f4d2206ddeb329e1e1401f37bc217bf6c1a61240f6072f

data/.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: Run Tests
+on: [push, pull_request]
+jobs:
+  Run-Tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.2'
+        bundler-cache: true
+    - 
+      name: Run Tests
+      run: bundle exec rake test

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in sequelizer.gemspec
 gemspec
+
+group :test do
+  gem "sequel-hexspace", github: "outcomesinsights/sequel-hexspace", branch: "master"
+end

data/lib/sequel/extensions/make_readyable.rb

@@ -0,0 +1,108 @@
+module Sequel
+  module MakeReadyable
+    ##
+    # This method is primarily geared towards Spark SQL-based databases.
+    #
+    # Given some options, prepares a set of views to represent a set
+    # of tables across a collection of different schemas and external,
+    # unmanaged tables.
+    #
+    #   DB.make_ready(use_schema: :schema)
+    #   # => USE `schema`
+    #
+    # When using search_path, tables from previous schema override tables
+    # from the next schema.  This is analogous to the way Unix searches 
+    # the PATH variable for programs.
+    #
+    # Assuming the following tables: schema1.a, schema2.a, schema2.b
+    #
+    #   DB.make_ready(search_path: [:schema1, :schema2])
+    #   # => CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a;`
+    #   # => CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b;`
+    #
+    # When using Pathnames, the extension on the file becomes the format
+    # to try to read from the file.  
+    #
+    #   DB.make_ready(search_path: [Pathname.new("c.parquet"), Pathname.new("d.orc")])
+    #   # => CREATE TEMPORARY VIEW `c` USING parquet OPTIONS ('path'='c.parquet')
+    #   # => CREATE TEMPORARY VIEW `d` USING orc OPTIONS ('path'='d.orc')
+    #
+    # @param [Hash] opts the options used to prepare the database
+    # @option opts [String] :use_schema The schema to be used as the primary schema
+    # @option opts [Array] :search_path A set of sympbols (to represent schemas) or Pathnames (to represent externally managed data files)
+    def make_ready(opts = {})
+      ReadyMaker.new(self, opts).run
+    end
+  end
+
+  private 
+  class ReadyMaker
+    attr_reader :db, :opts
+
+    def initialize(db, opts)
+      @db = db
+      @opts = opts
+    end
+  
+    def run
+      if opts[:use_schema]
+        db.extension :usable
+        db.use(opts[:use_schema])
+      end
+      only_tables = Array(opts[:only])
+      created_views = (Array(opts[:except]) || [])
+      (opts[:search_path] || []).each do |schema|
+        schema = schema.is_a?(Pathname) ? schema : schema.to_sym
+        source = get_source(db, schema)
+        tables = source.tables(schema: schema) - created_views
+        tables &= only_tables unless only_tables.empty?
+        tables.each do |table|
+          create_view(source, table, schema)
+          created_views << table
+        end
+      end
+    end
+
+    def create_view(source, table, schema)
+      if schema.to_s =~ %r{/}
+        source.create_view(table, temp: true)
+      else
+        source.create_view(table, db[Sequel.qualify(schema, table)], temp: true)
+      end
+    end
+
+    def get_source(db, schema)
+      if schema.to_s =~ %r{/}
+        FileSourcerer.new(db, Pathname.new(schema))
+      else
+        db
+      end
+    end
+
+    class FileSourcerer
+      attr_reader :db, :schema
+      def initialize(db, schema)
+        @db = db
+        @schema = schema
+      end
+
+      def tables(opts = {})
+        [schema.basename(".*").to_s.to_sym]
+      end
+
+      def create_view(table, opts = {})
+        db.create_view(table, {
+          temp: true,
+          using: format,
+          options: { path: schema.expand_path }
+        }.merge(opts))
+      end
+
+      def format
+        schema.extname[1..-1]
+      end
+    end
+  end
+
+  Database.register_extension(:make_readyable, MakeReadyable)
+end

data/lib/sequel/extensions/sqls.rb

@@ -0,0 +1,31 @@
+# frozen-string-literal: true
+#
+# The sqls extension will record each SQL statement sent to the
+# database
+#
+#   DB.extension :sqls
+#   DB[:table]
+#   DB.sqls # =>  ["SELECT * FROM table LIMIT 1"]
+#
+# Related module: Sequel::Sqls
+
+module Sequel
+  module Sqls
+    attr_reader :sqls
+
+    # Record SQL statements when logging query.
+    def log_connection_yield(sql, conn, args=nil)
+      @sqls_mutex.synchronize{sqls.push(sql)}
+      super
+    end
+
+    def self.extended(db)
+      db.instance_exec do
+        @sqls_mutex ||= Mutex.new
+        @sqls ||= []
+      end
+    end
+  end
+
+  Database.register_extension(:sqls, Sqls)
+end

data/lib/sequel/extensions/usable.rb

@@ -0,0 +1,15 @@
+module Sequel
+  module Usable
+    def use(schema_name)
+      run(use_sql(schema_name))
+    end
+
+    private
+
+    def use_sql(schema_name)
+      "USE #{quote_identifier(schema_name)}"
+    end
+  end
+
+  Database.register_extension(:usable, Usable)
+end

data/lib/sequelizer/options.rb

@@ -39,7 +39,8 @@ module Sequelizer
     # the string is returned without modification
     def fix_options(passed_options)
       return passed_options unless passed_options.nil? || passed_options.is_a?(Hash)
-      sequelizer_options = db_config.merge(OptionsHash.new(passed_options || {}).to_hash)
+      opts = OptionsHash.new(passed_options || {}).to_hash
+      sequelizer_options = db_config(opts).merge(opts)
 
       if sequelizer_options[:adapter] =~ /^postgres/
         sequelizer_options[:adapter] = 'postgres'
@@ -65,11 +66,12 @@ module Sequelizer
     #  - ~/.config/sequelizer.yml if it exists
     #  - config/database.yml if it exists
     #  - environment variables (also reads from .env)
-    def db_config
+    def db_config(opts)
       @db_config ||= begin
-        opts = OptionsHash.new(YamlConfig.user_config.options)
-        opts.merge!(YamlConfig.local_config.options)
-        opts.merge!(EnvConfig.new.options)
+        opts = OptionsHash.new(opts)
+        opts.merge!(YamlConfig.user_config.options) unless opts[:ignore_yaml]
+        opts.merge!(YamlConfig.local_config.options) unless opts[:ignore_yaml]
+        opts.merge!(EnvConfig.new.options) unless opts[:ignore_env]
         opts
       end
     end

data/lib/sequelizer/version.rb

@@ -1,4 +1,4 @@
 module Sequelizer
   # Version for the gem
-  VERSION = "0.1.2"
+  VERSION = "0.1.4"
 end

data/lib/sequelizer.rb

@@ -30,11 +30,11 @@ module Sequelizer
   def new_db(options = {})
     cached = find_cached(options)
     return cached if cached && !options[:force_new]
-    @cache[options] = ConnectionMaker.new(options).connection
+    @_sequelizer_cache[options] = ConnectionMaker.new(options).connection
   end
 
   def find_cached(options)
-    @cache ||= {}
-    @cache[options]
+    @_sequelizer_cache ||= {}
+    @_sequelizer_cache[options]
   end
 end

data/sequelizer.gemspec

@@ -25,6 +25,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_dependency 'sequel', '~> 5.0'
   spec.add_dependency 'dotenv', '~> 2.1'
-  spec.add_dependency 'thor', '~> 0.19'
+  spec.add_dependency 'thor', '~> 1.0'
   spec.add_dependency 'hashie', '~> 3.2'
 end

data/test/lib/sequel/extensions/test_make_readyable.rb

@@ -0,0 +1,88 @@
+require_relative "../../../test_helper"
+require "fileutils"
+require "pathname"
+require "sequel"
+require "sequel/extensions/make_readyable"
+
+class TestUsable < Minitest::Test
+  def setup
+    # These features are mostly intended for Spark, but sqlite is a close enough
+    # mock that we'll just roll with it
+    @db = Sequel.mock(host: :spark)
+    @db.extension :make_readyable
+    def @db.tables(opts = {})
+      case opts[:schema]
+      when :schema1
+        [:a]
+      when :schema2
+        [:a, :b]
+      when :schema3
+        [:a, :b]
+      end
+    end
+  end
+
+  def test_should_call_use_schema
+    @db.make_ready(use_schema: :some_schema)
+    assert_equal(["USE `some_schema`"], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths
+    @db.make_ready(search_path: [:schema1, :schema2, :schema3])
+    assert_equal([
+      "CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`",
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths_passed_as_strings
+    @db.make_ready(search_path: ["schema1", "schema2", "schema3"])
+    assert_equal([
+      "CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`",
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths_accepts_except
+    @db.make_ready(search_path: [:schema1, :schema2, :schema3], except: :a)
+    assert_equal([
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_tables_in_search_paths_accepts_only
+    @db.make_ready(search_path: [:schema1, :schema2, :schema3], only: :b)
+    assert_equal([
+      "CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
+    ], @db.sqls)
+  end
+
+  def test_should_create_views_based_on_path
+    dir = Pathname.new(Dir.mktmpdir)
+    a_file = dir + "a.parquet"
+    b_file = dir + "b.parquet"
+    FileUtils.touch(a_file.to_s)
+    FileUtils.touch(b_file.to_s)
+
+    @db.make_ready(search_path: [:schema1, a_file, b_file, :schema2])
+    sqls = @db.sqls.dup
+    assert_equal("CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`", sqls[0])
+    assert_match(%r{CREATE TEMPORARY VIEW `b` USING parquet OPTIONS \('path'='/tmp/[^/]+/b.parquet'\)}, sqls[1])
+  end
+
+  def test_should_create_views_format_based_on_path
+    dir = Pathname.new(Dir.mktmpdir)
+    a_file = dir + "a.parquet"
+    b_file = dir + "b.delta"
+    c_file = dir + "c.csv"
+    FileUtils.touch(a_file.to_s)
+    FileUtils.touch(b_file.to_s)
+    FileUtils.touch(c_file.to_s)
+
+    @db.make_ready(search_path: [a_file, b_file, c_file])
+    sqls = @db.sqls.dup
+    assert_match(%r{CREATE TEMPORARY VIEW `a` USING parquet OPTIONS \('path'='/tmp/[^/]+/a.parquet'\)}, sqls[0])
+    assert_match(%r{CREATE TEMPORARY VIEW `b` USING delta OPTIONS \('path'='/tmp/[^/]+/b.delta'\)}, sqls[1])
+    assert_match(%r{CREATE TEMPORARY VIEW `c` USING csv OPTIONS \('path'='/tmp/[^/]+/c.csv'\)}, sqls[2])
+  end
+end

data/test/lib/sequel/extensions/test_usable.rb

@@ -0,0 +1,12 @@
+require_relative '../../../test_helper'
+require 'sequel'
+require 'sequel/extensions/usable'
+
+class TestUsable < Minitest::Test
+  def test_should_call_use
+    db = Sequel.mock(host: :sqlite)
+    db.extension :usable
+    db.use(:some_schema)
+    assert_equal(db.sqls, ["USE `some_schema`"])
+  end
+end

data/test/lib/sequelizer/test_connection_maker.rb

@@ -12,23 +12,26 @@ class TestConnectionMaker < Minitest::Test
     end
   end
 
+  def with_ignored_yaml_config(opts = {})
+  end
+
+
   def with_yaml_config(options = {})
-    yaml_config = Minitest::Mock.new
-    yaml_config.expect :options, options
-    yaml_config.expect :options, options
-    Sequelizer::YamlConfig.stub :new, yaml_config do
-      yield
+    yaml_config = Sequelizer::YamlConfig.new
+    yaml_config.stub(:options, options) do
+      Sequelizer::YamlConfig.stub :new, yaml_config do
+        yield
+      end
     end
-    yaml_config.verify
   end
 
   def with_env_config(options = {})
-    env_config = Minitest::Mock.new
-    env_config.expect :options, options
-    Sequelizer::EnvConfig.stub :new, env_config do
-      yield
+    env_config = Sequelizer::EnvConfig.new
+    env_config.stub(:options, options) do
+      Sequelizer::EnvConfig.stub :new, env_config do
+        yield env_config
+      end
     end
-    env_config.verify
   end
 
   def test_reads_options_from_yaml_config
@@ -37,6 +40,12 @@ class TestConnectionMaker < Minitest::Test
     end
   end
 
+  def test_ignores_options_from_yaml_config_when_asked
+    with_yaml_config(@options) do
+      assert_nil Sequelizer::ConnectionMaker.new(ignore_yaml: true).options.to_hash[:adapter]
+    end
+  end
+
   def test_applies_settings_if_given
     with_yaml_config(@options.merge(postgres_db_opt_flim: :flam)) do
       with_env_config do
@@ -65,6 +74,14 @@ class TestConnectionMaker < Minitest::Test
     end
   end
 
+  def test_ignores_options_from_env_config_if_no_yaml_config
+    with_yaml_config do
+      with_env_config(@options) do
+        assert_nil Sequelizer::ConnectionMaker.new(ignore_env: true).options.to_hash[:adapter]
+      end
+    end
+  end
+
   def test_applies_configuration_to_connection
     opts = @options.merge(postgres_db_opt_search_path: "searchy", impala_db_opt_search_path: "searchy2")
     with_yaml_config(opts) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequelizer
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Ryan Duryea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-15 00:00:00.000000000 Z
+date: 2024-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
@@ -145,8 +145,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Guardfile
@@ -158,7 +158,10 @@ files:
 - examples/database.yml
 - examples/dot_env.txt
 - lib/sequel/extensions/db_opts.rb
+- lib/sequel/extensions/make_readyable.rb
 - lib/sequel/extensions/settable.rb
+- lib/sequel/extensions/sqls.rb
+- lib/sequel/extensions/usable.rb
 - lib/sequelizer.rb
 - lib/sequelizer/cli.rb
 - lib/sequelizer/connection_maker.rb
@@ -171,6 +174,8 @@ files:
 - lib/sequelizer/yaml_config.rb
 - sequelizer.gemspec
 - test/lib/sequel/extensions/test_db_opts.rb
+- test/lib/sequel/extensions/test_make_readyable.rb
+- test/lib/sequel/extensions/test_usable.rb
 - test/lib/sequelizer/test_connection_maker.rb
 - test/lib/sequelizer/test_env_config.rb
 - test/lib/sequelizer/test_gemfile_modifier.rb
@@ -196,12 +201,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.4.6
 signing_key: 
 specification_version: 4
 summary: Sequel database connections via config/database.yml or .env
 test_files:
 - test/lib/sequel/extensions/test_db_opts.rb
+- test/lib/sequel/extensions/test_make_readyable.rb
+- test/lib/sequel/extensions/test_usable.rb
 - test/lib/sequelizer/test_connection_maker.rb
 - test/lib/sequelizer/test_env_config.rb
 - test/lib/sequelizer/test_gemfile_modifier.rb

data/.travis.yml

@@ -1,21 +0,0 @@
-language: ruby
-rvm:
-- 2.4
-sudo: false
-env:
-  secure: xFZ00J+axPQADZ2Dqc61Ljo56vT8/uwVzOFw9zxKQs3ThMfI8+OvlI7LV8sf0XPC5Cp+BObYCxzsqIOY8M8yo+NfknlNmlrQaPqi3lf+3tKvEFeFiQZ/jvbGReIdxRdPViVls1W3zEDdRwe9zUAiz7C+xBYCRfZRoGjfm7gx/4c=
-before_install:
-- gem install bundler
-- bundle
-script: bundle exec rake test
-jobs:
-  include:
-  - stage: deploy
-    before_install: gem install tping
-    install: true
-    script:
-      - tping --token $TRAVIS_PRO_TOKEN --user outcomesinsights --repo t_shank --pro --branch $TRAVIS_BRANCH
-      - tping --token $TRAVIS_PRO_TOKEN --user outcomesinsights --repo jigsaw-diagram-editor --pro --branch $TRAVIS_BRANCH
-notifications:
-  slack:
-    secure: b+ao+3BuBtM5nj/m1gP5AbrrTIdQiV/HkT1deXvY4gg5xZQDheDeLmOI7wSFP1o67BrwrAY7rpcwIP7S/99fudrU3rKI3+GLn8KoefdAv78Z4tsMs9rodJJ3Z3ZmnEdMK2i2+hCLJ1pzZ9Ae3e+GDHsBPkTz4+TNE1lrOPxDIUo=