volt-sql 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 918bb2e8b15738d1e2d2b80cba4c09cfa24f26a8
+  data.tar.gz: a52245cae20149dec9fa0033c92ca382ef800eb9
+SHA512:
+  metadata.gz: 1383480587bfd8b8eed8ebcde8c56ebd68163366622fe9fb63c7d61afa52cea5bad104c6f0ff217bcaaecbd7782f7b5dbcdc77b9b10af66f46f981df6b1d182d
+  data.tar.gz: e52a71cfb0f1a3941fbd67f9ae62ba01e934b90a27f98440c57c27655dbdb95212af6f99764254f4d776aac4e11cbfb88a59bb1863939becb6e81f85e46b32c8

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in volt-mongo.gemspec
+gemspec
+
+gem 'volt', path: '/Users/ryanstout/Sites/volt/volt'
+gem 'rspec'
+gem 'sqlite3'
+gem 'pg', '~> 0.18.2'
+gem 'pg_json', '~> 0.1.29'

data/README.md

@@ -0,0 +1,31 @@
+# volt-sql
+
+**Note**: This is still a work in process.  We'll update this when the final version is out.
+
+This gem provides postgres support for volt.  Just include it in your gemfile and the gem will do the rest.  It is included in new projects by default (currently).
+
+```
+gem 'volt-sql'
+gem 'pg', '~> 0.18.2'
+gem 'pg_json', '~> 0.1.29'
+'
+```
+
+
+## Big Thanks
+
+First off, I wanted to say a big thanks to @jeremyevans for his hard work on Sequel, without which, this gem would not be possible.
+
+## How migrations work:
+
+When an app boots in dev mode, or a file is changed in dev mode, and a new field is detected, the following happens:
+
+if there are no orphan fields (fields that exist in the db, but not in the Models class):
+  - the field is added
+
+- If there is a single orphaned field and a single added field in the model:
+  - a migration is generated to rename the field
+  - the migration is run
+
+- If there are multiple orphans or fields added
+  - volt warns you and makes you deal with it (by creating migrations)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/app/sql/config/dependencies.rb

@@ -0,0 +1 @@
+# Component dependencies

data/app/sql/config/initializers/boot.rb

@@ -0,0 +1,5 @@
+require 'sql/lib/sql_adaptor_client'
+if RUBY_PLATFORM != 'opal'
+  require 'sql/lib/sql_adaptor_server'
+end
+require 'sql/lib/store_persistor'

data/app/sql/config/routes.rb

@@ -0,0 +1 @@
+# Component routes

data/app/sql/controllers/main_controller.rb

@@ -0,0 +1,5 @@
+require 'sql/config/initializers/boot'
+module Sql
+  class MainController < Volt::ModelController
+  end
+end

data/app/sql/lib/field_updater.rb

@@ -0,0 +1,103 @@
+# The FieldUpdater class is responsible for adding or updating a field, either
+# by calling sequel directly, or by generating a migration, then running it.
+require 'sql/lib/sql_logger'
+
+module Volt
+  module Sql
+    class FieldUpdater
+      include SqlLogger
+
+      def initialize(db, table_reconcile)
+        @db = db
+        @table_reconcile = table_reconcile
+      end
+
+      # Update a field (or create it)
+      def update_field(model_class, table_name, db_field, column_name, klasses, opts)
+        sequel_class, sequel_opts = Helper.column_type_and_options_for_sequel(klasses, opts)
+
+        # Check if column exists
+        if !db_field
+
+          log("Add field #{column_name} to #{table_name}")
+          # Column does not exist, add it.
+          # Make sure klass matches
+          @db.add_column(table_name, column_name, sequel_class, sequel_opts)
+        else
+          db_class, db_opts = @table_reconcile.sequel_class_and_opts_from_db(db_field)
+
+          if db_class != sequel_class || db_opts != sequel_opts
+
+            # Data type has changed, migrate
+            up_code = []
+            down_code = []
+
+            if db_opts != sequel_opts
+              # Fetch allow_null, keeping in mind it defaults to true
+              db_null = db_opts.fetch(:allow_null, true)
+              sequel_null = sequel_opts.fetch(:allow_null, true)
+
+              if db_null != sequel_null
+                # allow null changed
+                if sequel_null
+                  up_code << "set_column_allow_null #{table_name.inspect}, #{column_name.inspect}"
+                  down_code << "set_column_not_null #{table_name.inspect}, #{column_name.inspect}"
+                else
+                  up_code << "set_column_not_null #{table_name.inspect}, #{column_name.inspect}"
+                  down_code << "set_column_allow_null #{table_name.inspect}, #{column_name.inspect}"
+                end
+
+                db_opts.delete(:allow_null)
+                sequel_opts.delete(:allow_null)
+              end
+            end
+
+
+            if db_class != sequel_class || db_opts != sequel_opts
+              up_code << "set_column_type #{table_name.inspect}, #{column_name.inspect}, #{sequel_class}, #{sequel_opts.inspect}"
+              down_code << "set_column_type #{table_name.inspect}, #{column_name.inspect}, #{db_class}, #{db_opts.inspect}"
+            end
+
+
+            if up_code.present?
+              generate_and_run("column_change_#{table_name.to_s.gsub('/', '_')}_#{column_name}", up_code.join("\n"), down_code.join("\n"))
+            end
+
+            # TODO: Improve message
+            # raise "Data type changed, can not migrate field #{name} from #{db_field.inspect} to #{klass.inspect}"
+          end
+        end
+
+      end
+
+
+      def auto_migrate_field_rename(table_name, from_name, to_name)
+        log("Rename #{from_name} to #{to_name} on table #{table_name}")
+
+        name = "rename_#{table_name}_#{from_name}_to_#{to_name}"
+        up_code = "rename_column #{table_name.inspect}, #{from_name.inspect}, #{to_name.inspect}"
+        down_code = "rename_column #{table_name.inspect}, #{to_name.inspect}, #{from_name.inspect}"
+        generate_and_run(name, up_code, down_code)
+      end
+
+      def auto_migrate_remove_field(table_name, column_name, db_field)
+        log("Remove #{column_name} from table #{table_name}")
+
+        name = "remove_#{table_name}_#{column_name}"
+        up_code = "drop_column #{table_name.inspect}, #{column_name.inspect}"
+
+        sequel_class, sequel_options = @table_reconcile.sequel_class_and_opts_from_db(db_field)
+        down_code = "add_column #{table_name.inspect}, #{column_name.inspect}, #{sequel_class}, #{sequel_options.inspect}"
+        generate_and_run(name, up_code, down_code)
+      end
+
+      # private
+      def generate_and_run(name, up_code, down_code)
+        path = Volt::Sql::MigrationGenerator.create_migration(name, up_code, down_code)
+
+        Volt::MigrationRunner.new(@db).run_migration(path, :up)
+      end
+
+    end
+  end
+end

data/app/sql/lib/helper.rb

@@ -0,0 +1,121 @@
+# A few pure functions for converting between volt types/options and sequel
+# type/options
+
+module Volt
+  module Sql
+    module Helper
+
+
+      # This method takes in info from the db schema and returns the volt field
+      # klass and options that would have been used to create it.
+      #
+      # @returns [Array of klasses, Hash of options]
+      def self.klasses_and_options_from_db(db_field)
+        klasses = []
+        options = {}
+
+        # merge values based on map (options key from db_key)
+        {
+          :text => :text,
+          :size => :max_length,
+          :nil  => :allow_null,
+          :default => :ruby_default
+        }.each_pair do |opts_key, db_key|
+          options[opts_key] = db_field[db_key] if db_field.has_key?(db_key)
+        end
+
+        options.delete(:default) if options[:default] == nil
+
+        db_type = db_field[:db_type].to_sym
+
+        case db_field[:type]
+        when :string
+          klasses << String
+        when :datetime
+          klasses << VoltTime
+        when :boolean
+          klasses << Volt::Boolean
+        when :float
+          klasses << Float
+        else
+          case db_type
+          when :text
+          when :string
+            klasses << String
+          when :numeric
+            klasses << Numeric
+          when :integer
+            klasses << Fixnum
+          end
+        end
+
+        if klasses.size == 0
+          raise "Could not match database type #{db_type} in #{db_field.inspect}"
+        end
+
+        # Default is to allow nil
+        unless options[:nil] == false
+          klasses << NilClass
+        end
+        options.delete(:nil)
+
+        return klasses, options
+      end
+
+
+      # Takes in the klass and options specified on the model or an add_column
+      # and returns the correct klass/options for add_column in sequel.
+      def self.column_type_and_options_for_sequel(klasses, options)
+        options = options.dup
+
+        # Remove from start fields
+        klasses ||= [String, NilClass]
+
+        allow_nil = klasses.include?(NilClass)
+        klasses = klasses.reject {|klass| klass == NilClass }
+
+        if klasses.size > 1
+          raise MultiTypeException, 'the sql adaptor only supports a single type (or NilClass) for each field.'
+        end
+
+        klass = klasses.first
+
+        if options.has_key?(:nil)
+          options[:allow_null] = options.delete(:nil)
+        else
+          options[:allow_null] = allow_nil
+        end
+
+        if klass == String
+          # If no length restriction, make it text
+          if options[:size]
+            if options[:size] > 255
+              # Make the field text
+              options.delete(:size)
+              options[:text] = true
+            end
+          else
+            options[:text] = true
+          end
+        elsif klass == VoltTime
+          klass = Time
+        elsif klass == Volt::Boolean || klass == TrueClass || klass == FalseClass
+          klass = TrueClass # what sequel uses for booleans
+        end
+
+        return klass, options
+      end
+
+      # When asking for indexes on a table, the deferrable option will show
+      # as nil if it wasn't set, we remove this to normalize it to the volt
+      # options.
+      def self.normalized_indexes_from_table(db, table_name)
+        db.indexes(table_name).map do |name, options|
+          # Remove the deferrable that defaults to false (nil)
+          options = options.reject {|key, value| key == :deferrable && !value }
+          [name, options]
+        end.to_h
+      end
+    end
+  end
+end

data/app/sql/lib/index_updater.rb

@@ -0,0 +1,82 @@
+# The IndexUpdater looks at the current indexes on a table and those in the db
+# and reconciles them.  Since indexes are immutable, this means dropping any
+# that don't match and creating new ones.  (simple)  This also means we don't
+# need any migrations.
+
+require 'sql/lib/sql_logger'
+require 'sql/lib/helper'
+
+module Volt
+  module Sql
+    class IndexUpdater
+      include SqlLogger
+
+      def initialize(db, model_class, table_name)
+        @db = db
+        @table_name = table_name
+
+        model_indexes = model_class.indexes
+        db_indexes = Helper.normalized_indexes_from_table(@db, table_name)
+
+        model_indexes.each_pair do |name, options|
+          # See if we have a matching columns/options
+          if db_indexes[name] == options
+            # Matches, ignore it
+            db_indexes.delete(name)
+          else
+            # Something changed, if a db_index for the name exists,
+            # delete it, because the options changed
+            if (db_opts = db_indexes[name])
+              # Drop the index, drop it from the liast of db_indexes
+              drop_index(name, db_opts)
+              db_indexes.delete(name)
+            end
+
+            # Create the new index
+            add_index(name, options)
+          end
+        end
+
+        # drop any remaining db_indexes, because they are no longer defined in
+        # the model
+        db_indexes.each do |name, options|
+          drop_index(name, options)
+        end
+
+        @db.indexes(table_name)
+      end
+
+
+      def drop_index(name, options)
+        columns, options = columns_and_options(name, options)
+        log("drop index on #{columns.inspect}, #{options.inspect}")
+
+        @db.alter_table(@table_name) do
+          drop_index(columns, options.select {|k,v| k == :name })
+        end
+      end
+
+      def add_index(name, options)
+        columns, options = columns_and_options(name, options)
+        log("add index on #{columns.inspect}, #{options.inspect}")
+        @db.alter_table(@table_name) do
+          add_index(columns, options)
+        end
+      end
+
+      private
+
+      # Convert to the columns, options(with name) format used by sequel for
+      # add_index/drop_index
+      def columns_and_options(name, options)
+        options = options.dup
+        columns = options.delete(:columns)
+
+        options[:name] = name
+
+        return columns, options
+      end
+
+    end
+  end
+end

data/app/sql/lib/migration.rb

@@ -0,0 +1,52 @@
+# Add database methods for the migration class
+module Volt
+  class Migration
+    def initialize(db=nil)
+      @db = db || Volt.current_app.database.db
+    end
+
+    def add_column(table_name, column_name, klasses, options={})
+      sequel_type, sequel_options = Helper.column_type_and_options_for_sequel(klasses, options)
+      @db.alter_table(table_name) do
+        add_column(column_name, sequel_type, sequel_options)
+      end
+    end
+
+    def rename_column(table_name, from, to, options={})
+      # TODO: add options check
+      @db.alter_table(table_name) do
+        rename_column from, to
+      end
+    end
+
+    def drop_column(table_name, column_name)
+      @db.alter_table(table_name) do
+        drop_column column_name
+      end
+    end
+
+    def set_column_type(table_name, column_name, type, options={})
+      @db.alter_table(table_name) do
+        set_column_type(column_name, type, options)
+      end
+    end
+
+    def set_column_allow_null(table_name, column_name)
+      @db.alter_table(table_name) do
+        set_column_allow_null(column_name)
+      end
+    end
+
+    def set_column_not_null(table_name, column_name)
+      @db.alter_table(table_name) do
+        set_column_not_null(column_name)
+      end
+    end
+
+    def set_column_default(table_name, column_name, default)
+      @db.alter_table(table_name) do
+        set_column_default(column_name, default)
+      end
+    end
+  end
+end

data/app/sql/lib/migration_generator.rb

@@ -0,0 +1,44 @@
+# Run and create migrations programatically.
+require 'fileutils'
+
+module Volt
+  module Sql
+    module MigrationGenerator
+      def self.create_migration(name, up_content, down_content)
+        timestamp = Time.now.to_i
+        file_name = "#{timestamp}_#{name.underscore}"
+        class_name = name.camelize
+        output_file = "#{Dir.pwd}/config/db/migrations/#{file_name}.rb"
+
+        FileUtils.mkdir_p(File.dirname(output_file))
+
+        content = <<-END.gsub(/^ {8}/, '')
+        class #{class_name} < Volt::Migration
+          def up
+            #{indent_string(up_content, 4)}
+          end
+
+          def down
+            #{indent_string(down_content, 4)}
+          end
+        end
+        END
+
+        File.open(output_file, 'w') {|f| f.write(content) }
+
+        # return the path to the file
+        output_file
+      end
+
+      def self.indent_string(string, count)
+        string.split("\n").map.with_index do |line,index|
+          if index == 0
+            string
+          else
+            (' ' * count) + string
+          end
+        end.join("\n")
+      end
+    end
+  end
+end

data/app/sql/lib/reconcile.rb

@@ -0,0 +1,51 @@
+require 'sql/lib/table_reconcile'
+require 'uri'
+
+module Volt
+  # Add reconcile! directly to the model (on the server)
+  class Model
+    class_attribute :reconciled
+  end
+
+  class MultiTypeException < Exception
+
+  end
+
+  module Sql
+    class Reconcile
+      attr_reader :db
+      def initialize(adaptor, db)
+        @adaptor = adaptor
+        @db = db
+      end
+
+      # reconcile takes the database from its current state to the state defined
+      # in the model classes with the field helper
+      def reconcile!
+        Volt::RootModels.model_classes.each do |model_class|
+          TableReconcile.new(@adaptor, @db, model_class).run
+        end
+
+        # After the initial reconcile!, we add a listener for any new models
+        # created, so we can reconcile them (in specs mostly)
+        reset!
+        @@listener = RootModels.on('model_created') do |model_class|
+          # We do a full invalidate and wait for the next db access, because the
+          # model_created gets called before the class is actually fully defined.
+          # (ruby inherited limitation)
+          @adaptor.invalidate_reconcile!
+        end
+      end
+
+
+      # Called to clear the listener
+      def reset!
+        if defined?(@@listener) && @@listener
+          @@listener.remove
+          @@listener = nil
+        end
+      end
+
+    end
+  end
+end

data/app/sql/lib/sql_adaptor_client.rb

@@ -0,0 +1,110 @@
+module Volt
+  class DataStore
+    class SqlAdaptorClient < BaseAdaptorClient
+      data_store_methods :where, :offset, :skip, :order, :limit, :count, :includes
+
+      module SqlArrayStore
+        def skip(*args)
+          add_query_part(:offset, *args)
+        end
+
+        # Count without arguments or a block makes its own query to the backend.
+        # If you pass an arg or block, it will run ```all``` on the Cursor, then
+        # run a normal ruby ```.count``` on it, passing the args.
+        def count(*args, &block)
+          if args || block
+            @model.reactive_count(*args, &block)
+          else
+            cursor = add_query_part(:count)
+
+            cursor.persistor.value
+          end
+        end
+      end
+
+      # Due to the way define_method works, we need to remove the generated
+      # methods from data_store_methods before we over-ride them.
+      Volt::Persistors::ArrayStore.send(:remove_method, :skip)
+      Volt::Persistors::ArrayStore.send(:remove_method, :count)
+
+      # include sql's methods on ArrayStore
+      Volt::Persistors::ArrayStore.send(:include, SqlArrayStore)
+
+      module SqlArrayModel
+        def dataset
+          Volt::DataStore.fetch(Volt.current_app).db.from(collection_name)
+        end
+      end
+
+      Volt::ArrayModel.send(:include, SqlArrayModel)
+
+
+      # In the volt query dsl (and sql), there's a lot of ways to express the
+      # same query.  Its better for performance however if queries can be
+      # uniquely identified.  To make that happen, we normalize queries.
+      def self.normalize_query(query)
+        # query = convert_wheres_to_block(query)
+        query = merge_wheres_and_move_to_front(query)
+
+        query = reject_offset_zero(query)
+
+        query
+      end
+
+      # Where's can use either a hash arg, or a block.  If the where has a hash
+      # arg, we convert it to block style, so it can be unified.
+      def self.convert_wheres_to_block(query)
+        wheres = []
+
+        query.reject! do |query_part|
+          if query_part[0] == 'where'
+            wheres << query_part
+            # reject
+            true
+          else
+            # keep
+            false
+          end
+        end
+      end
+
+      def self.merge_wheres_and_move_to_front(query)
+        # Map first parts to string
+        query = query.map { |v| v[0] = v[0].to_s; v }
+        has_where = query.find { |v| v[0] == 'find' }
+
+        # if has_find
+        #   # merge any finds
+        #   merged_find_query = {}
+        #   query = query.reject do |query_part|
+        #     if query_part[0] == 'find'
+        #       # on a find, merge into finds
+        #       find_query = query_part[1]
+        #       merged_find_query.merge!(find_query) if find_query
+
+        #       # reject
+        #       true
+        #     else
+        #       false
+        #     end
+        #   end
+
+        #   # Add finds to the front
+        #   query.insert(0, ['find', merged_find_query])
+        # else
+        #   # No find was done, add it in the first position
+        #   query.insert(0, ['find'])
+        # end
+
+        query
+      end
+
+      def self.reject_offset_zero(query)
+        query.reject do |query_part|
+          query_part[0] == 'offset' && query_part[1] == 0
+        end
+      end
+
+    end
+  end
+end