volt-sql 0.0.1

data/app/sql/lib/sql_adaptor_server.rb

@@ -0,0 +1,438 @@
+require 'sequel'
+require 'sequel/extensions/pg_json'
+require 'sql/lib/where_call'
+require 'fileutils'
+require 'thread'
+require 'sql/lib/migration'
+require 'sql/lib/migration_generator'
+require 'sql/lib/reconcile'
+require 'sql/lib/sql_logger'
+require 'sql/lib/helper'
+require 'volt/utils/data_transformer'
+
+
+module Volt
+  class DataStore
+    class SqlAdaptorServer < BaseAdaptorServer
+      include Volt::Sql::SqlLogger
+
+      attr_reader :db, :sql_db, :adaptor_name
+
+      # :reconcile_complete is set to true after the initial load and reconcile.
+      # Any models created after this point will attempt to be auto-reconciled.
+      # This is mainly used for specs.
+      attr_reader :reconcile_complete
+
+
+      def initialize(volt_app)
+        @volt_app = volt_app
+        @db_mutex = Mutex.new
+
+        Sequel.default_timezone = :utc
+        super
+
+        # Add an invalidation callback when we add a new field.  This ensures
+        # any fields added after the first db call (which triggers the
+        # reconcile) will reconcile again.
+        inv_reconcile = lambda { invalidate_reconcile! }
+        Volt::Model.instance_eval do
+          alias :__field__ :field
+
+          define_singleton_method(:field) do |*args, &block|
+            # Call original field
+            result = __field__(*args, &block)
+            inv_reconcile.call
+            result
+          end
+        end
+
+        @volt_app.on('boot') do
+          # call ```db``` to reconcile
+          @app_booted = true
+          db
+        end
+      end
+
+      # Set db_driver on public
+      Volt.configure do |config|
+        config.db_driver = 'sqlite'
+      end
+
+      # check if the database can be connected to.
+      # @return Boolean
+      def connected?
+        return true
+        begin
+          db
+
+          true
+        rescue ::Sequel::ConnectionFailure => e
+          false
+        end
+      end
+
+      def db(skip_reconcile=false)
+        if @db && @reconcile_complete
+          return @db
+        end
+
+        @db_mutex.synchronize do
+          unless @db
+            begin
+              @adaptor_name = connect_to_db
+
+              @db.test_connection
+            rescue Sequel::DatabaseConnectionError => e
+              if e.message =~ /does not exist/
+                create_missing_database
+              else
+                raise
+              end
+
+            rescue Sequel::AdapterNotFound => e
+              missing_gem = e.message.match(/LoadError[:] cannot load such file -- ([^ ]+)$/)
+              if missing_gem
+                helpers = {
+                  'postgres' => "gem 'pg', '~> 0.18.2'\ngem 'pg_json', '~> 0.1.29'",
+                  'sqlite3'  => "gem 'sqlite3'",
+                  'mysql2'   => "gem 'mysql2'"
+                }
+
+                adaptor_name = missing_gem[1]
+                if (helper = helpers[adaptor_name])
+                  helper = "\nMake sure you have the following in your gemfile:\n" + helper + "\n\n"
+                else
+                  helper = ''
+                end
+                raise NameError.new("LoadError: cannot load the #{adaptor_name} gem.#{helper}")
+              else
+                raise
+              end
+            end
+
+            if @adaptor_name == 'postgres'
+              @db.extension :pg_json
+              # @db.extension :pg_json_ops
+            end
+
+            if ENV['LOG_SQL']
+              @db.loggers << Volt.logger
+            end
+          end
+
+          # Don't try to reconcile until the app is booted
+          reconcile! if !skip_reconcile && @app_booted
+        end
+
+        @db
+      end
+
+      # @param - a string URI, or a Hash of options
+      # @param - the adaptor name
+      def connect_uri_or_options
+        # check to see if a uri was specified
+        conf = Volt.config
+
+        uri = conf.db && conf.db.uri
+
+        if uri
+          adaptor = uri[/^([a-z]+)/]
+
+          return uri, adaptor
+        else
+          adaptor = (conf.db && conf.db.adapter || 'sqlite').to_s
+          if adaptor == 'sqlite'
+            # Make sure we have a config/db folder
+            FileUtils.mkdir_p('config/db')
+          end
+
+          data = Volt.config.db.to_h.symbolize_keys
+          data[:database] ||= "config/db/#{Volt.env.to_s}.db"
+          data[:adapter]  ||= adaptor
+
+          return data, adaptor
+        end
+      end
+
+      def connect_to_db
+        uri_opts, adaptor = connect_uri_or_options
+
+        @db = Sequel.connect(uri_opts)
+
+        if adaptor == 'sqlite'
+          @db.set_integer_booleans
+        end
+
+        adaptor
+      end
+
+      # In order to create the database, we have to connect first witout the
+      # database.
+      def create_missing_database
+        @db.disconnect
+        uri_opts, adaptor = connect_uri_or_options
+
+        if uri_opts.is_a?(String)
+          # A uri
+          *uri_opts, db_name = uri_opts.split('/')
+          uri_opts = uri_opts.join('/')
+        else
+          # Options hash
+          db_name = uri_opts.delete(:database)
+        end
+
+        @db = Sequel.connect(uri_opts)
+
+        # No database, try to create it
+        log "Database does not exist, attempting to create database #{db_name}"
+        @db.run("CREATE DATABASE #{db_name};")
+        @db.disconnect
+        @db = nil
+
+        connect_to_db
+      end
+
+      def reconcile!
+        unless @skip_reconcile
+          Sql::Reconcile.new(self, @db).reconcile!
+        end
+
+        @reconcile_complete = true
+      end
+
+      # Mark that a model changed and we need to rerun reconcile next time the
+      # db is accessed.
+      def invalidate_reconcile!
+        @reconcile_complete = false
+      end
+
+      # Used when creating a class that you don't want to reconcile after
+      def skip_reconcile
+        @skip_reconcile = true
+
+        begin
+          yield
+        ensure
+          @skip_reconcile = false
+        end
+      end
+
+      # Called when the db gets reset (from specs usually)
+      def reset!
+        Sql::Reconcile.new(self, @db).reset!
+      end
+
+      def insert(collection, values)
+        values = pack_values(collection, values)
+
+        db.from(collection).insert(values)
+      end
+
+      def update(collection, values)
+        values = pack_values(collection, values)
+
+        # Find the original so we can update it
+        table = db.from(collection)
+
+        # TODO: we should move this to a real upsert
+        begin
+          table.insert(values)
+          log(table.insert_sql(values))
+        rescue Sequel::UniqueConstraintViolation => e
+          # Already a record, update
+          id = values[:id]
+          log(table.where(id: id).update_sql(values))
+          table.where(id: id).update(values)
+        end
+
+        nil
+      end
+
+      def query(collection, query)
+        allowed_methods = %w(where where_with_block offset limit count)
+
+        result = db.from(collection.to_sym)
+
+        query.each do |query_part|
+          method_name, *args = query_part
+
+          unless allowed_methods.include?(method_name.to_s)
+            fail "`#{method_name}` is not part of a valid query"
+          end
+
+          # Symbolize Keys
+          args = args.map do |arg|
+            if arg.is_a?(Hash)
+              arg = self.class.nested_symbolize_keys(arg)
+            end
+            arg
+          end
+
+          if method_name == :where_with_block
+            # Where calls with block are handled differently.  We have to replay
+            # the query that was captured on the client with QueryIdentifier
+
+            # Grab the AST that was generated from the block call on the client.
+            block_ast = args.pop
+            args = self.class.move_to_db_types(args)
+
+            result = result.where(*args) do |ident|
+              Sql::WhereCall.new(ident).call(block_ast)
+            end
+          else
+            args = self.class.move_to_db_types(args)
+            result = result.send(method_name, *args)
+          end
+        end
+
+        if result.respond_to?(:all)
+          log(result.sql)
+          result = result.all.map do |hash|
+            # Volt expects symbol keys
+            hash.symbolize_keys
+          end#.tap {|v| puts "QUERY: " + v.inspect }
+
+          # Unpack extra values
+          result = unpack_values!(result)
+        end
+
+        result
+      end
+
+      def delete(collection, query)
+        query = self.class.move_to_db_types(query)
+        db.from(collection).where(query).delete
+      end
+
+      # remove the collection entirely
+      def drop_collection(collection)
+        db.drop_collection(collection)
+      end
+
+      def drop_database
+        RootModels.clear_temporary
+
+        # Drop all tables
+        db(true).drop_table(*db.tables)
+
+        RootModels.model_classes.each do |model_klass|
+          model_klass.reconciled = false
+        end
+
+        invalidate_reconcile!
+      end
+
+
+      def self.move_to_db_types(values)
+        values = nested_symbolize_keys(values)
+
+        values = Volt::DataTransformer.transform(values, false) do |value|
+          if defined?(VoltTime) && value.is_a?(VoltTime)
+            value.to_time
+          elsif value.is_a?(Symbol)
+            # Symbols get turned into strings
+            value.to_s
+          else
+            value
+          end
+        end
+
+        values
+      end
+
+      private
+
+      def self.nested_symbolize_keys(values)
+        DataTransformer.transform_keys(values) do |key|
+          key.to_sym
+        end
+      end
+
+      # Take the values and symbolize them, and also remove any values that
+      # aren't going in fields and move them into extra.
+      #
+      # Then change VoltTime's to Time for Sequel
+      def pack_values(collection, values)
+        values = self.class.move_to_db_types(values)
+
+        klass = Volt::Model.class_at_path([collection])
+        # Find any fields in values that aren't defined with a ```field```,
+        # and put them into extra.
+        extra = {}
+        values = values.select do |key, value|
+          if klass.fields[key] || key == :id
+            # field is defined, keep in values
+            true
+          else
+            # field does not exist, move to extra
+            extra[key] = value
+
+            false
+          end
+        end
+
+        # Copy the extras into the values
+        values[:extra] = serialize(extra) if extra.present?
+
+        # Volt.logger.info("Insert into #{collection}: #{values.inspect}")
+
+        values
+      end
+
+      # Loop through the inputs array and change values in place to be unpacked.
+      # Unpacking means moving the extra field out and into the main fields.
+      #
+      # Then transform Time to VoltTime
+      def unpack_values!(inputs)
+        values = inputs.each do |values|
+          extra = values.delete(:extra)
+
+          if extra
+            extra = deserialize(extra)
+            self.class.nested_symbolize_keys(extra.to_h).each_pair do |key, new_value|
+              unless values[key]
+                values[key] = new_value
+              end
+            end
+          end
+        end
+
+        values = Volt::DataTransformer.transform(values) do |value|
+          if defined?(VoltTime) && value.is_a?(Time)
+            value = VoltTime.from_time(value)
+          else
+            value
+          end
+        end
+
+        values
+      end
+
+    end
+
+
+    # Specific adaptors for each database
+    class PostgresAdaptorServer < SqlAdaptorServer
+      def serialize(extra)
+        Sequel.pg_json(extra)
+      end
+
+      def deserialize(extra)
+        extra
+      end
+    end
+
+    class SqliteAdaptorServer < SqlAdaptorServer
+      def serialize(extra)
+        JSON.dump(extra)
+      end
+
+      def deserialize(extra)
+        JSON.parse(extra)
+      end
+    end
+
+    class MysqlAdaptorServer < SqlAdaptorServer
+    end
+  end
+end

data/app/sql/lib/sql_logger.rb

@@ -0,0 +1,13 @@
+module Volt
+  module Sql
+    module SqlLogger
+
+      def log(msg)
+        if ENV['LOG_SQL']
+          Volt.logger.log_with_color(msg, :cyan)
+        end
+      end
+
+    end
+  end
+end

data/app/sql/lib/store_persistor.rb

@@ -0,0 +1,10 @@
+# Reopen the store class and tell it not to allow "on the fly" collections.
+module Volt
+  module Persistors
+    class Store
+      def on_the_fly_collections?
+        false
+      end
+    end
+  end
+end

data/app/sql/lib/table_reconcile.rb

@@ -0,0 +1,123 @@
+# Table Reconcile is responsible for migrating a table from the database state
+# to the model state.
+require 'sql/lib/sql_logger'
+require 'sql/lib/field_updater'
+require 'sql/lib/index_updater'
+
+module Volt
+  module Sql
+    class TableReconcile
+      include SqlLogger
+
+      attr_reader :field_updater
+
+      def initialize(adaptor, db, model_class)
+        @model_class = model_class
+        @adaptor = adaptor
+        @db = db
+        @field_updater = FieldUpdater.new(@db, self)
+      end
+
+      def run
+        table_name = @model_class.collection_name
+
+        ensure_table(table_name)
+
+        update_fields(@model_class, table_name)
+
+        IndexUpdater.new(@db, @model_class, table_name)
+
+        @model_class.reconciled = true
+      end
+
+      # Create an empty table if one does not exist
+      def ensure_table(table_name)
+        # Check if table exists
+        if !@db.tables || !@db.tables.include?(table_name)
+          log("Creating Table #{table_name}")
+          adaptor_name = @adaptor.adaptor_name
+          @db.create_table(table_name) do
+            # guid id
+            column :id, String, :unique => true, :null => false, :primary_key => true
+
+            # When using underscore notation on a field that does not exist, the
+            # data will be stored in extra.
+            if adaptor_name == 'postgres'
+              # Use jsonb
+              column :extra, 'json'
+            else
+              column :extra, String
+            end
+          end
+          # TODO: there's some issue with @db.schema and no clue why, but I
+          # have to run this again to get .schema to work later.
+          @db.tables
+        end
+      end
+
+
+      def sequel_class_and_opts_from_db(db_field)
+        vclasses, vopts = Helper.klasses_and_options_from_db(db_field)
+        return Helper.column_type_and_options_for_sequel(vclasses, vopts)
+      end
+
+      # Pulls the db_fields out of sequel
+      def db_fields_for_table(table_name)
+        db_fields = {}
+        result = @db.schema(table_name).each do |col|
+          db_fields[col[0].to_sym] = col[1]
+        end
+
+        db_fields
+      end
+
+      def update_fields(model_class, table_name)
+        if (fields = model_class.fields)
+          db_fields = db_fields_for_table(table_name)
+
+          db_fields.delete(:id)
+          db_fields.delete(:extra)
+
+          orphan_fields = db_fields.keys - fields.keys
+          new_fields = fields.keys - db_fields.keys
+
+          # If a single field was renamed, we can auto-migrate
+          if (orphan_fields.size == 1 && new_fields.size == 1)
+            from_name = orphan_fields[0]
+            to_name = new_fields[0]
+            auto_migrate_field_rename(table_name, from_name, to_name)
+
+            # Move in start fields
+            db_fields[to_name] = db_fields.delete(from_name)
+          end
+
+          if new_fields.size == 0 && orphan_fields.size > 0
+            # one or more fields were removed
+            orphan_fields.each do |field_name|
+              @field_updater.auto_migrate_remove_field(table_name, field_name, db_fields[field_name])
+            end
+          end
+
+
+          if orphan_fields.size == 0
+
+            fields.each do |name, klass_and_opts|
+              name = name.to_sym
+              klasses, opts = klass_and_opts
+
+              # Get the original state for the field
+              db_field = db_fields.delete(name)
+
+              @field_updater.update_field(model_class, table_name, db_field, name, klasses, opts)
+            end
+
+            # remove any fields we didn't see in the models
+          end
+        else
+          # Either >1 orphaned fields, or more than one new fields
+          raise "Could not auto migrate #{table_name}"
+        end
+      end
+    end
+  end
+end

data/app/sql/lib/where_call.rb

@@ -0,0 +1,64 @@
+# Queries on the client side can be captured using block syntax.  On the client
+# a Volt::QueryIdentifier is passed to the block.  It will create a AST for
+# queries which is sent.
+#
+# Sql::WhereCall can reply the query to the underlying database engine (in
+# this case, sequel)
+
+module Volt
+  module Sql
+    class WhereCall
+      VALID_METHODS = ['&', '|', '~', '>', '<', '>=', '<=' , '=~', '!~']
+      def initialize(ident)
+        @ident = ident
+      end
+
+      def call(ast)
+        walk(ast)
+      end
+
+      def walk(ast)
+        if ast.is_a?(Array) && !ast.is_a?(Sequel::SQL::Identifier)
+          op = ast.shift
+
+          case op
+          when 'c'
+            return op_call(*ast)
+          when 'a'
+            # We popped off the 'a', so we just return the array
+            return ast
+          else
+            raise "invalid op: #{op.inspect} - #{ast.inspect} - #{ast.is_a?(Array).inspect}"
+          end
+        else
+          # Not an operation, return
+          return ast
+        end
+      end
+
+      def op_call(self_obj, method_name, *args)
+        if self_obj == 'ident'
+          self_obj = @ident
+        end
+
+        # walk on the self obj
+        self_obj = walk(self_obj)
+
+        # Method name security checks
+        case method_name
+        when 'send'
+          raise "Send is not supported in queries"
+        end
+
+        if method_name !~ /^[a-zA-Z0-9_]+$/ && !VALID_METHODS.include?(method_name)
+          raise "Only method names matching /[a-zA-Z0-9_]/ are allowed from client side queries (called `#{method_name}`)"
+        end
+
+        walked_args = args.map {|arg| walk(arg) }
+
+        # We have to use __send__ because send is handled differently
+        self_obj.__send__(method_name, *walked_args)
+      end
+    end
+  end
+end

data/app/sql/views/main/index.html

@@ -0,0 +1,2 @@
+<:Body>
+-- component --

data/lib/volt/sql/version.rb

@@ -0,0 +1,5 @@
+module Volt
+  module Sql
+    VERSION = '0.0.1'
+  end
+end

data/lib/volt/sql.rb

@@ -0,0 +1,27 @@
+# If you need to require in code in the gem's app folder, keep in mind that
+# the app is not on the load path when the gem is required.  Use
+# app/{gemname}/config/initializers/boot.rb to require in client or server
+# code.
+#
+# Also, in volt apps, you typically use the lib folder in the
+# app/{componentname} folder instead of this lib folder.  This lib folder is
+# for setting up gem code when Bundler.require is called. (or the gem is
+# required.)
+#
+# If you need to configure volt in some way, you can add a Volt.configure block
+# in this file.
+
+
+Volt.configure do |config|
+  # Set the datastore to sql
+  config.public.datastore_name = 'sql'
+
+  # Include the sql component on the client
+  config.default_components << 'sql'
+end
+
+module Volt
+  module Sql
+    # Your code goes here...
+  end
+end