rasti-db 1.5.0 → 2.3.0

data/lib/rasti/db/nql/syntax.treetop

@@ -42,44 +42,49 @@ module Rasti
           comparison_equal
         end
 
-        rule field
-          _tables:(table:field_name '.')* _column:field_name <Nodes::Field>
+        rule attribute
+          _tables:(table:attribute_name '.')* _column:attribute_name <Nodes::Attribute>
         end
 
         rule comparison_include
-          field:field space* comparator:':' space* argument:basic <Nodes::Comparisons::Include>
+          attribute:attribute space* comparator:':' space* argument:argument <Nodes::Comparisons::Include>
         end
 
         rule comparison_not_include
-          field:field space* comparator:'!:' space* argument:basic <Nodes::Comparisons::NotInclude>
+          attribute:attribute space* comparator:'!:' space* argument:argument <Nodes::Comparisons::NotInclude>
         end
 
         rule comparison_like
-          field:field space* comparator:'~' space* argument:basic <Nodes::Comparisons::Like>
+          attribute:attribute space* comparator:'~' space* argument:argument <Nodes::Comparisons::Like>
         end
 
         rule comparison_greater_than
-          field:field space* comparator:'>' space* argument:basic <Nodes::Comparisons::GreaterThan>
+          attribute:attribute space* comparator:'>' space* argument:argument <Nodes::Comparisons::GreaterThan>
         end
 
         rule comparison_greater_than_or_equal
-          field:field space* comparator:'>=' space* argument:basic <Nodes::Comparisons::GreaterThanOrEqual>
+          attribute:attribute space* comparator:'>=' space* argument:argument <Nodes::Comparisons::GreaterThanOrEqual>
         end
 
         rule comparison_less_than
-          field:field space* comparator:'<' space* argument:basic <Nodes::Comparisons::LessThan>
+          attribute:attribute space* comparator:'<' space* argument:argument <Nodes::Comparisons::LessThan>
         end
 
         rule comparison_less_than_or_equal
-          field:field space* comparator:'<=' space* argument:basic <Nodes::Comparisons::LessThanOrEqual>
+          attribute:attribute space* comparator:'<=' space* argument:argument <Nodes::Comparisons::LessThanOrEqual>
         end
 
         rule comparison_not_equal
-          field:field space* comparator:'!=' space* argument:basic <Nodes::Comparisons::NotEqual>
+          attribute:attribute space* comparator:'!=' space* argument:argument <Nodes::Comparisons::NotEqual>
         end
 
         rule comparison_equal
-          field:field space* comparator:'=' space* argument:basic <Nodes::Comparisons::Equal>
+          attribute:attribute space* comparator:'=' space* argument:argument <Nodes::Comparisons::Equal>
+        end
+
+        rule argument
+          array /
+          basic
         end
 
         rule basic
@@ -91,11 +96,19 @@ module Rasti
           string
         end
 
+        rule array
+          open:'[' space* contents:(array_content / basic) space* close:']' <Nodes::Constants::Array>
+        end
+
+        rule array_content
+          left:basic space* ',' space* right:(array_content / basic) <Nodes::ArrayContent>
+        end
+
         rule space
           [\s\t\n]
         end
 
-        rule field_name
+        rule attribute_name
           [a-z_]+
         end
 
@@ -130,7 +143,7 @@ module Rasti
         end
 
         rule valid_character 
-          [0-9a-zA-ZÁÀÄÂÃÅĀĂǍáàäâãåāăǎÉÈËÊĒĔĖĚéèëêēĕėěÍÌÏÎĨĬǏíìïîĩĭǐÓÒÖÔÕŌŎŐǑóòöôõōŏőǒÚÙÜÛŨŪŬŮŰǓúùüûũūŭůűǔÑñçÇ%@#+-_'?!$*/\s]
+          [0-9a-zA-ZÁÀÄÂÃÅĀĂǍáàäâãåāăǎÉÈËÊĒĔĖĚéèëêēĕėěÍÌÏÎĨĬǏíìïîĩĭǐÓÒÖÔÕŌŎŐǑóòöôõōŏőǒÚÙÜÛŨŪŬŮŰǓúùüûũūŭůűǔÑñçÇ%@#+\--Z\\^_'?!$*/\s]
         end
 
         rule boolean
@@ -159,7 +172,7 @@ module Rasti
         end
 
         rule reserved_character
-          [&|.():!=<>~]
+          [&|.():!=<>~,\]\[]
         end
 
       end

data/lib/rasti/db/query.rb

@@ -5,13 +5,12 @@ module Rasti
       DATASET_CHAINED_METHODS = [:where, :exclude, :or, :order, :reverse_order, :limit, :offset].freeze
 
       include Enumerable
-      include Helpers::WithSchema
 
-      def initialize(collection_class:, dataset:, relations_graph:nil, schema:nil)
+      def initialize(environment:, collection_class:, dataset:, relations_graph:nil)
+        @environment = environment
         @collection_class = collection_class
-        @dataset = dataset
-        @relations_graph = relations_graph || Relations::Graph.new(dataset.db, schema, collection_class)
-        @schema = schema
+        @dataset = dataset.qualify collection_class.collection_name
+        @relations_graph = relations_graph || Relations::Graph.new(environment, collection_class)
       end
 
       DATASET_CHAINED_METHODS.each do |method|
@@ -58,15 +57,36 @@ module Rasti
         build_query relations_graph: relations_graph.with_all_attributes_for(relations)
       end
 
+      def select_computed_attributes(*computed_attributes)
+        ds = computed_attributes.inject(dataset) do |ds, name|
+          computed_attribute = collection_class.computed_attributes[name]
+          computed_attribute.apply_join(ds).select_append(computed_attribute.identifier.as(name))
+        end
+        build_query dataset: ds
+      end
+
       def all
-        with_graph(dataset.all).map do |row| 
+        with_graph(dataset.all).map do |row|
           collection_class.model.new row
         end
       end
       alias_method :to_a, :all
 
-      def each(&block)
-        all.each(&block)
+      def each(batch_size:nil, &block)
+        if batch_size.nil?
+          all.each(&block)
+        else
+          each_batch(size: batch_size) do |models|
+            models.each { |model| block.call model }
+          end
+        end
+      end
+
+      def each_batch(size:, &block)
+        primary_keys.each_slice(size) do |pks|
+          query = where(collection_class.primary_key => pks)
+          block.call query.all
+        end
       end
 
       def graph(*relations)
@@ -74,12 +94,10 @@ module Rasti
       end
 
       def join(*relations)
-        graph = Relations::Graph.new dataset.db, schema, collection_class, relations
-        
-        ds = graph.add_joins(dataset)
-                  .distinct
-                  .select_all(collection_class.collection_name)
-        
+        graph = Relations::Graph.new environment, collection_class, relations
+
+        ds = graph.add_joins(dataset).distinct
+
         build_query dataset: ds
       end
 
@@ -97,12 +115,12 @@ module Rasti
 
       def first
         row = dataset.first
-        row ? collection_class.model.new(with_graph(row)) : nil
+        row ? build_model(row) : nil
       end
 
       def last
         row = dataset.last
-        row ? collection_class.model.new(with_graph(row)) : nil
+        row ? build_model(row) : nil
       end
 
       def detect(*args, &block)
@@ -119,33 +137,42 @@ module Rasti
 
         raise NQL::InvalidExpressionError.new(filter_expression) if sentence.nil?
 
+        ds = sentence.computed_attributes(collection_class).inject(dataset) do |ds, name|
+          collection_class.computed_attributes[name].apply_join ds
+        end
+        query = build_query dataset: ds
+
         dependency_tables = sentence.dependency_tables
-        query = dependency_tables.empty? ? self : join(*dependency_tables)
-        
-        query.where sentence.filter_condition
+        query = query.join(*dependency_tables) unless dependency_tables.empty?
+
+        query.where sentence.filter_condition(collection_class)
       end
 
       private
 
-      attr_reader :collection_class, :dataset, :relations_graph, :schema
+      attr_reader :environment, :collection_class, :dataset, :relations_graph
 
       def build_query(**args)
         current_args = {
+          environment: environment,
           collection_class: collection_class,
           dataset: dataset,
-          relations_graph: relations_graph,
-          schema: schema
+          relations_graph: relations_graph
         }
 
         Query.new(**current_args.merge(args))
       end
 
+      def build_model(row)
+        collection_class.model.new with_graph(row)
+      end
+
       def chainable(&block)
         build_query dataset: instance_eval(&block)
       end
 
       def with_related(relation_name, primary_keys)
-        ds = collection_class.relations[relation_name].apply_filter dataset, schema, primary_keys
+        ds = collection_class.relations[relation_name].apply_filter environment, dataset, primary_keys
         build_query dataset: ds
       end
 
@@ -155,6 +182,11 @@ module Rasti
         data
       end
 
+      def qualify(collection_name, data_source_name: nil)
+        data_source_name ||= collection_class.data_source_name
+        environment.qualify data_source_name, collection_name
+      end
+
       def nql_parser
         NQL::SyntaxParser.new
       end

data/lib/rasti/db/relations/base.rb

@@ -33,25 +33,39 @@ module Rasti
           self.class == OneToOne
         end
 
-        def join_relation_name(prefix)
-          with_prefix prefix, name
+        def from_one?
+          one_to_one? || one_to_many?
         end
 
-        private
+        def from_many?
+          many_to_one? || many_to_many?
+        end
 
-        attr_reader :options
+        def to_one?
+          one_to_one? || many_to_one?
+        end
 
-        def qualified_source_collection_name(schema=nil)
-          schema.nil? ? Sequel[source_collection_class.collection_name] : Sequel[schema][source_collection_class.collection_name]
+        def to_many?
+          one_to_many? || many_to_many?
         end
 
-        def qualified_target_collection_name(schema=nil)
-          schema.nil? ? Sequel[target_collection_class.collection_name] : Sequel[schema][target_collection_class.collection_name]
+        def join_relation_name(prefix)
+          with_prefix prefix, name
         end
 
+        private
+
+        attr_reader :options
+
         def with_prefix(prefix, name)
           [prefix, name].compact.join('__').to_sym
         end
+
+        def validate_join!
+          if source_collection_class.data_source_name != target_collection_class.data_source_name
+            raise "Invalid join of multiple data sources: #{source_collection_class.data_source_name}.#{source_collection_class.collection_name} > #{target_collection_class.data_source_name}.#{target_collection_class.collection_name}" 
+          end
+        end
         
       end
     end

data/lib/rasti/db/relations/graph.rb

@@ -3,9 +3,8 @@ module Rasti
     module Relations
       class Graph
 
-        def initialize(db, schema, collection_class, relations=[], selected_attributes={}, excluded_attributes={})
-          @db = db
-          @schema = schema
+        def initialize(environment, collection_class, relations=[], selected_attributes={}, excluded_attributes={})
+          @environment = environment
           @collection_class = collection_class
           @graph = build_graph relations, 
                                Hash::Indifferent.new(selected_attributes), 
@@ -13,8 +12,7 @@ module Rasti
         end
 
         def merge(relations:[], selected_attributes:{}, excluded_attributes:{})
-          Graph.new db, 
-                    schema, 
+          Graph.new environment, 
                     collection_class, 
                     (flat_relations | relations), 
                     flat_selected_attributes.merge(selected_attributes),
@@ -38,9 +36,8 @@ module Rasti
           return if rows.empty?
 
           graph.roots.each do |node|
-            relation_of(node).fetch_graph rows, 
-                                          db, 
-                                          schema, 
+            relation_of(node).fetch_graph environment,
+                                          rows, 
                                           node[:selected_attributes],
                                           node[:excluded_attributes] ,
                                           subgraph_of(node)
@@ -48,18 +45,16 @@ module Rasti
         end
 
         def add_joins(dataset, prefix=nil)
-          graph.roots.each do |node|
+          graph.roots.inject(dataset) do |ds, node|
             relation = relation_of node
-            dataset = relation.add_join dataset, schema, prefix
-            dataset = subgraph_of(node).add_joins dataset, relation.join_relation_name(prefix)
+            dataset_with_relation = relation.add_join environment, ds, prefix
+            subgraph_of(node).add_joins dataset_with_relation, relation.join_relation_name(prefix)
           end
-
-          dataset
         end
 
         private
 
-        attr_reader :db, :schema, :collection_class, :graph
+        attr_reader :environment, :collection_class, :graph
 
         def relation_of(node)
           collection_class.relations.fetch(node[:name])
@@ -93,8 +88,7 @@ module Rasti
             excluded[id] = descendant[:excluded_attributes]
           end
 
-          Graph.new db, 
-                    schema, 
+          Graph.new environment, 
                     relation_of(node).target_collection_class, 
                     relations, 
                     selected, 

data/lib/rasti/db/relations/many_to_many.rb

@@ -15,23 +15,42 @@ module Rasti
           @relation_collection_name ||= options[:relation_collection_name] || [source_collection_class.collection_name, target_collection_class.collection_name].sort.join('_').to_sym
         end
 
-        def qualified_relation_collection_name(schema=nil)
-          schema.nil? ? Sequel[relation_collection_name] : Sequel[schema][relation_collection_name]
+        def relation_data_source_name
+          @relation_data_source_name ||= options[:relation_data_source_name] || source_collection_class.data_source_name
         end
 
-        def fetch_graph(rows, db, schema=nil, selected_attributes=nil, excluded_attributes=nil, relations_graph=nil)
+        def fetch_graph(environment, rows, selected_attributes=nil, excluded_attributes=nil, relations_graph=nil)
           pks = rows.map { |row| row[source_collection_class.primary_key] }
 
-          target_collection = target_collection_class.new db, schema
+          if target_collection_class.data_source_name == relation_data_source_name
+            target_data_source = environment.data_source_of target_collection_class
 
-          relation_name = qualified_relation_collection_name schema
+            dataset = target_data_source.db.from(environment.qualify_collection(target_collection_class))
+                                           .join(qualified_relation_collection_name(environment), target_foreign_key => target_collection_class.primary_key)
+                                           .where(Sequel[relation_collection_name][source_foreign_key] => pks)
+                                           .select_all(target_collection_class.collection_name)
 
-          join_rows = target_collection.dataset
-                                       .join(relation_name, target_foreign_key => target_collection_class.primary_key)
-                                       .where(Sequel[relation_name][source_foreign_key] => pks)
-                                       .select_all(target_collection_class.collection_name)
-                                       .select_append(Sequel[relation_name][source_foreign_key].as(:source_foreign_key))
-                                       .all
+            selected_attributes ||= target_collection_class.collection_attributes - excluded_attributes if excluded_attributes
+            dataset = dataset.select(*selected_attributes.map { |a| Sequel[target_collection_class.collection_name][a] }) if selected_attributes
+
+            join_rows = dataset.select_append(Sequel[relation_collection_name][source_foreign_key].as(:source_foreign_key)).all
+          else
+            relation_data_source = environment.data_source relation_data_source_name
+
+            relation_index = relation_data_source.db.from(relation_data_source.qualify(relation_collection_name))
+                                                    .where(source_foreign_key => pks)
+                                                    .select_hash_groups(target_foreign_key, source_foreign_key)
+
+            query = target_collection_class.new environment
+            query = query.exclude_attributes(*excluded_attributes) if excluded_attributes
+            query = query.select_attributes(*selected_attributes) if selected_attributes
+
+            join_rows = query.where(target_collection_class.primary_key => relation_index.keys).raw.flat_map do |row|
+              relation_index[row[target_collection_class.primary_key]].map do |source_primary_key|
+                row.merge(source_foreign_key: source_primary_key)
+              end
+            end
+          end
 
           relations_graph.fetch_graph join_rows if relations_graph
 
@@ -41,17 +60,19 @@ module Rasti
           end
 
           rows.each do |row| 
-            row[name] = relation_rows.fetch row[target_collection_class.primary_key], []
+            row[name] = relation_rows.fetch row[source_collection_class.primary_key], []
           end
         end
 
-        def add_join(dataset, schema=nil, prefix=nil)
+        def add_join(environment, dataset, prefix=nil)
+          validate_join!
+          
           many_to_many_relation_alias = with_prefix prefix, "#{relation_collection_name}_#{SecureRandom.base64}"
 
-          qualified_relation_source = prefix ? Sequel[prefix] : qualified_source_collection_name(schema)
+          relation_name = prefix ? Sequel[prefix] : Sequel[source_collection_class.collection_name]
 
           many_to_many_condition = {
-            Sequel[many_to_many_relation_alias][source_foreign_key] => qualified_relation_source[source_collection_class.primary_key]
+            Sequel[many_to_many_relation_alias][source_foreign_key] => relation_name[source_collection_class.primary_key]
           }
 
           relation_alias = join_relation_name prefix
@@ -60,17 +81,30 @@ module Rasti
             Sequel[relation_alias][target_collection_class.primary_key] => Sequel[many_to_many_relation_alias][target_foreign_key]
           }
 
-          dataset.join(qualified_relation_collection_name(schema).as(many_to_many_relation_alias), many_to_many_condition)
-                 .join(qualified_target_collection_name(schema).as(relation_alias), relation_condition)
+          dataset.join(qualified_relation_collection_name(environment).as(many_to_many_relation_alias), many_to_many_condition)
+                 .join(environment.qualify_collection(target_collection_class).as(relation_alias), relation_condition)
+        end
+
+        def apply_filter(environment, dataset, primary_keys)
+          if source_collection_class.data_source_name == relation_data_source_name
+            dataset.join(qualified_relation_collection_name(environment), source_foreign_key => source_collection_class.primary_key)
+                   .where(Sequel[relation_collection_name][target_foreign_key] => primary_keys)
+                   .select_all(source_collection_class.collection_name)
+                   .distinct
+          else
+            data_source = environment.data_source relation_data_source_name
+            fks = data_source.db.from(data_source.qualify(relation_collection_name))
+                                .where(target_collection_class.foreign_key => primary_keys)
+                                .select_map(source_collection_class.foreign_key)
+                                .uniq
+            dataset.where(source_collection_class.primary_key => fks)
+          end
         end
 
-        def apply_filter(dataset, schema=nil, primary_keys=[])
-          relation_name = qualified_relation_collection_name schema
+        private
 
-          dataset.join(relation_name, source_foreign_key => target_collection_class.primary_key)
-                 .where(Sequel[relation_name][target_foreign_key] => primary_keys)
-                 .select_all(source_collection_class.collection_name)
-                 .distinct
+        def qualified_relation_collection_name(environment)
+          environment.qualify relation_data_source_name, relation_collection_name
         end
 
       end