conceptql 0.1.1 → 0.2.0

data/lib/conceptql/{nodes → operators}/sum.rb

@@ -1,20 +1,25 @@
 require_relative 'pass_thru'
 
 module ConceptQL
-  module Nodes
+  module Operators
     class Sum < PassThru
+      desc <<-EOF
+Sums value_as_number across all results that match on all but start_date, end_date.
+For start_date and end_date the min and max of each respectively is returned.'
+      EOF
+
       def query(db)
         db.from(unioned(db))
-          .select_group(*(COLUMNS - [:start_date, :end_date, :criterion_id, :value_as_numeric]))
+          .select_group(*(COLUMNS - [:start_date, :end_date, :criterion_id, :value_as_number]))
           .select_append(Sequel.lit('?', 0).as(:criterion_id))
           .select_append{ min(start_date).as(:start_date) }
           .select_append{ max(end_date).as(:end_date) }
-          .select_append{sum(value_as_numeric).as(:value_as_numeric)}
+          .select_append{sum(value_as_number).as(:value_as_number)}
           .from_self
       end
 
       def unioned(db)
-        children.map { |c| c.evaluate(db) }.inject do |uni, q|
+        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
           uni.union(q)
         end
       end

data/lib/conceptql/{nodes/temporal_node.rb → operators/temporal_operator.rb}

@@ -1,13 +1,16 @@
-require_relative 'binary_operator_node'
+require_relative 'binary_operator_operator'
 
 module ConceptQL
-  module Nodes
-    # Base class for all temporal nodes
+  module Operators
+    # Base class for all temporal operators
     #
     # Subclasses must implement the where_clause method which should probably return
     # a proc that can be executed as a Sequel "virtual row" e.g.
     # Proc.new { l.end_date < r.start_date }
-    class TemporalNode < BinaryOperatorNode
+    class TemporalOperator < BinaryOperatorOperator
+      reset_categories
+      category %w(Temporal Relative)
+
       def query(db)
         db.from(db.from(left_stream(db))
                   .join(right_stream(db), l__person_id: :r__person_id)

data/lib/conceptql/{nodes → operators}/time_window.rb

@@ -1,8 +1,8 @@
-require_relative 'node'
+require_relative 'operator'
 require_relative '../date_adjuster'
 
 module ConceptQL
-  module Nodes
+  module Operators
     # A TimeWindow adjusts the start_date and end_date the incoming stream by the values specified in
     # the start and end arguments.
     #
@@ -19,7 +19,13 @@ module ConceptQL
     # Both start and end arguments must be provided, but if you do not wish to adjust a date just
     # pass '', '0', or nil as that argument.  E.g.:
     # start: 'd', end: '' # Only adjust start_date by positive 1 day and leave end_date uneffected
-    class TimeWindow < Node
+    class TimeWindow < Operator
+      desc 'Adjusts the start_date and end_date columns to create a new window of time for each result.'
+      option :start, type: :string
+      option :end, type: :string
+      allows_one_upstream
+      category %(Temporal Manipulation)
+
       def query(db)
         db.extension :date_arithmetic
         db.from(stream.evaluate(db))
@@ -38,6 +44,10 @@ module ConceptQL
         adjusted_date(:end, :end_date)
       end
 
+      def manipulator
+        @manipulator ||= options.fetch(:date_manipulator, Sequel)
+      end
+
       # NOTE: This produces PostgreSQL-specific date adjustment.  I'm not yet certain how to generalize this
       # or make different versions based on RDBMS
       def adjusted_date(option_arg, column)
@@ -45,13 +55,15 @@ module ConceptQL
         arg ||= ''
         return ['end_date', column].join('___').to_sym if arg.downcase == 'end'
         return ['start_date', column].join('___').to_sym if arg.downcase == 'start'
-        DateAdjuster.new(arg).adjustments.inject(Sequel.expr(column)) do |sql, (units, quantity)|
+        return Sequel.cast(Date.parse(arg).strftime('%Y-%m-%d'), Date).as(column) if arg =~ /^\d{4}-\d{2}-\d{2}$/
+        adjusted_date = DateAdjuster.new(arg).adjustments.inject(Sequel.expr(column)) do |sql, (units, quantity)|
           if quantity > 0
-            Sequel.date_add(sql, units => quantity)
+            manipulator.date_add(sql, units => quantity)
           else
-            Sequel.date_sub(sql, units => quantity.abs)
+            manipulator.date_sub(sql, units => quantity.abs)
           end
-        end.as(column)
+        end
+        Sequel.cast(adjusted_date, Date).as(column)
       end
     end
   end

data/lib/conceptql/operators/to_seer_visits.rb

@@ -0,0 +1,24 @@
+require_relative 'operator'
+
+module ConceptQL
+  module Operators
+    class ToSeerVisits < Operator
+      def type
+        :visit_occurrence
+      end
+
+      def query(db)
+        query = options.map do |k, v|
+          next if v.nil?
+          db[:observation]
+            .where(observation_source_value: k.to_s.upcase, value_as_string: v)
+            .select(:visit_occurrence_id)
+            .from_self
+        end.compact.inject { |q, i| i.intersect(q) }
+        db[:visit_occurrence].where(visit_occurrence_id: query)
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/operators/trim_date_end.rb

@@ -0,0 +1,55 @@
+require_relative 'temporal_operator'
+
+module ConceptQL
+  module Operators
+    # Trims the end_date of the LHS set of results by the RHS's earliest
+    # start_date (per person)
+    # If a the RHS contains a start_date that comes before the LHS's start_date
+    # that LHS result is completely discarded.
+    #
+    # If there is no RHS result for an LHS result, the LHS result is passed
+    # thru unaffected.
+    #
+    # If the RHS result's start_date is later than the LHS end_date, the LHS
+    # result is passed thru unaffected.
+    class TrimDateEnd < TemporalOperator
+      desc <<-EOF
+Trims the end_date of the LHS set of results by the RHS's earliest
+start_date (per person)
+If a the RHS contains a start_date that comes before the LHS's start_date
+that LHS result is completely discarded.
+
+If there is no RHS result for an LHS result, the LHS result is passed
+thru unaffected.
+
+If the RHS result's start_date is later than the LHS end_date, the LHS
+result is passed thru unaffected.
+      EOF
+      allows_one_upstream
+      category %(Temporal Manipulation)
+
+      def query(db)
+        grouped_right = db.from(right_stream(db)).select_group(:person_id).select_append(Sequel.as(Sequel.function(:min, :start_date), :start_date))
+
+        where_criteria = Sequel.expr { l__start_date <= r__start_date }
+        where_criteria = where_criteria.|(r__start_date: nil)
+
+        # If the RHS's min start date is less than the LHS start date,
+        # the entire LHS date range is truncated, which implies the row itself
+        # is ineligible to pass thru
+        db.from(db.from(left_stream(db))
+                  .left_join(Sequel.as(grouped_right, :r), l__person_id: :r__person_id)
+                  .where(where_criteria)
+                  .select(*new_columns)
+                  .select_append(Sequel.as(Sequel.function(:least, :l__end_date, :r__start_date), :end_date))
+               )
+      end
+
+      private
+      def new_columns
+        (COLUMNS - [:end_date]).map { |col| "l__#{col}".to_sym }
+      end
+    end
+  end
+end
+

data/lib/conceptql/operators/trim_date_start.rb

@@ -0,0 +1,56 @@
+require_relative 'temporal_operator'
+
+module ConceptQL
+  module Operators
+    # Trims the start_date of the LHS set of results by the RHS's latest
+    # end_date (per person)
+    # If a the RHS contains an end_date that comes after the LHS's end_date
+    # that LHS result is completely discarded.
+    #
+    # If there is no RHS result for an LHS result, the LHS result is passed
+    # thru unaffected.
+    #
+    # If the RHS result's end_date is earlier than the LHS start_date, the LHS
+    # result is passed thru unaffected.
+    class TrimDateStart < TemporalOperator
+      desc <<-EOF
+Trims the start_date of the LHS set of results by the RHS's latest
+end_date (per person)
+If a the RHS contains an end_date that comes after the LHS's end_date
+that LHS result is completely discarded.
+
+If there is no RHS result for an LHS result, the LHS result is passed
+thru unaffected.
+
+If the RHS result's end_date is earlier than the LHS start_date, the LHS
+result is passed thru unaffected.
+      EOF
+      allows_one_upstream
+      category %(Temporal Manipulation)
+
+      def query(db)
+        grouped_right = db.from(right_stream(db)).select_group(:person_id).select_append(Sequel.as(Sequel.function(:max, :end_date), :end_date))
+
+        where_criteria = Sequel.expr { l__end_date >= r__end_date }
+        where_criteria = where_criteria.|(r__end_date: nil)
+
+        # If the RHS's min start date is less than the LHS start date,
+        # the entire LHS date range is truncated, which implies the row itself
+        # is ineligible to pass thru
+        db.from(db.from(left_stream(db))
+                  .join(Sequel.as(grouped_right, :r), l__person_id: :r__person_id)
+                  .where(where_criteria)
+                  .select(*new_columns)
+                  .select_append(Sequel.as(Sequel.function(:greatest, :l__start_date, :r__end_date), :start_date))
+               )
+      end
+
+      private
+      def new_columns
+        (COLUMNS - [:start_date]).map { |col| "l__#{col}".to_sym }
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/{nodes → operators}/union.rb

@@ -1,11 +1,15 @@
 require_relative 'pass_thru'
 
 module ConceptQL
-  module Nodes
+  module Operators
     class Union < PassThru
+      desc 'Pools sets of incoming results into a single large set of results.'
+      allows_many_upstreams
+      category 'Set Logic'
+
       def query(db)
         values.map do |expression|
-          expression.evaluate(db)
+          expression.evaluate(db).from_self
         end.inject do |q, query|
           q.union(query, all: true)
         end

data/lib/conceptql/operators/visit.rb

@@ -0,0 +1,15 @@
+require_relative 'operator'
+
+module ConceptQL
+  module Operators
+    class Visit < Operator
+      desc 'Generates all visit_occurrence records, or, if fed a stream, fetches all visit_occurrence records for the people represented in the incoming result set.'
+      types :visit_occurrence
+      allows_one_upstream
+
+      def types
+        [:visit_occurrence]
+      end
+    end
+  end
+end

data/lib/conceptql/{nodes → operators}/visit_occurrence.rb

@@ -1,8 +1,12 @@
-require_relative 'casting_node'
+require_relative 'casting_operator'
 
 module ConceptQL
-  module Nodes
-    class VisitOccurrence < CastingNode
+  module Operators
+    class VisitOccurrence < CastingOperator
+      desc 'Returns all visits in the database, or if given a upstream, converts all results to the set of visit_occurrences related to those results.'
+      allows_one_upstream
+      types :visit_occurrence
+
       def my_type
         :visit_occurrence
       end

data/lib/conceptql/query.rb

@@ -1,48 +1,50 @@
 require 'psych'
+require 'forwardable'
+require_relative 'behaviors/preppable'
 require_relative 'tree'
 
 module ConceptQL
   class Query
+    extend Forwardable
+    def_delegators :prepped_query, :all, :count, :execute, :order
+
     attr :statement
     def initialize(db, statement, tree = Tree.new)
       @db = db
+      @db.extend_datasets(ConceptQL::Behaviors::Preppable)
       @statement = statement
       @tree = tree
     end
 
-    def queries
-      build_query(db)
-    end
-
     def query
-      queries.last
+      build_query(db)
     end
 
     def sql
-      tree.opts[:sql_only] = true
-      nodes.map { |node| node.sql(db) }.join(";\n") + ';'
+      (tree.scope.sql(db) << operator.sql(db)).join(";\n\n") + ';'
     end
 
-    # To avoid a performance penalty, only execute the last
-    # SQL statement in an array of ConceptQL statements so that define's
-    # "create_table" SQL isn't executed twice
-    def execute
-      query.all
+    def types
+      tree.root(self).types
     end
 
-    def types
-      tree.root(self).last.types
+    def operator
+      @operator ||= tree.root(self)
     end
 
     private
     attr :yaml, :tree, :db
 
     def build_query(db)
-      nodes.map { |n| n.evaluate(db) }
+      operator.evaluate(db).tap { |q| q.prep_proc = prep_proc }
+    end
+
+    def prep_proc
+      @prep_proc = Proc.new { puts 'PREPPING'; tree.scope.prep(db) }
     end
 
-    def nodes
-      @nodes ||= tree.root(self)
+    def prepped_query
+      query
     end
   end
 end

data/lib/conceptql/scope.rb

@@ -0,0 +1,69 @@
+require_relative 'utils/temp_table'
+module ConceptQL
+  # Scope coordinates the creation of any TempTables that might
+  # be used when a Recall operator is present in the statement.
+  #
+  # Any time an operator is given a label, it becomes a candidate
+  # for a Recall operator to reuse the output of that operator
+  # somewhere else in the statement.
+  #
+  # Scope keeps track of all labeled operators and provides an
+  # API for Recall operators to fetch the results/types from
+  # labeled operators.
+  class Scope
+    attr_accessor :person_ids
+    attr :known_operators
+    def initialize
+      @known_operators = {}
+      @flagged = {}
+    end
+
+    def add_operator(operator)
+      # Recall operators respond to source, so when we add such an
+      # operator, we want to flag it so that we can come back to it later
+      # and create the appropriate temp tables
+      if operator.respond_to?(:source)
+        flagged[operator.source] = true
+      end
+      return unless operator.label
+      known_operators[operator.label] = operator
+    end
+
+    def from(db, label)
+      temp_table(db, label).from(db)
+    end
+
+    def types(label)
+      fetch_operator(label).types
+    end
+
+    def sql(db)
+      temp_tables(db).values.map do |temp|
+        temp.sql(db)
+      end
+    end
+
+    def prep(db)
+      temp_tables(db).values.map do |temp|
+        temp.build(db)
+      end
+    end
+
+    private
+    attr :flagged
+
+    def temp_tables(db)
+      Hash[flagged.keys.map do |label|
+        [label, TempTable.new(fetch_operator(label), db)]
+      end]
+    end
+
+    def temp_table(db, label)
+      temp_tables(db)[label]
+    end
+
+    def fetch_operator(label)
+      known_operators[label] || raise("No operator with label: '#{label}'")
+    end
+  end
+end

data/lib/conceptql/tree.rb

@@ -1,38 +1,53 @@
 require_relative 'nodifier'
-require 'active_support/core_ext/hash'
+require_relative 'converter'
+require_relative 'scope'
+require 'facets/hash/deep_rekey'
+require 'facets/array/recurse'
 
 module ConceptQL
+  # Tree is used to walk through a ConceptQL statement, instantiate
+  # all operators, and then provide access to the root operator
   class Tree
-    attr :nodifier, :behavior, :defined, :opts
+    attr :nodifier, :behavior, :defined, :opts, :temp_tables, :scope
     def initialize(opts = {})
       @nodifier = opts.fetch(:nodifier, Nodifier.new)
       @behavior = opts.fetch(:behavior, nil)
       @defined = {}
+      @temp_tables = {}
       @opts = {}
+      @scope = opts.fetch(:scope, Scope.new)
     end
 
-    def root(*queries)
-      @root ||= traverse(queries.flatten.map(&:statement).flatten.map(&:deep_symbolize_keys))
+    def root(query)
+      @root ||= start_traverse(query.statement)
     end
 
     private
-    def traverse(obj)
-      case obj
+
+    def start_traverse(stmt)
+      case stmt
       when Hash
-        if obj.keys.length > 1
-          obj = Hash[obj.map { |key, value| [ key, traverse(value) ]}]
-          return obj
-        end
-        type = obj.keys.first
-        values = traverse(obj[type])
-        obj = nodifier.create(type, values, self)
-        obj.extend(behavior) if behavior
-        obj
+        traverse(converter.convert(stmt))
       when Array
-        obj.map { |value| traverse(value) }
-      else
-        obj
+        traverse(stmt)
+      end
+    end
+
+    def traverse(stmt)
+      stmt.recurse(Array, Hash) do |arr_or_hash|
+        if arr_or_hash.is_a?(Array)
+          type = arr_or_hash.shift
+          obj = nodifier.create(scope, type, *arr_or_hash)
+          obj.extend(behavior) if behavior
+          obj
+        else
+          arr_or_hash
+        end
       end
     end
+
+    def converter
+      @converter ||= Converter.new
+    end
   end
 end

data/lib/conceptql/utils/temp_table.rb

@@ -0,0 +1,72 @@
+require 'zlib'
+
+module ConceptQL
+  # TempTable coordinates the creation of any temporary tables a statement
+  # might need.
+  #
+  # Currently, temp tables are used to share a set of results when a Recall
+  # operator is present in a statement.
+  #
+  # It also provides an API to generate the SQL that Sequel would use to create
+  # and populate the temp table
+  class TempTable
+    attr :operator
+    def initialize(operator, db)
+      @operator = operator
+      fake_it!(db)
+    end
+
+    def build(db)
+      @built ||= build_it(db)
+    end
+
+    def sql(db)
+      # Sequel doesn't (currently) provide an API for getting the SQL for the
+      # creation of tables, so we're calling one of the private methods
+      sql = db[db.send(:create_table_as_sql, table_name, operator.evaluate(db), temp: true)].sql
+      ["-- #{operator.label}", sql].join("\n")
+    end
+
+    def from(db)
+      db[table_name]
+    end
+
+    private
+
+    def table_name
+      namify(operator.label)
+    end
+
+    def build_it(db)
+      db.create_table!(table_name, as: operator.evaluate(db), temp: true)
+      true
+    end
+
+    def namify(name)
+      digest = Zlib.crc32 name
+      ('_' + digest.to_s).to_sym
+    end
+
+    # Creates the temp table, but populates it with a single fake row of null
+    # values.  Some operations, such as generating the SQL for a statement
+    # Want the temp tables to exist, but we don't _really_ need the actual
+    # results in the temp table since they might take a long time to calculate.
+    def fake_it!(db)
+      db.create_table!(table_name, temp: true, as: fake_row(db))
+    end
+
+    def fake_row(db)
+      db
+        .select(Sequel.cast(nil, Bignum).as(:person_id))
+        .select_append(Sequel.cast(nil, Bignum).as(:criterion_id))
+        .select_append(Sequel.cast(nil, String).as(:criterion_type))
+        .select_append(Sequel.cast(nil, Date).as(:start_date))
+        .select_append(Sequel.cast(nil, Date).as(:end_date))
+        .select_append(Sequel.cast(nil, Bignum).as(:value_as_number))
+        .select_append(Sequel.cast(nil, String).as(:value_as_string))
+        .select_append(Sequel.cast(nil, Bignum).as(:value_as_concept_id))
+        .select_append(Sequel.cast(nil, String).as(:units_source_value))
+        .select_append(Sequel.cast(nil, String).as(:source_value))
+    end
+  end
+end

data/lib/conceptql/version.rb

@@ -1,3 +1,3 @@
 module ConceptQL
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end