conceptql 0.1.1 → 0.2.0

data/lib/conceptql/graph.rb

@@ -1,7 +1,7 @@
 require 'psych'
 require 'graphviz'
 require_relative 'tree'
-require_relative 'nodes/node'
+require_relative 'operators/operator'
 require_relative 'behaviors/dottable'
 
 module ConceptQL
@@ -14,16 +14,16 @@ module ConceptQL
       @tree = opts.fetch(:tree, Tree.new)
       @title = opts.fetch(:title, nil)
       @suffix = opts.fetch(:suffix, 'pdf')
-      ConceptQL::Nodes::Node.send(:include, ConceptQL::Behaviors::Dottable)
+      ConceptQL::Operators::Operator.send(:include, ConceptQL::Behaviors::Dottable)
     end
 
     def graph_it(file_path)
       build_graph(g)
-      g.output(suffix.to_sym =>  file_path + ".#{suffix}")
+      graph.output(suffix.to_sym =>  file_path + ".#{suffix}")
     end
 
-    def g
-      @g ||= begin
+    def graph
+      @graph ||= begin
         opts = { type: :digraph }
         opts[:label] = title if title
         GraphViz.new(:G, opts)
@@ -34,15 +34,16 @@ module ConceptQL
     attr :yaml, :tree, :db
 
     def build_graph(g)
-      tree.root(self).each.with_index do |last_node, index|
-        last_node.graph_it(g, db)
+      tree.root(self).each.with_index do |last_operator, index|
+        last_operator.build_temp_tables(db)
+        last_operator.graph_it(g, db)
         if dangler
-          blank_node = g.add_nodes("_#{index}")
-          blank_node[:shape] = 'none'
-          blank_node[:height] = 0
-          blank_node[:label] = ''
-          blank_node[:fixedsize] = true
-          last_node.link_to(g, blank_node, db)
+          blank_operator = g.add_nodes("_#{index}")
+          blank_operator[:shape] = 'none'
+          blank_operator[:height] = 0
+          blank_operator[:label] = ''
+          blank_operator[:fixedsize] = true
+          last_operator.link_to(g, blank_operator, db)
         end
       end
     end

data/lib/conceptql/graph_nodifier.rb

@@ -1,9 +1,9 @@
 require_relative 'behaviors/dottable'
-require_relative 'nodes/node'
+require_relative 'operators/operator'
 
 module ConceptQL
   class GraphNodifier
-    class DotNode < ConceptQL::Nodes::Node
+    class DotOperator < ConceptQL::Operators::Operator
       include ConceptQL::Behaviors::Dottable
 
       TYPES = {
@@ -13,6 +13,7 @@ module ConceptQL
         icd9: :condition_occurrence,
         icd10: :condition_occurrence,
         condition_type: :condition_occurrence,
+        medcode: :condition_occurrence,
 
         # Procedures
         procedure: :procedure_occurrence,
@@ -21,6 +22,7 @@ module ConceptQL
         hcpcs: :procedure_occurrence,
         icd9_procedure: :procedure_occurrence,
         procedure_cost: :procedure_cost,
+        medcode_procedure: :procedure_occurrence,
 
         # Visits
         visit_occurrence: :visit_occurrence,
@@ -40,6 +42,8 @@ module ConceptQL
 
         # Observation
         loinc: :observation,
+        from_seer_visits: :observation,
+        to_seer_visits: :observation,
 
         # Drug
         drug_exposure: :drug_exposure,
@@ -47,11 +51,12 @@ module ConceptQL
         drug_cost: :drug_cost,
         drug_type_concept_id: :drug_exposure,
         drug_type_concept: :drug_exposure,
+        prodcode: :drug_exposure,
 
-        # Date Nodes
+        # Date Operators
         date_range: :date,
 
-        # Miscelaneous nodes
+        # Miscelaneous operators
         concept: :misc,
         vsac: :misc
       }
@@ -78,12 +83,12 @@ module ConceptQL
       end
 
       def types
-        types = [TYPES[name.to_sym] || children.map(&:types)].flatten.uniq
+        types = [TYPES[name.to_sym] || upstreams.map(&:types)].flatten.uniq
         types.empty? ? [:misc] : types
       end
     end
 
-    class BinaryOperatorNode < DotNode
+    class BinaryOperatorOperator < DotOperator
       def display_name
         output = name
         output += "\n#{displayable_options.map{|k,v| "#{k}: #{v}"}.join("\n")}"
@@ -105,7 +110,7 @@ module ConceptQL
       def graph_it(g, db)
         left.graph_it(g, db)
         right.graph_it(g, db)
-        cluster_name = "cluster_#{node_name}"
+        cluster_name = "cluster_#{operator_name}"
         me = g.send(cluster_name) do |sub|
           sub[rank: 'same', label: display_name, color: 'black']
           sub.send("#{cluster_name}_left").send('[]', shape: 'point', color: type_color(types))
@@ -113,7 +118,7 @@ module ConceptQL
         end
         left.link_to(g, me.send("#{cluster_name}_left"))
         right.link_to(g, me.send("#{cluster_name}_right"))
-        @__graph_node = me.send("#{cluster_name}_left")
+        @__graph_operator = me.send("#{cluster_name}_left")
       end
 
       def types
@@ -125,19 +130,37 @@ module ConceptQL
       end
     end
 
-    class DefineNode < DotNode
+    class LetOperator < DotOperator
+      def graph_it(g, db)
+        cluster_name = "cluster_#{operator_name}"
+        linkable = nil
+        g.send(cluster_name) do |sub|
+          linkable = upstreams.reverse.map do |upstream|
+            upstream.graph_it(sub, db)
+          end.first
+          sub[label: display_name, color: 'black']
+        end
+        @__graph_operator = linkable
+      end
+
+      def types
+        upstreams.last.types
+      end
+    end
+
+    class DefineOperator < DotOperator
       def shape
         :cds
       end
     end
 
-    class RecallNode < DotNode
+    class RecallOperator < DotOperator
       def shape
         :cds
       end
     end
 
-    class VsacNode < DotNode
+    class VsacOperator < DotOperator
       def initialize(name, values, types)
         @types = types
         super(name, values)
@@ -150,22 +173,31 @@ module ConceptQL
 
     BINARY_OPERATOR_TYPES = %w(before after meets met_by started_by starts contains during overlaps overlapped_by finished_by finishes coincides except person_filter less_than less_than_or_equal equal not_equal greater_than greater_than_or_equal filter).map { |temp| [temp, "not_#{temp}"] }.flatten.map(&:to_sym)
 
+    def temp_tables
+      @temp_tables ||= {}
+    end
+
     def types
       @types ||= {}
     end
 
     def create(type, values, tree)
-      if BINARY_OPERATOR_TYPES.include?(type)
-        return BinaryOperatorNode.new(type, values)
+      operator = if BINARY_OPERATOR_TYPES.include?(type)
+        BinaryOperatorOperator.new(type, values)
+      elsif type == :let
+        LetOperator.new(type, values)
       elsif type == :define
-        return DefineNode.new(type, values).tap { |n| n.tree = self }
+        DefineOperator.new(type, values)
       elsif type == :recall
-        return RecallNode.new(type, values).tap { |n| n.tree = self }
+        RecallOperator.new(type, values)
       elsif type == :vsac
         types = values.pop
-        return VsacNode.new(type, values, types)
+        VsacOperator.new(type, values, types)
+      else
+        DotOperator.new(type, values)
       end
-      DotNode.new(type, values)
+      operator.tree = self
+      operator
     end
   end
 end

data/lib/conceptql/nodifier.rb

@@ -1,11 +1,36 @@
-require 'active_support/inflector'
+require 'facets/hash/deep_rekey'
+require 'facets/pathname/chdir'
+require 'facets/string/modulize'
+
 module ConceptQL
   class Nodifier
-    def create(type, values, tree)
-      require_relative "nodes/#{type}"
-      node = "conceptQL/nodes/#{type}".camelize.constantize.new(values)
-      node.tree = tree
-      node
+    attr_reader :operators
+
+    def initialize
+      @operators = {}
+      dir = Pathname.new(__FILE__).dirname()
+      dir.chdir do
+        Pathname.glob("operators/*.rb").each do |file|
+          require_relative file
+          operator = file.basename('.*').to_s.to_sym
+          klass = Object.const_get("conceptQL/operators/#{operator}".modulize)
+          @operators[operator] = klass
+        end
+      end
+    end
+
+    def create(scope, operator, *values)
+      operator = operator.to_sym
+      if operators[operator].nil?
+        raise "Can't find operator for '#{operator}' in #{operators.keys.sort}"
+      end
+      operator = operators[operator].new(*values)
+      operator.scope = scope
+      operator
+    end
+
+    def to_metadata
+      Hash[operators.map { |k, v| [k, v.to_metadata]}.select { |k, v| v[:desc] }]
     end
   end
 end

data/lib/conceptql/operators/after.rb

@@ -0,0 +1,25 @@
+require_relative 'temporal_operator'
+
+module ConceptQL
+  module Operators
+    class After < TemporalOperator
+      desc <<-EOF
+Compares all results on a person-by-person basis between the left hand results (LHR) and the right hand resuls (RHR).
+For any result in the LHR whose start_date occurs after the earliest end_date of the RHR, that result is passed through.
+All other results are discarded, including all results in the RHR.
+L-------N-------L
+R-----R
+   R-----R
+        L-----Y----L
+      EOF
+      def right_stream(db)
+        right.evaluate(db).from_self.group_by(:person_id).select(:person_id, Sequel.function(:min, :end_date).as(:end_date)).as(:r)
+      end
+
+      def where_clause
+        Proc.new { l__start_date > r__end_date }
+      end
+    end
+  end
+end
+

data/lib/conceptql/operators/any_overlap.rb

@@ -0,0 +1,15 @@
+require_relative 'temporal_operator'
+
+module ConceptQL
+  module Operators
+    class AnyOverlap < TemporalOperator
+      desc 'If a result from the LHR overlaps in any way a result from the RHR it is passed along.'
+      def where_clause
+        l_partly_in_r = Sequel.expr { r__start_date <= l__start_date }.&(Sequel.expr { l__start_date <= r__end_date })
+        r_partly_in_l = Sequel.expr { l__start_date <= r__start_date }.&(Sequel.expr { r__start_date <= l__end_date })
+        l_partly_in_r.|(r_partly_in_l)
+      end
+    end
+  end
+end
+

data/lib/conceptql/operators/before.rb

@@ -0,0 +1,21 @@
+require_relative 'temporal_operator'
+
+module ConceptQL
+  module Operators
+    class Before < TemporalOperator
+      desc <<-EOF
+Compares all results on a person-by-person basis between the left hand results (LHR) and the right hand resuls (RHR).
+For any result in the LHR whose end_date occurs before the most recent start_date of the RHR, that result is passed through.
+All other results are discarded, including all results in the RHR.
+      EOF
+
+      def right_stream(db)
+        right.evaluate(db).from_self.group_by(:person_id).select(:person_id, Sequel.function(:max, :start_date).as(:start_date)).as(:r)
+      end
+
+      def where_clause
+        Proc.new { l__end_date < r__start_date }
+      end
+    end
+  end
+end

data/lib/conceptql/{nodes/binary_operator_node.rb → operators/binary_operator_operator.rb}

@@ -1,17 +1,21 @@
-require_relative 'node'
+require_relative 'operator'
+require 'facets/string/titlecase'
 
 module ConceptQL
-  module Nodes
-    # Base class for all nodes that take two streams, a left-hand and a right-hand
-    class BinaryOperatorNode < Node
-      def children
+  module Operators
+    # Base class for all operators that take two streams, a left-hand and a right-hand
+    class BinaryOperatorOperator < Operator
+      option :left, type: :upstream
+      option :right, type: :upstream
+
+      def upstreams
         [left]
       end
 
       def graph_it(g, db)
         left.graph_it(g, db)
         right.graph_it(g, db)
-        cluster_name = "cluster_#{node_name}"
+        cluster_name = "cluster_#{operator_name}"
         me = g.send(cluster_name) do |sub|
           sub[rank: 'same', label: display_name, color: 'black']
           sub.send("#{cluster_name}_left").send('[]', shape: 'point', color: type_color(types))
@@ -19,11 +23,11 @@ module ConceptQL
         end
         left.link_to(g, me.send("#{cluster_name}_left"), db)
         right.link_to(g, me.send("#{cluster_name}_right"), db)
-        @__graph_node = me.send("#{cluster_name}_left")
+        @__graph_operator = me.send("#{cluster_name}_left")
       end
 
       def display_name
-        self.class.name.split('::').last.titleize
+        self.class.name.split('::').last.snakecase.titlecase
       end
 
       private

data/lib/conceptql/{nodes/casting_node.rb → operators/casting_operator.rb}

@@ -1,28 +1,29 @@
-require_relative 'node'
+require_relative 'operator'
 
 module ConceptQL
-  module Nodes
-    # Parent class of all casting nodes
+  module Operators
+    # Parent class of all casting operators
     #
     # Subclasses must implement the following methods:
     # - my_type
     # - i_point_at
     # - these_point_at_me
     #
-    # i_point_at returns a list of types for which the node's table of origin
+    # i_point_at returns a list of types for which the operator's table of origin
     # has foreign_keys pointing to another table, e.g.:
     # procedure_occurrence has an FK to visit_occurrence, so we'd put
     # :visit_occurrence in the i_point_at array
     #
     # these_point_at_me is a list of types for which that type's table
-    # of origin has a FK  pointing to the current node's
+    # of origin has a FK  pointing to the current operator's
     # table of origin, e.g.:
     # procedure_cost has an FK to procedure_occurrence so we'd
     # put :procedure_cost in procedure_occurrence's these_point_at_me array
     #
-    # Also, if a casting node is passed no streams, it will return all the
+    # Also, if a casting operator is passed no streams, it will return all the
     # rows in its table as results.
-    class CastingNode < Node
+    class CastingOperator < Operator
+      category 'Casting'
       def types
         [type]
       end

data/lib/conceptql/{nodes → operators}/complement.rb

@@ -1,12 +1,16 @@
 require_relative 'pass_thru'
 
 module ConceptQL
-  module Nodes
+  module Operators
     class Complement < PassThru
+      desc 'Splits up the incoming result set by type and passes through all results for each type that are NOT in the current set.'
+      allows_one_upstream
+      category 'Set Logic'
+
       def query(db)
-        child = children.first
-        child.types.map do |type|
-          positive_query = db.from(child.evaluate(db))
+        upstream = upstreams.first
+        upstream.types.map do |type|
+          positive_query = db.from(upstream.evaluate(db))
             .select(:criterion_id)
             .exclude(:criterion_id => nil)
             .where(:criterion_type => type.to_s)
@@ -18,18 +22,17 @@ module ConceptQL
         end
       end
 
-=begin
-This is an alternate, but equally accurate way to do complement.
-We'll need to benchmark which is faster.
+
+      # This is an alternate, but equally accurate way to do complement.
+      # We'll need to benchmark which is faster.
       def query2(db)
-        child = children.first
-        froms = child.types.map do |type|
+        upstream = upstreams.first
+        froms = upstream.types.map do |type|
           select_it(db.from(make_table_name(type)), type)
         end
         big_from = froms.inject { |union_query, q| union_query.union(q, all:true) }
-        db.from(big_from).except(child.evaluate(db))
+        db.from(big_from).except(upstream.evaluate(db))
       end
-=end
     end
   end
 end

data/lib/conceptql/operators/concept.rb

@@ -0,0 +1,70 @@
+require_relative 'operator'
+require_relative '../query'
+
+module ConceptQL
+  module Operators
+    class Concept < Operator
+      desc 'Given the UUID of another ConceptQL statement, returns the results of that statement.'
+      attr :cql_query
+      def query(db)
+        set_cql_query(db)
+        db.from(cql_query.query.from_self)
+      end
+
+      def upstreams
+        @upstreams ||= cql_query.operators
+      end
+
+      def graph_prep(db)
+        set_cql_query(db)
+        puts statement.inspect
+        @arguments = [description(db)]
+      end
+
+      def print_prep(db)
+        set_cql_query(db)
+      end
+
+      def ensure_temp_tables(db)
+        set_cql_query(db)
+        super
+      end
+
+      def build_temp_tables(db)
+        set_cql_query(db)
+        super
+      end
+
+      private
+
+      def set_cql_query(db)
+        @cql_query ||= begin
+          set_statement(db)
+          Query.new(db, statement)
+        end
+      end
+
+      def statement
+        raise "Statement is nil!" unless @statement
+        @statement
+      end
+
+      def set_statement(db)
+        @statement ||= get_statement(db)
+      end
+
+      def get_statement(db)
+        concept = db[:concepts].where(concept_id: arguments.first).limit(1)
+        statement = concept.select_map(:statement).first.tap { |f| ConceptQL.logger.debug f.inspect }
+        statement = JSON.parse(statement) if statement.is_a?(String)
+        statement
+      end
+
+      def description(db)
+        @description ||= db[:concepts].where(concept_id: arguments.first).select_map(:label).first
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/{nodes → operators}/condition_type.rb

@@ -1,15 +1,20 @@
-require_relative 'node'
+require_relative 'operator'
 
 module ConceptQL
-  module Nodes
-    # Represents a node that will grab all conditions that match the
+  module Operators
+    # Represents a operator that will grab all conditions that match the
     # condition type passed in
     #
     # Condition Type represents which position the condition held in
     # the raw data, e.g. primary inpatient header or 15th outpatient detail
     #
     # Multiple types can be specified at once
-    class ConditionType < Node
+    class ConditionType < Operator
+      desc 'Searches for conditions that match the given set of Condition Types'
+      argument :condition_types, type: :codelist, vocab: 'Condition Type'
+      category %(Occurrence Type)
+      predominant_types :condition_occurrence
+
       def type
         :condition_occurrence
       end
@@ -29,7 +34,7 @@ module ConceptQL
       def to_concept_id(ctype)
         ctype = ctype.to_s.downcase
         position = nil
-        if ctype =~ /(\d|_primary)$/
+        if ctype =~ /(\d|_primary)$/ && ctype.count('_') > 1
           parts = ctype.split('_')
           position = parts.pop.to_i
           position -= 1 if ctype =~ /^outpatient/
@@ -51,6 +56,9 @@ module ConceptQL
             condition_era_0_day_window: [38000246],
             condition_era_30_day_window: [38000247]
           }
+          hash[:primary] = %w(inpatient_detail inpatient_header outpatient_detail outpatient_header).map { |w| hash[w.to_sym].first }
+          hash[:outpatient_primary] = %w(outpatient_detail outpatient_header).map { |w| hash[w.to_sym].first }
+          hash[:inpatient_primary] = %w(inpatient_detail inpatient_header).map { |w| hash[w.to_sym].first }
           hash[:inpatient] = hash[:inpatient_detail] + hash[:inpatient_header]
           hash[:outpatient] = hash[:outpatient_detail] + hash[:outpatient_header]
           hash[:condition_era] = hash[:condition_era_0_day_window] + hash[:condition_era_30_day_window]

data/lib/conceptql/operators/contains.rb

@@ -0,0 +1,17 @@
+require_relative 'temporal_operator'
+
+module ConceptQL
+  module Operators
+    class Contains < TemporalOperator
+      desc <<-EOF
+Any result in the LHR whose start_date is on or before and whose end_date is on or after a result from the RHR.
+L--X-L
+R-----R
+L------Y--------L
+EOF
+      def where_clause
+        [Proc.new { l__start_date <= r__start_date}, Proc.new { r__end_date <= l__end_date }]
+      end
+    end
+  end
+end

data/lib/conceptql/operators/count.rb

@@ -0,0 +1,26 @@
+require_relative 'pass_thru'
+
+module ConceptQL
+  module Operators
+    class Count < PassThru
+      desc 'Counts the number of results the exactly match across all columns.'
+      allows_one_upstream
+
+      def query(db)
+        db.from(unioned(db))
+          .group(*COLUMNS)
+          .select(*(COLUMNS - [:value_as_number]))
+          .select_append{count(1).as(:value_as_number)}
+          .from_self
+      end
+
+      def unioned(db)
+        upstreams.map { |c| c.evaluate(db) }.inject do |uni, q|
+          uni.union(q)
+        end
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/operators/cpt.rb

@@ -0,0 +1,25 @@
+require_relative 'standard_vocabulary_operator'
+
+module ConceptQL
+  module Operators
+    class Cpt < StandardVocabularyOperator
+      preferred_name 'CPT'
+      desc 'Searches the procedure_occurrence table for all procedures with matching CPT codes'
+      argument :cpts, type: :codelist, vocab: 'CPT4'
+      predominant_types :procedure_occurrence
+
+      def table
+        :procedure_occurrence
+      end
+
+      def vocabulary_id
+        4
+      end
+
+      def concept_column
+        :procedure_concept_id
+      end
+    end
+  end
+end
+

data/lib/conceptql/{nodes → operators}/date_range.rb

@@ -1,13 +1,18 @@
-require_relative 'node'
+require_relative 'operator'
 
 module ConceptQL
-  module Nodes
-    # Represents a node that will create a date_range for every person in the database
+  module Operators
+    # Represents a operator that will create a date_range for every person in the database
     #
     # Accepts two params: start and end formateed as 'YYYY-MM-DD' or 'START' or 'END'
     # 'START' represents the first date of data in the data source,
     # 'END' represents the last date of data in the data source,
-    class DateRange < Node
+    class DateRange < Operator
+      desc 'Used to represent a date literal.'
+      option :start, type: :string
+      option :end, type: :string
+      category %(Temporal Manipulation)
+
       def query(db)
         db.from(:person)
           .select_append(Sequel.cast_string('person').as(:criterion_type))
@@ -31,8 +36,8 @@ module ConceptQL
       # TODO: Select the earliest and latest dates of observation from
       # the proper CDM table to represent the start and end of data
       def date_from(db, str)
-        return db.from(:visit_occurrence).select { min(:start_date) } if str.upcase == 'START'
-        return db.from(:visit_occurrence).select { max(:end_date) } if str.upcase == 'END'
+        return db.from(:observation_period).select { min(:observation_period_start_date) } if str.upcase == 'START'
+        return db.from(:observation_period).select { max(:observation_period_end_date) } if str.upcase == 'END'
         return Sequel.lit('CONVERT(DATETIME, ?)', str) if db.database_type == :mssql
         return Sequel.lit('date ?', str)
       end