conceptql 0.0.1

data/lib/conceptql/behaviors/dottable.rb

@@ -0,0 +1,71 @@
+require 'active_support/inflector'
+module ConceptQL
+  module Behaviors
+    module Dottable
+      @@counter = 0
+      TYPE_COLORS = {
+        person: 'blue',
+        visit_occurrence: 'orange',
+        condition_occurrence: 'red',
+        procedure_occurrence: 'green3',
+        procedure_cost: 'gold',
+        death: 'brown',
+        payer_plan_period: 'blue',
+        drug_exposure: 'purple',
+        observation: 'magenta',
+        misc: 'black'
+      }
+      def node_name
+        @__node_name ||= self.class.name.split('::').last.underscore.gsub(/\W/, '_').downcase + '_' + (@@counter += 1).to_s
+      end
+
+      def display_name
+        @__display_name ||= begin
+          output = self.class.name.split('::').last.titleize
+          output += ": #{arguments.join(', ')}" unless arguments.empty?
+          if output.length > 100
+            parts = output.split
+            output = parts.each_slice(output.length / parts.count).map do |subparts|
+              subparts.join(' ')
+            end.join ('\n')
+          end
+          output += "\n#{options.map{|k,v| "#{k}: #{v}"}.join("\n")}" unless options.nil? || options.empty?
+          output
+        end
+      end
+
+      def type_color(*types)
+        types.flatten!
+        types.length == 1 ? TYPE_COLORS[types.first] || 'black' : 'black'
+      end
+
+      def graph_node(g)
+        @__graph_node ||= begin
+          me = g.add_nodes(node_name)
+          me[:label] = display_name
+          me[:color] = type_color(types)
+          me
+        end
+      end
+
+      def link_to(g, dest_node)
+        types.each do |type|
+          e = g.add_edges(graph_node(g), dest_node)
+          e[:color] = type_color(type)
+        end
+      end
+
+      def graph_it(g, db)
+        graph_prep(db) if respond_to?(:graph_prep)
+        children.each do |child|
+          child.graph_it(g, db)
+        end
+        graph_node(g)
+        children.each do |child|
+          child.link_to(g, graph_node(g))
+        end
+      end
+    end
+  end
+end
+

data/lib/conceptql/cli.rb

@@ -0,0 +1,135 @@
+require 'thor'
+require 'sequelizer'
+require 'psych'
+require 'json'
+require 'pp'
+require_relative 'query'
+
+module ConceptQL
+  class CLI < Thor
+    include Sequelizer
+
+    class_option :adapter,
+      aliases: :a,
+      desc: 'adapter for database'
+    class_option :host,
+      aliases: :h,
+      banner: 'localhost',
+      desc: 'host for database'
+    class_option :username,
+      aliases: :u,
+      desc: 'username for database'
+    class_option :password,
+      aliases: :P,
+      desc: 'password for database'
+    class_option :port,
+      aliases: :p,
+      type: :numeric,
+      banner: '5432',
+      desc: 'port for database'
+    class_option :database,
+      aliases: :d,
+      desc: 'database for database'
+    class_option :search_path,
+      aliases: :s,
+      desc: 'schema for database (PostgreSQL only)'
+
+    desc 'run_statement statement_file', 'Evals the statement from the statement file and executes it agains the DB specified by DB_URL'
+    def run_statement(statement_file)
+      q = ConceptQL::Query.new(db(options), criteria_from_file(statement_file))
+      puts q.query.sql
+      puts q.statement.to_yaml
+      pp q.execute
+    end
+
+    desc 'fake_graph file', 'Evals the file and shows the contents as a ConceptQL graph'
+    def fake_graph(file)
+      require_relative 'graph'
+      require_relative 'tree'
+      require_relative 'graph_nodifier'
+      ConceptQL::Graph.new(criteria_from_file(file),
+         dangler: true,
+         tree: ConceptQL::Tree.new(nodifier: ConceptQL::GraphNodifier.new)
+      ).graph_it('/tmp/graph')
+      system('open /tmp/graph.pdf')
+    end
+
+    desc 'show_graph file', 'Evals the file and shows the contents as a ConceptQL graph'
+    def show_graph(file)
+      graph_it(criteria_from_file(file))
+    end
+
+    desc 'show_and_tell_file file', 'Evals the file and shows the contents as a ConceptQL graph, then executes the statement against our test database'
+    option :full
+    def show_and_tell_file(file)
+      show_and_tell(criteria_from_file(file), options)
+    end
+
+    desc 'show_and_tell_db conceptql_id', 'Fetches the ConceptQL from a DB and shows the contents as a ConceptQL graph, then executes the statement against our test database'
+    option :full
+    def show_and_tell_db(conceptql_id)
+      result = fetch_conceptql(conceptql_id, options)
+      puts "Concept: #{result[:label]}"
+      show_and_tell(result[:statement].to_hash, options, result[:label])
+    end
+
+    desc 'show_db_graph conceptql_id', 'Shows a graph for the conceptql statement represented by conceptql_id in the db specified by db_url'
+    def show_db_graph(conceptql_id)
+      result = fetch_conceptql(conceptql_id, options)
+      graph_it(result[:statement].to_hash, db, result[:label])
+    end
+
+    private
+    def fetch_conceptql(conceptql_id)
+      my_db = db(options)
+      my_db.extension(:pg_array, :pg_json)
+      my_db[:concepts].where(concept_id: conceptql_id).select(:statement, :label).first
+    end
+
+    def show_and_tell(statement, options, title = nil)
+      my_db = db(options)
+      q = ConceptQL::Query.new(my_db, statement)
+      puts 'YAML'
+      puts q.statement.to_yaml
+      puts 'JSON'
+      puts JSON.pretty_generate(q.statement)
+      STDIN.gets
+      graph_it(statement, my_db, title)
+      STDIN.gets
+      puts q.query.sql
+      STDIN.gets
+      results = q.execute
+      if options[:full]
+        pp results
+      else
+        pp filtered(results)
+      end
+      puts results.length
+    end
+
+    def graph_it(statement, db = nil, title = nil)
+      require_relative 'graph'
+      require_relative 'tree'
+      ConceptQL::Graph.new(statement,
+                           dangler: true,
+                           title: title,
+                           db: db
+                          ).graph_it('/tmp/graph')
+      system('open /tmp/graph.pdf')
+    end
+
+    def criteria_from_file(file)
+      case File.extname(file)
+      when '.json'
+        JSON.parse(File.read(file))
+      else
+        eval(File.read(file))
+      end
+    end
+
+    def filtered(results)
+      results.each { |r| r.delete_if { |k,v| v.nil? } }
+    end
+  end
+end
+

data/lib/conceptql/date_adjuster.rb

@@ -0,0 +1,45 @@
+require_relative '../conceptql'
+
+module ConceptQL
+  # Used to translate a string of terse date adjustments into a set of adjustments that are compatible with most RDBMSs
+  class DateAdjuster
+    attr :str
+    def initialize(str)
+      @str = str
+    end
+
+    # Returns an array of strings that represent date modifiers
+    def adjustments
+      @adjustments ||= parse(str)
+    end
+
+    private
+    def lookup
+      {
+        'y' => 'year',
+        'm' => 'month',
+        'd' => 'day'
+      }
+    end
+
+    def parse(str)
+      ConceptQL.logger.debug(str)
+      return [] if str.nil?
+      return ["#{str} days"] if str.match(/^[-+]?\d+$/)
+      str.downcase.scan(/([-+]?\d*[dmy])/).map do |adjustment|
+        adjustment = adjustment.first
+        quantity = 1
+        if adjustment.match(/\d/)
+          quantity = adjustment.to_i
+        else
+          if adjustment.chars.first == '-'
+            quantity = -1
+          end
+        end
+        unit = lookup[adjustment.chars.last]
+        unit += 's' if quantity.abs > 1
+        [quantity, unit].join(' ')
+      end
+    end
+  end
+end

data/lib/conceptql/graph.rb

@@ -0,0 +1,49 @@
+require 'psych'
+require 'graphviz'
+require_relative 'tree'
+require_relative 'nodes/node'
+require_relative 'behaviors/dottable'
+
+module ConceptQL
+  class Graph
+    attr :g, :file_path, :dangler, :title, :statement, :db, :suffix
+    def initialize(statement, opts = {})
+      @statement = statement
+      @db = opts.fetch(:db, nil)
+      @dangler = opts.fetch(:dangler, false)
+      @tree = opts.fetch(:tree, Tree.new)
+      @title = opts.fetch(:title, nil)
+      @suffix = opts.fetch(:suffix, 'pdf')
+      ConceptQL::Nodes::Node.send(:include, ConceptQL::Behaviors::Dottable)
+    end
+
+    def graph_it(file_path)
+      build_graph(g)
+      g.output(suffix.to_sym =>  file_path + ".#{suffix}")
+    end
+
+    def g
+      @g ||= begin
+        opts = { type: :digraph }
+        opts[:label] = title if title
+        GraphViz.new(:G, opts)
+      end
+    end
+
+    private
+    attr :yaml, :tree, :db
+
+    def build_graph(g)
+      last_node = tree.root(self)
+      last_node.graph_it(g, db)
+      if dangler
+        blank_node = g.add_nodes('')
+        blank_node[:shape] = 'none'
+        blank_node[:height] = 0
+        blank_node[:fixedsize] = true
+        last_node.link_to(g, blank_node)
+      end
+    end
+  end
+end
+

data/lib/conceptql/graph_nodifier.rb

@@ -0,0 +1,123 @@
+require_relative 'behaviors/dottable'
+require_relative 'nodes/node'
+
+module ConceptQL
+  class GraphNodifier
+    class DotNode < ConceptQL::Nodes::Node
+      include ConceptQL::Behaviors::Dottable
+
+      TYPES = {
+        # Conditions
+        condition: :condition_occurrence,
+        primary_diagnosis: :condition_occurrence,
+        icd9: :condition_occurrence,
+
+        # Procedures
+        procedure: :procedure_occurrence,
+        cpt: :procedure_occurrence,
+        hcpcs: :procedure_occurrence,
+        icd9_procedure: :procedure_occurrence,
+        procedure_cost: :procedure_cost,
+
+        # Visits
+        visit_occurrence: :visit_occurrence,
+        place_of_service: :visit_occurrence,
+
+        # Person
+        person: :person,
+        gender: :person,
+        race: :person,
+
+        # Payer
+        payer: :payer_plan_period,
+
+        # Death
+        death: :death,
+
+        # Observation
+        loinc: :observation,
+
+        # Drug
+        drug_exposure: :drug_exposure,
+        drug_cost: :drug_cost,
+
+        # Date Nodes
+        date_range: :date,
+
+        # Miscelaneous nodes
+        concept: :misc
+      }
+
+      attr :values, :name
+      def initialize(name, values)
+        @name = name.to_s
+        super(values)
+      end
+
+      def display_name
+        @__display_name ||= begin
+          output = @name.dup
+          output += ": #{arguments.join(', ')}" unless arguments.empty?
+          if output.length > 100
+            parts = output.split
+            output = parts.each_slice(output.length / parts.count).map do |subparts|
+              subparts.join(' ')
+            end.join ('\n')
+          end
+          output += "\n#{options.map{|k,v| "#{k}: #{v}"}.join("\n")}" unless options.nil? || options.empty?
+          output
+        end
+      end
+
+      def types
+        types = [TYPES[name.to_sym] || children.map(&:types)].flatten.uniq
+        types.empty? ? [:misc] : types
+      end
+    end
+
+    class BinaryOperatorNode < DotNode
+      def display_name
+        output = name
+        output += "\n#{options.map{|k,v| "#{k}: #{v}"}.join("\n")}" unless options.nil? || options.empty?
+        output
+      end
+
+      def left
+        options[:left]
+      end
+
+      def right
+        options[:right]
+      end
+
+      def graph_it(g, db)
+        left.graph_it(g, db)
+        right.graph_it(g, db)
+        cluster_name = "cluster_#{node_name}"
+        me = g.send(cluster_name) do |sub|
+          sub[rank: 'same', label: display_name, color: 'black']
+          sub.send("#{cluster_name}_left").send('[]', shape: 'point', color: type_color(types))
+          sub.send("#{cluster_name}_right").send('[]', shape: 'point')
+        end
+        left.link_to(g, me.send("#{cluster_name}_left"))
+        right.link_to(g, me.send("#{cluster_name}_right"))
+        @__graph_node = me.send("#{cluster_name}_left")
+      end
+
+      def types
+        left.types
+      end
+
+      def arguments
+        options.values
+      end
+    end
+
+    BINARY_OPERATOR_TYPES = %w(before after meets met_by started_by starts contains during overlaps overlapped_by finished_by finishes coincides except person_filter less_than less_than_or_equal equal not_equal greater_than greater_than_or_equal filter).map { |temp| [temp, "not_#{temp}"] }.flatten.map(&:to_sym)
+
+    def create(type, values)
+      return BinaryOperatorNode.new(type, values) if BINARY_OPERATOR_TYPES.include?(type)
+      DotNode.new(type, values)
+    end
+  end
+end

data/lib/conceptql/logger.rb

@@ -0,0 +1,10 @@
+require 'logger'
+module ConceptQL
+  def self.logger
+    @logger ||= begin
+      l = Logger.new('/tmp/cql.log')
+      l.level = Logger::DEBUG
+      l
+    end
+  end
+end

data/lib/conceptql/nodes/after.rb

@@ -0,0 +1,12 @@
+require_relative 'temporal_node'
+
+module ConceptQL
+  module Nodes
+    class After < TemporalNode
+      def where_clause
+        Proc.new { l__start_date > r__end_date }
+      end
+    end
+  end
+end
+

data/lib/conceptql/nodes/before.rb

@@ -0,0 +1,11 @@
+require_relative 'temporal_node'
+
+module ConceptQL
+  module Nodes
+    class Before < TemporalNode
+      def where_clause
+        Proc.new { l__end_date < r__start_date }
+      end
+    end
+  end
+end

data/lib/conceptql/nodes/binary_operator_node.rb

@@ -0,0 +1,41 @@
+require_relative 'node'
+
+module ConceptQL
+  module Nodes
+    # Base class for all nodes that take two streams, a left-hand and a right-hand
+    class BinaryOperatorNode < Node
+      def children
+        [left]
+      end
+
+      def graph_it(g, db)
+        left.graph_it(g, db)
+        right.graph_it(g, db)
+        cluster_name = "cluster_#{node_name}"
+        me = g.send(cluster_name) do |sub|
+          sub[rank: 'same', label: display_name, color: 'black']
+          sub.send("#{cluster_name}_left").send('[]', shape: 'point', color: type_color(types))
+          sub.send("#{cluster_name}_right").send('[]', shape: 'point')
+        end
+        left.link_to(g, me.send("#{cluster_name}_left"))
+        right.link_to(g, me.send("#{cluster_name}_right"))
+        @__graph_node = me.send("#{cluster_name}_left")
+      end
+
+      def display_name
+        self.class.name.split('::').last.titleize
+      end
+
+      private
+      def left
+        @left ||= options[:left]
+      end
+
+      def right
+        @right ||= options[:right]
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/nodes/casting_node.rb

@@ -0,0 +1,75 @@
+require_relative 'node'
+
+module ConceptQL
+  module Nodes
+    # Parent class of all casting nodes
+    #
+    # Subclasses must implement the following methods:
+    # - my_type
+    # - i_point_at
+    # - these_point_at_me
+    #
+    # i_point_at returns a list of types for which the node's table of origin
+    # has foreign_keys pointing to another table, e.g.:
+    # procedure_occurrence has an FK to visit_occurrence, so we'd put
+    # :visit_occurrence in the i_point_at array
+    #
+    # these_point_at_me is a list of types for which that type's table
+    # of origin has a FK  pointing to the current node's
+    # table of origin, e.g.:
+    # procedure_cost has an FK to procedure_occurrence so we'd
+    # put :procedure_cost in procedure_occurrence's these_point_at_me array
+    #
+    # Also, if a casting node is passed no streams, it will return all the
+    # rows in its table as results.
+    class CastingNode < Node
+      def types
+        [my_type]
+      end
+
+      def castables
+        (i_point_at + these_point_at_me)
+      end
+
+      def query(db)
+        return db.from(make_table_name(my_type)) if stream.nil?
+        base_query(db, stream.evaluate(db))
+      end
+
+      private
+
+      def base_query(db, stream_query)
+        if (stream.types & castables).length < stream.types.length
+          # We have a situation where one of the incoming streams
+          # isn't castable so we'll just return all rows for
+          # all people
+          db.from(make_table_name(my_type))
+            .where(person_id: db.from(stream_query).select_group(:person_id))
+        else
+          # Every type in the stream is castable, so let's setup a query that
+          # casts each type to a set of IDs, union those IDs and fetch
+          # them from the source table
+          my_ids = stream.types.map do |type|
+            cast_type(db, type, stream_query)
+          end.inject do |union_query, query|
+            union_query.union(query)
+          end
+
+          db.from(make_table_name(my_type))
+            .where(type_id(my_type) => my_ids)
+        end
+      end
+
+      def cast_type(db, type, stream_query)
+        query = if i_point_at.include?(type)
+          db.from(make_table_name(my_type))
+            .where(type_id(type) => db.from(stream_query.select_group(type_id(type))))
+        else
+          db.from(make_table_name(type))
+            .where(type_id(type) => db.from(stream_query.select_group(type_id(type))))
+        end
+        query.select(type_id(my_type))
+      end
+    end
+  end
+end

data/lib/conceptql/nodes/complement.rb

@@ -0,0 +1,16 @@
+require_relative 'pass_thru'
+
+module ConceptQL
+  module Nodes
+    class Complement < PassThru
+      def query(db)
+        child = children.first
+        child.types.map do |type|
+          select_it(db.from(make_table_name(type)).exclude(type_id(type) => child.evaluate(db).select(type_id(type)).from_self.exclude(type_id(type) => nil)), [type])
+        end.inject do |union_query, q|
+          union_query.union(q, all: true)
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/nodes/concept.rb

@@ -0,0 +1,38 @@
+require_relative 'node'
+require_relative '../tree'
+
+module ConceptQL
+  module Nodes
+    class Concept < Node
+      attr :statement
+      def query(db)
+        set_statement(db)
+        stream.evaluate(db)
+      end
+
+      def children
+        @children ||= [ Tree.new.root(self) ]
+      end
+
+      def graph_prep(db)
+        set_statement(db)
+        @arguments = [description(db)]
+      end
+
+      private
+      def set_statement(db)
+        @statement ||= db[:concepts].where(concept_id: arguments.first).select_map(:statement).first.tap { |f| ConceptQL.logger.debug f.inspect }.to_hash
+      end
+
+      def description(db)
+        @description ||= db[:concepts].where(concept_id: arguments.first).select_map(:label).first
+      end
+
+      def db
+        tree.db
+      end
+    end
+  end
+end
+
+

data/lib/conceptql/nodes/condition_type.rb

@@ -0,0 +1,63 @@
+require_relative 'node'
+
+module ConceptQL
+  module Nodes
+    # Represents a node that will grab all conditions that match the
+    # condition type passed in
+    #
+    # Condition Type represents which position the condition held in
+    # the raw data, e.g. primary inpatient header or 15th outpatient detail
+    #
+    # Multiple types can be specified at once
+    class ConditionType < Node
+      def types
+        [:condition_occurrence]
+      end
+
+      def query(db)
+        db.from(:condition_occurrence_with_dates)
+          .where(condition_type_concept_id: condition_occurrence_type_concept_ids)
+      end
+
+    private
+      def condition_occurrence_type_concept_ids
+        arguments.map do |arg|
+          to_concept_id(arg.to_s)
+        end.flatten
+      end
+
+      def to_concept_id(ctype)
+        ctype = ctype.to_s.downcase
+        position = nil
+        if ctype =~ /(\d|_primary)$/
+          parts = ctype.split('_')
+          position = parts.pop.to_i
+          position -= 1 if ctype =~ /^outpatient/
+          ctype = parts.join('_')
+        end
+        retval = concept_ids[ctype.to_sym]
+        return retval[position] if position
+        return retval
+      end
+
+      def concept_ids
+        @concept_ids ||= begin
+          hash = {
+            inpatient_detail: (38000183..38000198).to_a,
+            inpatient_header: (38000199..38000214).to_a,
+            outpatient_detail: (38000215..38000229).to_a,
+            outpatient_header: (38000230..38000244).to_a,
+            ehr_problem_list: [38000245],
+            condition_era_0_day_window: [38000246],
+            condition_era_30_day_window: [38000247]
+          }
+          hash[:inpatient] = hash[:inpatient_detail] + hash[:inpatient_header]
+          hash[:outpatient] = hash[:outpatient_detail] + hash[:outpatient_header]
+          hash[:condition_era] = hash[:condition_era_0_day_window] + hash[:condition_era_30_day_window]
+          hash
+        end
+      end
+    end
+  end
+end
+