conceptql 0.2.0 → 0.3.0

data/lib/conceptql/behaviors/provenanceable.rb

@@ -0,0 +1,30 @@
+module ConceptQL
+  module Provenanceable
+    def to_concept_id(ctype)
+      ctype = ctype.to_s.downcase
+      return ctype.to_i unless ctype.to_i.zero?
+      position = nil
+      if ctype =~ /(\d|_primary)$/ && ctype.count('_') > 1
+        parts = ctype.split('_')
+        position = parts.pop.to_i
+        position -= 1 if needs_adjustment?(ctype)
+        ctype = parts.join('_')
+      end
+      retval = concept_ids[ctype.to_sym]
+      return retval[position] if position
+      return retval
+    end
+
+    def concept_ids
+      @concept_ids ||= Psych.load_file(config_dir + 'provenance.yml')
+    end
+
+    def config_dir
+      Pathname.new(__FILE__).dirname + '..' + '..' + '..' + 'config'
+    end
+
+    def needs_adjustment?(ctype)
+      ctype =~ /^outpatient/ || ctype =~ /^carrier/
+    end
+  end
+end

data/lib/conceptql/cli.rb

@@ -1,9 +1,12 @@
+$: << "lib"
+
 require 'thor'
 require 'sequelizer'
-require 'psych'
 require 'json'
 require 'pp'
+require 'csv'
 require_relative 'query'
+require_relative 'knitter'
 
 module ConceptQL
   class CLI < Thor
@@ -36,16 +39,23 @@ module ConceptQL
 
     desc 'run_statement statement_file', 'Reads the ConceptQL statement from the statement file and executes it against the DB'
     def run_statement(statement_file)
-      q = ConceptQL::Query.new(db(options), criteria_from_file(statement_file))
+      q = cdb(options).query(criteria_from_file(statement_file))
       puts q.sql
-      puts q.statement.to_yaml
-      pp q.all
+      puts JSON.pretty_generate(q.statement)
+      pp q.query.all
+    end
+
+    desc "annotate_statement", "Reads in a statement and annotates it"
+    def annotate_statement(statement_file)
+      q = ConceptQL::Query.new(db(options), criteria_from_file(statement_file))
+      pp q.annotate
     end
 
     desc 'show_graph statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
     option :watch_file
     def show_graph(file)
       graph_it(criteria_from_file(file))
+      system('open /tmp/graph.pdf')
     end
 
     desc 'show_and_tell_file statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph, then executes the statement against the DB'
@@ -57,26 +67,68 @@ module ConceptQL
 
     desc 'fake_graph file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
     def fake_graph(file)
-      require_relative 'fake_grapher'
-      ConceptQL::FakeGrapher.new.graph_it(criteria_from_file(file), '/tmp/graph')
+      require_relative 'fake_annotater'
+      annotated = ConceptQL::FakeAnnotater.new(criteria_from_file(file)).annotate
+      pp annotated
+      ConceptQL::AnnotateGrapher.new.graph_it(annotated, '/tmp/graph.pdf')
       system('open /tmp/graph.pdf')
     end
 
     desc 'metadata', 'Generates the metadata.js file for the JAM'
     def metadata
-      File.write('/tmp/metadata.js', "var metadata = #{ConceptQL::Nodifier.new.to_metadata.to_json};")
-    end
-
-    desc 'convert', 'Converts from hash-based syntax to list-based syntax'
-    def convert(file)
-      require 'conceptql/converter'
-      require 'json'
-      begin
-        puts JSON.pretty_generate(ConceptQL::Converter.new.convert(criteria_from_file(file)))
-      rescue
-        puts "Couldn't convert #{file}"
-        puts $!.message
-        puts $!.backtrace.join("\n")
+      File.write('metadata.js', "$metadata = #{ConceptQL.metadata(warn: true).to_json};")
+      File.write('metadata.json', ConceptQL.metadata.to_json)
+      if system("which", "json_pp", out: File::NULL, err: File::NULL)
+        system("cat metadata.json | json_pp", out: "metadata_pp.json")
+      end
+    end
+
+    desc 'selection_operators', 'Generates a TSV of all the selection operators'
+    def selection_operators
+      require 'csv'
+      CSV.open('selection_operators.tsv', 'w', col_sep: "\t") do |csv|
+        csv << ["name"]
+        ConceptQL.metadata[:operators].values.select { |v| v[:basic_type] == :selection }.map { |v| v[:preferred_name] }.each do |name|
+          csv << [name]
+        end
+      end
+
+      CSV.open('domains.tsv', 'w', col_sep: "\t") do |csv|
+        csv << ["name"]
+        ConceptQL::Operators::TABLE_COLUMNS.each do |domain, columns|
+          next unless columns.include?(:person_id)
+          next if domain.to_s =~ /era\Z/
+          csv << [domain]
+        end
+      end
+    end
+
+    desc 'knit', 'Processes ConceptQL fenced code segments and produces a Markdown file'
+    option :ignore_cache, type: :boolean
+    def knit(file)
+      opts = {}
+      if options[:ignore_cache]
+        opts[:cache_options] = { ignore: true }
+      end
+      ConceptQL::Knitter.new(ConceptQL::Database.new(db), file, opts).knit
+    end
+
+    desc 'dumpit', 'Dumps out test data into CSVs'
+    def dumpit(path)
+      path = Pathname.new(path)
+      path.mkpath unless path.exist?
+      headers_path = path + 'headers'
+      headers_path.mkpath unless headers_path.exist?
+      db.tables.each do |table|
+        puts "Dumping #{table}..."
+        ds = db[table]
+        rows = ds.select_map(ds.columns)
+        CSV.open(path + "#{table}.csv", "wb") do |csv|
+          rows.each { |row| csv << row }
+        end
+        CSV.open(headers_path + "#{table}.csv", "wb") do |csv|
+          csv << ds.columns
+        end
       end
     end
 
@@ -93,7 +145,7 @@ module ConceptQL
     desc 'show_db_graph conceptql_id', 'Shows a graph for the conceptql statement represented by conceptql_id in the db specified by db_url'
     def show_db_graph(conceptql_id)
       result = fetch_conceptql(conceptql_id, options)
-      graph_it(result[:statement].to_hash, db, result[:label])
+      graph_it(result[:statement], db, result[:label])
     end
 
     def fetch_conceptql(conceptql_id)
@@ -105,10 +157,9 @@ module ConceptQL
     def show_and_tell(statement, options, title = nil)
       my_db = db(options)
       q = ConceptQL::Query.new(my_db, statement)
-      puts 'YAML'
-      puts q.statement.to_yaml
+      statement = q.annotate
       puts 'JSON'
-      puts JSON.pretty_generate(q.statement)
+      puts JSON.pretty_generate(statement)
       graph_it(statement, title)
       STDIN.gets
       puts q.sql
@@ -123,20 +174,9 @@ module ConceptQL
     end
 
     def graph_it(statement, title = nil)
-      require_relative 'graph'
-      require_relative 'tree'
-      conn = db(options)
-      ConceptQL::Graph.new(statement,
-                           dangler: true,
-                           title: title,
-                           db: conn
-                          ).graph_it('/tmp/graph')
-      system('open /tmp/graph.pdf')
-      if options[:watch_file]
-        require_relative 'debugger'
-        debugger = ConceptQL::Debugger.new(statement, db: conn, watch_ids: File.readlines(options[:watch_file]).map(&:to_i))
-        debugger.capture_results('/tmp/debug.xlsx')
-      end
+      my_db = db(options)
+      q = ConceptQL::Query.new(my_db, statement).annotate
+      ConceptQL::AnnotateGrapher.new.graph_it(q, '/tmp/graph.pdf')
     end
 
     def criteria_from_file(file)
@@ -151,6 +191,10 @@ module ConceptQL
     def filtered(results)
       results.each { |r| r.delete_if { |k,v| v.nil? } }
     end
+
+    def cdb(options)
+      ConceptQL::Database.new(db(options))
+    end
   end
 end
 

data/lib/conceptql/database.rb

@@ -0,0 +1,42 @@
+require 'facets/hash/revalue'
+
+module ConceptQL
+  class Database
+    attr :db, :opts
+
+    def initialize(db, opts={})
+      @db = db
+      db_type = :impala
+      if db
+        extensions.each do |extension|
+          db.extension extension
+        end
+        db_type = db.database_type.to_sym
+      end
+
+      @opts = opts.revalue { |v| v ? v.to_sym : v }
+      @opts[:data_model] ||= :omopv4
+      @opts[:database_type] ||= db_type
+      db.set(db_opts) if db.respond_to?(:set)
+    end
+
+    def query(statement, opts={})
+      Query.new(db, statement, @opts.merge(opts))
+    end
+
+    def db_opts
+      db_opts = {}
+      if opts[:database_type] == :impala
+        db_opts.merge!(runtime_filter_mode: "OFF")
+        if mem_limit = (opts[:impala_mem_limit] || ENV['IMPALA_MEM_LIMIT'])
+          db_opts.merge!(mem_limit: mem_limit)
+        end
+      end
+      db_opts
+    end
+
+    def extensions
+      [:date_arithmetic, :error_sql]
+    end
+  end
+end

data/lib/conceptql/date_adjuster.rb

@@ -3,9 +3,13 @@ require_relative '../conceptql'
 module ConceptQL
   # Used to translate a string of terse date adjustments into a set of adjustments that are compatible with most RDBMSs
   class DateAdjuster
-    attr :str
-    def initialize(str)
-      @str = str
+
+    VALID_INPUT = /\A#{Regexp.union([/START/i, /END/i, /\d{4}-\d{2}-\d{2}/, /([-+]?\d+[dmy]?)+/, /\s*/])}\z/
+
+    attr :str, :manipulator
+    def initialize(str, opts = {})
+      @str = str || ""
+      @manipulator = opts[:manipulator] || Sequel
     end
 
     # Returns an array of strings that represent date modifiers
@@ -13,6 +17,21 @@ module ConceptQL
       @adjustments ||= parse(str)
     end
 
+    def adjust(column, reverse=false)
+      return Sequel.expr(:end_date) if str.downcase == 'end'
+      return Sequel.expr(:start_date) if str.downcase == 'start'
+      return Sequel.cast(Date.parse(str).strftime('%Y-%m-%d'), Date) if str =~ /^\d{4}-\d{2}-\d{2}$/
+      adjusted_date = adjustments.inject(Sequel.expr(column)) do |sql, (units, quantity)|
+        quantity *= -1 if reverse
+        if quantity > 0
+          manipulator.date_add(sql, units => quantity)
+        else
+          manipulator.date_sub(sql, units => quantity.abs)
+        end
+      end
+      Sequel.cast(adjusted_date, Date)
+    end
+
     private
     def lookup
       {
@@ -23,7 +42,6 @@ module ConceptQL
     end
 
     def parse(str)
-      ConceptQL.logger.debug(str)
       return [] if str.nil? || str.empty?
       return [[lookup['d'], str.to_i]] if str.match(/^[-+]?\d+$/)
       str.downcase.scan(/([-+]?\d*[dmy])/).map do |adjustment|

data/lib/conceptql/fake_annotater.rb

@@ -0,0 +1,129 @@
+require 'facets/array/recurse'
+require 'facets/hash/deep_rekey'
+
+module ConceptQL
+  class FakeAnnotater
+    attr :statement
+
+    DOMAINS = {
+      # Conditions
+      condition: :condition_occurrence,
+      primary_diagnosis: :condition_occurrence,
+      icd9: :condition_occurrence,
+      icd10: :condition_occurrence,
+      condition_type: :condition_occurrence,
+      medcode: :condition_occurrence,
+
+      # Procedures
+      procedure: :procedure_occurrence,
+      cpt: :procedure_occurrence,
+      drg: :procedure_occurrence,
+      hcpcs: :procedure_occurrence,
+      icd9_procedure: :procedure_occurrence,
+      medcode_procedure: :procedure_occurrence,
+
+      # Procedure Cost
+      procedure_cost: :procedure_cost,
+
+      # Visits
+      visit_occurrence: :visit_occurrence,
+      place_of_service_code: :visit_occurrence,
+
+      # Person
+      person: :person,
+      gender: :person,
+      race: :person,
+
+      # Payer
+      payer: :payer_plan_period,
+
+      # Death
+      death: :death,
+
+      # Observation
+      loinc: :observation,
+      from_seer_visits: :observation,
+      to_seer_visits: :observation,
+
+      # Drug
+      drug_exposure: :drug_exposure,
+      rxnorm: :drug_exposure,
+      drug_cost: :drug_cost,
+      drug_type_concept_id: :drug_exposure,
+      drug_type_concept: :drug_exposure,
+      prodcode: :drug_exposure,
+
+      # Date Operators
+      date_range: :date,
+
+      # Miscelaneous operators
+      concept: :misc,
+      vsac: :misc,
+      numeric: :person,
+      algorithm: :misc,
+    }
+
+    def initialize(statement)
+      @statement = statement
+    end
+
+    def annotate
+      traverse(statement)
+    end
+
+    def traverse(stmt)
+      stmt.recurse(Array, Hash) do |arr_or_hash|
+        if arr_or_hash.is_a?(Array)
+          arr_or_hash.unshift(arr_or_hash.shift.to_sym)
+          domains = DOMAINS[arr_or_hash.first.to_sym]
+          unless domains
+            domains = previous_domains(arr_or_hash)
+          else
+            domains = [domains].flatten
+          end
+          annotate_hash = if arr_or_hash.last.is_a?(Hash)
+            arr_or_hash.last[:annotation] ||= {}
+          else
+            annotate_hash = {}
+            arr_or_hash.push(annotation: annotate_hash)
+            annotate_hash
+          end
+          annotate_hash[:counts] = {}
+          domains.each do |domain|
+            annotate_hash[:counts][domain] = {}
+          end
+          save_domains(arr_or_hash)
+        else
+          arr_or_hash.deep_rekey!
+          arr_or_hash[:annotation] ||= {}
+        end
+        arr_or_hash
+      end
+    end
+
+    def previous_domains(arr)
+      extract = if arr.first == :recall
+        [fetch_domains(arr[1])].compact
+      elsif arr.last.is_a?(Hash) && arr.last[:left]
+        [arr.last[:left]]
+      else
+        arr.select { |e| e.is_a?(Array) }
+      end
+      extract.map { |e| e.last[:annotation][:counts].keys }.flatten.compact.uniq
+    end
+
+    def fetch_domains(label)
+      recorded_domains[label]
+    end
+
+    def save_domains(op)
+      label = op.last[:label]
+      return unless label
+      recorded_domains[label] = op
+    end
+
+    def recorded_domains
+      @recorded_domains ||= {}
+    end
+  end
+end

data/lib/conceptql/knitter.rb

@@ -0,0 +1,221 @@
+require 'digest'
+require 'json'
+require_relative 'annotate_grapher'
+require_relative 'fake_annotater'
+
+module ConceptQL
+  class Knitter
+    attr :file, :db, :options
+    CONCEPTQL_CHUNK_START = /```ConceptQL/
+    RESULT_KEYS = %i(person_id criterion_id criterion_domain start_date end_date source_value)
+
+    def initialize(db, file, options = {})
+      @file = Pathname.new(file)
+      raise "File must end in .md.cql!" unless file =~ /\.md\.cql$/
+      @db = db
+      @options = options.dup
+    end
+
+    def knit
+      lines = file.readlines
+      chunks = lines.slice_before { |l| l =~ CONCEPTQL_CHUNK_START }.to_a
+      outputs = []
+      outputs << chunks.shift unless chunks.first =~ CONCEPTQL_CHUNK_START
+      outputs += chunks.map do |chunk|
+        cql, *remainder = chunk.slice_after { |l| l =~ /^```\n$/ }.to_a
+        cql = ConceptQLChunk.new(cql, cache, self)
+        [cql.output, remainder].flatten
+      end.flatten
+      File.write(file.to_s.sub(/.cql$/, ''), outputs.join)
+    end
+
+    def diagram_dir
+      @diagram_dir ||= (dir + file.basename('.md.cql')).tap { |d| d.rmtree if d.exist? ; d.mkpath }
+    end
+
+    def diagram_relative_path
+      @diagram_relative_path ||= diagram_dir.basename
+    end
+
+    def diagram_path(stmt, &block)
+      png_contents = cache.fetch_or_create(stmt.inspect, &block)
+      file_name = (cache.hash_it(stmt) + ".png")
+      new_path = (diagram_dir + file_name)
+      new_path.write(png_contents)
+      diagram_relative_path + file_name
+    end
+
+    private
+    class ConceptQLChunk
+      attr :lines, :cache, :knitter
+      def initialize(lines, cache, knitter)
+        @cache = cache
+        @lines = lines.to_a
+        @knitter = knitter
+      end
+
+      def output
+        diagram_markup
+        cache.fetch_or_create(lines.join) do
+          create_output
+        end
+      end
+
+      def titleize(title)
+        return '' unless title
+        title.map(&:strip).join(" ").gsub('#', '')
+      end
+
+      def make_statement_and_title
+        lines.shift
+        lines.pop
+        title, statement = lines.slice_before { |l| l =~ /^\s*#/ }.to_a
+        if statement.nil?
+          statement = title
+          title = nil
+        end
+        @statement = eval(statement.join)
+        @title = titleize(title)
+      end
+
+      def statement
+        @statement || make_statement_and_title
+        @statement
+      end
+
+      def title
+        @title || make_statement_and_title
+        @title
+      end
+
+      def create_output
+        output = []
+        output << title unless title.empty?
+        output << '' unless title.empty?
+        output << "```JSON"
+        output << ''
+        output << statement.to_json
+        output << ''
+        output << "```"
+        output << ''
+        output << diagram_markup
+        output << ''
+        output << table
+        output << ''
+        output.compact.join("\n")
+      end
+
+      def diagram_markup
+        diagram_path = diagram(statement)
+        return "![#{title}](#{diagram_path})" if diagram_path
+        nil
+      end
+
+      def table
+        results = nil
+        begin
+          results = query(statement).query.limit(10).all
+        rescue
+          puts $!.message
+          puts $!.backtrace.join("\n")
+        end
+
+        if results.nil?
+          "```No Results.  Statement is experimental.```"
+        else
+          if results.empty?
+            "```No Results found.```"
+          else
+            resultify(results)
+          end
+        end
+      end
+
+      def resultify(results)
+        rows = []
+        rows << rowify(RESULT_KEYS)
+        rows << rowify(RESULT_KEYS.map { |c| c.to_s.gsub(/./, '-')})
+        results.each do |result|
+          rows << rowify(result.values_at(*RESULT_KEYS))
+        end
+        rows.join("\n")
+      end
+
+      def rowify(columns)
+        "| #{columns.join(" | ")} |"
+      end
+
+      def diagram(stmt)
+        knitter.diagram_path(stmt) do |path_name|
+          annotated = nil
+          begin
+            annotated = query(stmt).annotate
+          rescue
+            puts $!.message
+            puts $!.backtrace.join("\n")
+            p stmt
+            annotated = FakeAnnotater.new(stmt).annotate
+          end
+          ConceptQL::AnnotateGrapher.new.graph_it(annotated, path_name, output_type: 'png')
+        end
+      end
+
+      def query(stmt)
+        knitter.db.query(stmt)
+      end
+    end
+
+    class Cache
+      attr :db, :options, :file
+
+      def initialize(db, file, options = {})
+        @db = db
+        @file = file
+        @options = options.nil? ? {} : options.dup
+        remove_cache if @options[:ignore]
+      end
+
+      def remove_cache
+        cache_dir.rmtree
+        @cache_dir = nil
+      end
+
+      def cache_file_path(str)
+        cache_dir + hash_it(str)
+      end
+
+      def fetch_or_create(str, &block)
+        cache_file = cache_file_path(str)
+        return cache_file.read if cache_file.exist?
+        #p ["cache miss for", str, cache_file]
+        output = block.call(cache_file)
+        cache_file.write(output) unless cache_file.exist?
+        cache_file.read
+      end
+
+      def cache_dir
+        @cache_dir ||= (file.dirname + ".#{hash_it(hash_fodder)}").tap { |d| d.mkpath }
+      end
+
+      def hash_fodder
+        (db_opts.inspect + file.basename.to_s)
+      end
+
+      def db_opts
+        db.opts.values_at(*%i(adapter user password host database search_path))
+      end
+
+      def hash_it(str)
+        Digest::SHA256.hexdigest("#{str}")
+      end
+    end
+
+    def dir
+      file.dirname
+    end
+
+    def cache
+      @cache ||= Cache.new(db.db, file, options[:cache_options])
+    end
+  end
+end