conceptql 0.3.0 → 0.4.0

data/lib/conceptql/database.rb

@@ -1,42 +1,214 @@
-require 'facets/hash/revalue'
+# frozen_string_literal: true
+
+require "digest"
+require "logger"
+require_relative "lexicon"
 
 module ConceptQL
   class Database
-    attr :db, :opts
+    attr_reader :db, :opts
+
+    @lexicon_mutex = Mutex.new
 
-    def initialize(db, opts={})
+    EXTENSIONS = %i[
+      date_arithmetic
+      error_sql
+      make_readyable
+      null_dataset
+      pg_ctas_explain
+      pg_vacuum_table
+      smart_select_remove
+      sql_comments
+      usable
+    ].freeze
+
+    def initialize(db, opts = {})
       @db = db
-      db_type = :impala
-      if db
-        extensions.each do |extension|
+      # Symbolize all keys and values
+      @opts = ConceptQL::Utils.rekey(opts, rekey_values: true)
+
+      @opts[:data_model] ||= (ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym
+
+      self.class.db_extensions(db, @opts[:data_model], @opts[:use_cold_col])
+      db_type = db ? db.database_type.to_sym : :postgres
+
+      @lexicon = opts[:lexicon]
+      if @lexicon && @opts[:cache_lookups]
+        raise ArgumentError, "cache_lookups needs the Database to build its own Lexicon; do not pass lexicon:"
+      end
+
+      @opts[:database_type] ||= (ENV["CONCEPTQL_DATABASE_TYPE"] || db_type).to_sym
+      @opts[:scope_opts] = {
+        force_temp_tables: opts.fetch(:force_temp_tables, ENV["CONCEPTQL_FORCE_TEMP_TABLES"] == "true"),
+        scratch_database: opts.fetch(:scratch_database, ENV["DOCKER_SCRATCH_DATABASE"])
+      }.merge(opts[:scope_opts] || {})
+      @opts[:scope_opts][:lexicon] = lexicon
+
+      @shared_utilization_tables = {}
+      @shared_utilization_mutex = Mutex.new
+      @source_tables = {} if @opts[:cache_lookups]
+    end
+
+    def query(statement, opts = {})
+      opts[:scope_opts] = (@opts[:scope_opts] || {}).merge(opts.delete(:scope_opts) || {})
+      if @opts[:share_utilization] && opts[:scope_opts][:table_prefix].to_s.empty?
+        raise ArgumentError, "share_utilization needs a table_prefix in scope_opts, to name its tables"
+      end
+
+      Query.new(self, ConceptQL::Utils.rekey(statement), @opts.merge(opts))
+    end
+
+    def data_model
+      @data_model ||= DataModel.get(opts[:data_model], nodifier: nodifier)
+    end
+
+    def nodifier
+      @nodifier ||= Nodifier.new(self)
+    end
+
+    def base_data_model
+      data_model.base
+    end
+
+    class << self
+      def db_extensions(db, data_model, use_cold_col)
+        return unless db
+
+        EXTENSIONS.each do |extension|
           db.extension extension
         end
-        db_type = db.database_type.to_sym
+
+        use_cold_col = db.is_a?(Sequel::Mock::Database) if use_cold_col.nil?
+        return unless use_cold_col
+
+        db.extension(:cold_col)
+        db.load_schema(ConceptQL.schemas_dir / "#{data_model}.yml")
+        db.load_schema(ConceptQL.schemas_dir / "ohdsi_vocabs.yml")
+      end
+
+      def lexicon_db
+        @lexicon_mutex.synchronize do
+          @lexicon_db = make_lexicon_db unless defined?(@lexicon_db)
+        end
+        @lexicon_db
       end
 
-      @opts = opts.revalue { |v| v ? v.to_sym : v }
-      @opts[:data_model] ||= :omopv4
-      @opts[:database_type] ||= db_type
-      db.set(db_opts) if db.respond_to?(:set)
+      # The LEXICON_URL database, or nil when it is unset. Lexicon tries the
+      # data db first and, when neither has the vocabulary tables, falls back
+      # to LexiconNoDB, so no database (and no sqlite3 gem) is needed here.
+      def make_lexicon_db
+        return unless ENV["LEXICON_URL"]
+
+        db_opts = {}
+        if ENV["CONCEPTQL_LOG_LEXICON"]
+          log_path = Pathname.new("log") / "conceptql_lexicon.log"
+          log_path.dirname.mkpath
+          db_opts[:logger] = Logger.new(log_path)
+        end
+        lexicon_db = Sequel.connect(ENV["LEXICON_URL"], db_opts)
+        lexicon_db.extension(:date_arithmetic)
+        lexicon_db
+      end
     end
 
-    def query(statement, opts={})
-      Query.new(db, statement, @opts.merge(opts))
+    # Whether a data model's source table exists in db. With cache_lookups on,
+    # a table that exists is remembered per db and table for the life of this
+    # Database, as the Lexicon keeps its lookups. A missing table is checked
+    # again on every call: Sequel's table_exists? also answers false on a
+    # database error, and one transient error must not turn off the database
+    # checks for the whole export. Only the data model's fixed source tables
+    # come here, never a table that conceptql creates during a run.
+    def source_table_exists?(db, table)
+      return db.table_exists?(table) unless @source_tables
+
+      key = [ db, table ]
+      @source_tables.fetch(key) { db.table_exists?(table) && (@source_tables[key] = true) }
     end
 
-    def db_opts
-      db_opts = {}
-      if opts[:database_type] == :impala
-        db_opts.merge!(runtime_filter_mode: "OFF")
-        if mem_limit = (opts[:impala_mem_limit] || ENV['IMPALA_MEM_LIMIT'])
-          db_opts.merge!(mem_limit: mem_limit)
-        end
+    def lexicon
+      @lexicon ||= Lexicon.new(self.class.lexicon_db, db, cache: @opts[:cache_lookups])
+    end
+
+    # This Database's operators for a data model, as a frozen name => class
+    # Hash: a vocabulary operator for every vocabulary in config/vocabularies.csv
+    # (plus the custom file) and in this Database's lexicon, with the built-in
+    # operators merged over them. A built-in wins a name clash (gender, race and
+    # ethnicity are also vocabularies). Built lazily, once per data model.
+    #
+    # The data model is an argument because a Nodifier can be given one that
+    # differs from this Database's.
+    def operators(data_model = opts[:data_model])
+      data_model = data_model.to_sym
+      @operators ||= {}
+      @operators[data_model] ||= begin
+        vocabulary_operators = Vocabularies::DynamicVocabularies.new(lexicon).operators(data_model)
+        vocabulary_operators.merge(Operators.static_operators.fetch(data_model)).freeze
+      end
+    end
+
+    # Builds the table that the name_concepts_table option names. It holds
+    # the concept rows whose ids appear in the data columns that the
+    # display-name joins read, so those joins hash a small table instead of
+    # the whole concept table. The caller drops it.
+    def create_name_concepts_table
+      table = opts[:name_concepts_table]
+      raise ArgumentError, "create_name_concepts_table needs the name_concepts_table option" unless table
+
+      ids = data_model.name_concept_columns.flat_map do |source, columns|
+        next [] unless db.table_exists?(source)
+
+        (columns & db[source].columns).map { |column| db[source].exclude(column => nil).select(Sequel.as(column, :id)) }
+      end
+      raise ArgumentError, "no data columns to read for data model #{opts[:data_model]}" if ids.empty?
+
+      db.create_table(table, as: data_model.concepts_table(db).where(id: ids.reduce { |all, one| all.union(one) }))
+    end
+
+    # The share_utilization option: the name of the table that holds a
+    # Utilizable operator's list (its select_it(query(db)), passed as key).
+    # The key's SQL contains the window table and adjustments, so two
+    # different windows get two tables. The name is a Symbol, because
+    # sequel-duckdb's table_exists? reads a Sequel::SQL::Identifier wrong.
+    def shared_utilization_table(key, scope)
+      name = :"#{scope.opts[:table_prefix]}_util_#{Digest::MD5.hexdigest(key.sql)[0, 16]}"
+      scope.scratch_database.to_s.empty? ? name : Sequel.qualify(scope.scratch_database.to_sym, name)
+    end
+
+    # The sql_statements entry that creates a shared_utilization_table, or nil
+    # when the table exists. The caller runs the entries in order. Nothing is
+    # claimed: every call lists a table until it exists.
+    def shared_utilization_statement(name, key)
+      return if db.table_exists?(name)
+
+      @shared_utilization_mutex.synchronize { @shared_utilization_tables[key.sql] = name }
+      [ name, key.sql ]
+    end
+
+    # Creates a shared_utilization_table that does not exist yet, when a
+    # query first reads rows through conceptql. Another thread that reads the
+    # same table waits here and then reads the table the first thread created.
+    def create_shared_utilization_table(name, key, rdbms, scope)
+      @shared_utilization_mutex.synchronize do
+        next if db.table_exists?(name)
+
+        db.create_table(name, rdbms.create_options(scope, key).merge(as: key))
+        rdbms.post_create(db, name)
+        @shared_utilization_tables[key.sql] = name
       end
-      db_opts
     end
 
-    def extensions
-      [:date_arithmetic, :error_sql]
+    # The shared tables that conceptql listed in sql_statements or created.
+    # conceptql never drops them: the caller does.
+    def shared_tables
+      @shared_utilization_mutex.synchronize { @shared_utilization_tables.values }
+    end
+
+    def database_type
+      @opts[:database_type]
+    end
+
+    def type_for(column_name)
+      Scope::COLUMN_TYPES.fetch(column_name)
     end
   end
 end

data/lib/conceptql/date_adjuster.rb

@@ -1,61 +1,91 @@
-require_relative '../conceptql'
+# frozen_string_literal: true
+
+require_relative "../conceptql"
 
 module ConceptQL
   # Used to translate a string of terse date adjustments into a set of adjustments that are compatible with most RDBMSs
   class DateAdjuster
+    VALID_INPUT = /\A#{Regexp.union([ /START/i, /END/i, /\d{4}-\d{2}-\d{2}/, /[ers]*([-+]?[\ddwmy]+)+/i,
+                                     /\s*/ ])}\z/.freeze
+
+    attr_reader :op, :str, :manipulator
 
-    VALID_INPUT = /\A#{Regexp.union([/START/i, /END/i, /\d{4}-\d{2}-\d{2}/, /([-+]?\d+[dmy]?)+/, /\s*/])}\z/
+    def initialize(op, str, opts = {})
+      str = str.strftime("%Y-%m-%d") if str.respond_to?(:strftime)
+      raise "Invalid adjustment string: #{str.pretty_inspect}" unless str.nil? || str =~ VALID_INPUT
 
-    attr :str, :manipulator
-    def initialize(str, opts = {})
+      @op = op
       @str = str || ""
       @manipulator = opts[:manipulator] || Sequel
     end
 
-    # Returns an array of strings that represent date modifiers
-    def adjustments
-      @adjustments ||= parse(str)
-    end
+    def adjust(column, reverse = false)
+      return Sequel.expr(:end_date) if str.downcase == "end"
+      return Sequel.expr(:start_date) if str.downcase == "start"
+      return op.rdbms.cast_date(Date.parse(str).strftime("%Y-%m-%d")) if str =~ /^\d{4}-\d{2}-\d{2}$/
 
-    def adjust(column, reverse=false)
-      return Sequel.expr(:end_date) if str.downcase == 'end'
-      return Sequel.expr(:start_date) if str.downcase == 'start'
-      return Sequel.cast(Date.parse(str).strftime('%Y-%m-%d'), Date) if str =~ /^\d{4}-\d{2}-\d{2}$/
-      adjusted_date = adjustments.inject(Sequel.expr(column)) do |sql, (units, quantity)|
-        quantity *= -1 if reverse
-        if quantity > 0
+      origin_column = column
+
+      chrs = str.chars
+      while (chr = chrs.shift) =~ /[res]/i
+        if chr.downcase == "r"
+          reverse = true
+        else
+          origin_column = chr.downcase == "e" ? :end_date : :start_date
+          origin_column = Sequel.qualify(column.table.to_sym, origin_column) if column.respond_to?(:table)
+        end
+      end
+
+      adjusted_date = adjustments(reverse).inject(Sequel.expr(origin_column)) do |sql, (units, quantity)|
+        # Turns out weeks aren't supported in Sequel, so we'll just multiply
+        # the number of weeks by 7 and adjust the date by that number of days
+        if units == :weeks
+          units = :days
+          quantity *= 7
+        end
+
+        if quantity.positive?
           manipulator.date_add(sql, units => quantity)
         else
           manipulator.date_sub(sql, units => quantity.abs)
         end
       end
-      Sequel.cast(adjusted_date, Date)
+      op.rdbms.cast_date(adjusted_date)
+    end
+
+    # Returns an array of strings that represent date modifiers
+    def adjustments(reverse = false)
+      @adjustments ||= parse(str, reverse)
     end
 
     private
+
     def lookup
       {
-        'y' => :years,
-        'm' => :months,
-        'd' => :days
+        "y" => :years,
+        "m" => :months,
+        "w" => :weeks,
+        "d" => :days
       }
     end
 
-    def parse(str)
+    def parse(str, reverse)
       return [] if str.nil? || str.empty?
-      return [[lookup['d'], str.to_i]] if str.match(/^[-+]?\d+$/)
-      str.downcase.scan(/([-+]?\d*[dmy])/).map do |adjustment|
+      return [ [ lookup["d"], str.to_i ] ] if str.match(/^[-+]?\d+$/)
+
+      str.downcase.scan(/([-+]?\d*[dwmy])/).map do |adjustment|
         adjustment = adjustment.first
+
         quantity = 1
         if adjustment.match(/\d/)
           quantity = adjustment.to_i
-        else
-          if adjustment.chars.first == '-'
-            quantity = -1
-          end
+        elsif adjustment.chars.first == "-"
+          quantity = -1
         end
+        quantity *= -1 if reverse
+
         unit = lookup[adjustment.chars.last]
-        [unit, quantity]
+        [ unit, quantity ]
       end
     end
   end

data/lib/conceptql/diagram.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require "json"
+require "active_support/core_ext/hash/indifferent_access"
+
+module ConceptQL
+  # Converts ConceptQL statements into trees a diagram renderer can draw.
+  #
+  # Two shapes are produced:
+  #
+  # * RenderTree -- the `conceptql-diagram/v1` format: one node per operator,
+  #   carrying the operator's metadata (base type, human name) and what
+  #   conceptql's own annotation worked out for it (output types, errors,
+  #   warnings and, optionally, per-domain counts).
+  # * JigsawTree -- the Jigsaw diagram editor's tree JSON (name, values,
+  #   parameters, children), as produced by the editor's ConceptqlAdapter.
+  module Diagram
+    FORMAT = "conceptql-diagram/v1"
+
+    class << self
+      # The full `conceptql-diagram/v1` document for one or more statements.
+      #
+      # Without `counts: true` no claims database is needed: pass a cdb built
+      # on a nil db (the default).
+      def render(statements, cdb: default_cdb, counts: false)
+        {
+          format: FORMAT,
+          statements: RenderTree.new(cdb, counts: counts).build_all(statements)
+        }
+      end
+
+      def default_cdb(data_model = nil)
+        opts = {}
+        opts[:data_model] = data_model.to_sym if data_model
+        ConceptQL::Database.new(nil, opts)
+      end
+
+      # A statement file may hold a single statement or a list of them.
+      # Mirrors JDE's ConceptqlAdapter#nested.
+      def statement_list(statements)
+        statements = JSON.parse(statements) if statements.is_a?(String)
+        return [] if statements == []
+        return statements if statements.first.is_a?(Array)
+
+        [ statements ]
+      end
+    end
+
+    class RenderTree
+      # Operator options that are structure or bookkeeping, not parameters
+      # a diagram should show.
+      STRUCTURAL_OPTIONS = %i[annotation name left right id].freeze
+
+      attr_reader :cdb
+
+      def initialize(cdb, counts: false)
+        @cdb = cdb
+        @counts = counts
+      end
+
+      def counts?
+        @counts
+      end
+
+      def build_all(statements)
+        Diagram.statement_list(statements).map { |statement| build(statement) }
+      end
+
+      def build(statement)
+        node(cdb.query(statement).annotate(skip_counts: !counts?))
+      end
+
+      private
+
+      def node(annotated)
+        name, *rest = annotated
+        hashes, rest = rest.partition { |v| v.is_a?(Hash) }
+        opts = hashes.reduce({}, :merge)
+        upstreams, values = rest.partition { |v| v.is_a?(Array) }
+        annotation = opts[:annotation] || {}
+
+        children = upstreams + opts.values_at(:left, :right).compact
+        node = {
+          name: name.to_s,
+          base: metadata_for(name)[:basic_type]&.to_s,
+          humanName: opts[:name] || metadata_for(name)[:preferred_name] || name.to_s
+        }
+        node[:vocabularyId] = metadata_for(name)[:preferred_name] if vocabulary?(name)
+        node.merge!(
+          outputTypes: output_types(annotation),
+          parameters: opts.except(*STRUCTURAL_OPTIONS),
+          values: values,
+          children: children.map { |child| node(child) }
+        )
+        node[:counts] = counts(annotation) if counts?
+        node[:errors] = annotation[:errors] if annotation[:errors].present?
+        node[:warnings] = annotation[:warnings] if annotation[:warnings].present?
+        node
+      end
+
+      # conceptql fills annotation[:counts] from Operator#domains, so its keys
+      # are the node's output types. Vocabulary metadata nests them
+      # ([["condition_occurrence"]]) where cast operators do not ([:person]),
+      # so flatten defensively. Sorted so that a counted run (whose keys arrive
+      # in database order) matches an uncounted one.
+      def output_types(annotation)
+        (annotation[:counts] || {}).keys.flatten.map(&:to_s).uniq.sort
+      end
+
+      def counts(annotation)
+        (annotation[:counts] || {})
+          .sort_by { |domain, _| domain.to_s }
+          .to_h { |domain, c| [ domain.to_s, { rows: Integer(c[:rows] || 0), n: Integer(c[:n] || 0) } ] }
+      end
+
+      def metadata_for(name)
+        operators_metadata[canonical_name(name)] || {}
+      end
+
+      def operators_metadata
+        @operators_metadata ||= ConceptQL.metadata(cdb)[:operators]
+      end
+
+      # A single-vocabulary operator (icd9, loinc, ...). Its metadata
+      # preferred_name is the vocabulary's ID (Entry#preferred_name:
+      # omopv5_id || id, e.g. "ICD9CM", "LOINC"), which a renderer can show
+      # when the humanName (the vocabulary's short name) is too long.
+      # Multi-vocabulary operators ("CPT or HCPCS") span several IDs, so they
+      # are not included.
+      def vocabulary?(name)
+        klass = operator_classes[canonical_name(name)]
+        !klass.nil? && klass <= ConceptQL::Operators::Vocabulary
+      end
+
+      def operator_classes
+        @operator_classes ||= cdb.operators(cdb.opts[:data_model])
+      end
+
+      def canonical_name(name)
+        name = name.to_s
+        operators_metadata.key?(name) ? name : aliases[name]
+      end
+
+      def aliases
+        @aliases ||= operators_metadata.each_with_object({}) do |(name, md), h|
+          Array(md[:aliases]).each { |a| h[a.to_s] = name }
+        end
+      end
+    end
+
+    # Port of the Jigsaw diagram editor's ConceptqlAdapter#to_jigsaw_json.
+    # The only change is where binary (filter) operators are looked up: the
+    # editor asked its Rails-wide cdb, this takes one (db-less by default).
+    class JigsawTree
+      BINARY_PARAMETERS = %w[left right].freeze
+      IGNORED_PARAMETERS = %w[id].freeze
+
+      class Node
+        def initialize(operator, binary)
+          @operator = operator
+          @binary = binary
+        end
+
+        def name
+          operator[0]
+        end
+
+        def values
+          operator[1..].select { |v| v.is_a?(String) || v.is_a?(Integer) }
+        end
+
+        def parameters
+          return unless operator[-1].is_a?(Hash)
+
+          operator[-1]
+            .reject { |k, _v| IGNORED_PARAMETERS.include?(k.to_s) }
+            .with_indifferent_access
+        end
+
+        def children
+          operator[1..right_bound].grep(Array)
+        end
+
+        def binary?
+          @binary.include?(name)
+        end
+
+        private
+
+        attr_reader :operator
+
+        def right_bound
+          parameters.present? ? -2 : -1
+        end
+      end
+
+      def initialize(statements, cdb: Diagram.default_cdb)
+        @statements = Diagram.statement_list(statements)
+        @cdb = cdb
+      end
+
+      def to_jigsaw_json
+        { name: "root", children: statements.map { |s| build_node(s) } }
+      end
+
+      private
+
+      attr_reader :statements, :cdb
+
+      def binary
+        @binary ||= ConceptQL.metadata(cdb)[:operators].select { |_k, v| v[:basic_type] == :filter }.keys
+      end
+
+      def build_node(operator)
+        node = Node.new(operator, binary)
+        {
+          name: node.name,
+          values: node.values,
+          parameters: build_parameters(node),
+          children: build_children(node)
+        }.reject { |_k, v| v.blank? }
+      end
+
+      def build_parameters(node)
+        return unless node.parameters
+
+        node.parameters.reject { |p| BINARY_PARAMETERS.include?(p.to_s) }
+      end
+
+      def build_children(node)
+        kids = node.children.map { |child| build_node(child) }
+        kids += build_binary_children(node) if node.binary?
+        kids
+      end
+
+      def build_binary_children(node)
+        return [] unless node.parameters
+
+        BINARY_PARAMETERS.filter_map { |p| build_node(node.parameters[p]) if node.parameters[p] }
+      end
+    end
+  end
+end

data/lib/conceptql/lexicon/lexicon_gdm.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "lexicon_strategy"
+
+module ConceptQL
+  class LexiconGDM < LexiconStrategy
+    class << self
+      def db_has_all_vocabulary_tables?(db)
+        %i[ancestors concepts mappings vocabularies].all? { |t| db.table_exists?(t) }
+      end
+    end
+
+    def vocabularies_query
+      db[:vocabularies]
+        .select(Sequel[:id].as(:id),
+                Sequel[:id].as(:omopv5_vocabulary_id),
+                Sequel[:vocabulary_name].as(:vocabulary_short_name),
+                Sequel[:vocabulary_name].as(:vocabulary_full_name),
+                Sequel[:domain],
+                Sequel.expr(1).as(:from_lexicon))
+    end
+
+    def vocabularies
+      vocabularies_query.all
+    end
+
+    def is_a_relationships(_data_db)
+      db[:mappings].where { Sequel.function(:lower, :relationship_id) =~ "is_a" }
+    end
+
+    def ancestors_table(_data_db)
+      db[:ancestors]
+    end
+
+    def concepts_table(_data_db, _some_schema = nil)
+      db[:concepts]
+    end
+
+    def table_is_missing?(_data_db)
+      !db.table_exists?(:concepts)
+    end
+
+    def strategy
+      :gdm
+    end
+  end
+end

data/lib/conceptql/lexicon/lexicon_no_db.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "lexicon_strategy"
+
+module ConceptQL
+  # The strategy used when neither the dataset db nor the lexicon db has
+  # vocabulary tables (for instance a db-less ConceptQL::Database, as the
+  # diagram renderer uses). Its db is a Sequel mock, so every lookup returns
+  # no rows, but SQL that joins or selects from the vocabulary tables can
+  # still be BUILT.
+  #
+  # It must answer the same table methods as the other strategies, because
+  # LexiconStrategy's lookups (concepts, descendants_of,
+  # related_concept_ids, ...) are written against them. The tables are named
+  # in the GDM layout (concepts / ancestors / mappings), matching
+  # #vocabularies_query, which already reads GDM's `vocabularies` table, and
+  # the columns that LexiconStrategy's queries use (id, concept_code,
+  # vocabulary_id, concept_text; ancestor_id / descendant_id;
+  # concept_1_id / concept_2_id).
+  class LexiconNoDB < LexiconStrategy
+    def vocabularies_query
+      db[:vocabularies]
+    end
+
+    def vocabularies
+      []
+    end
+
+    def is_a_relationships(_data_db)
+      db[:mappings].where { Sequel.function(:lower, :relationship_id) =~ "is_a" }
+    end
+
+    def ancestors_table(_data_db)
+      db[:ancestors]
+    end
+
+    def concepts_table(_data_db, _some_schema = nil)
+      db[:concepts]
+    end
+
+    def table_is_missing?(_db)
+      true
+    end
+
+    def strategy
+      :no_db
+    end
+  end
+end