conceptql 0.4.0 → 0.5.0

data/lib/conceptql/operators/snf.rb

@@ -5,8 +5,7 @@ require_relative "operator"
 module ConceptQL
   module Operators
     class Snf < Operator
-      # GDM only: omopv4_plus has no provenance vocabularies (conceptql-myp).
-      register __FILE__, :gdm, :gdm_wide
+      register __FILE__
       include ConceptQL::Behaviors::Utilizable
 
       desc "Selects admission records of type 'SNF'"

data/lib/conceptql/operators/vocabulary.rb

@@ -38,10 +38,23 @@ module ConceptQL
         args -= bad_arguments
 
         missing_args = args - dm.known_codes(db, vocabulary_id, args)
+        add_warning("unknown code(s)", *missing_args) unless missing_args.empty?
 
-        return if missing_args.empty?
+        warn_about_reused_codes(db, args - missing_args)
+      end
 
-        add_warning("unknown code(s)", *missing_args)
+      # A code that the vocabulary reused for a new meaning selects records of
+      # both meanings: DRG 410 selects chemotherapy stays before 2007-10-01 and
+      # biliary tract procedures after it (conceptql-ttk). Each item names the
+      # code and all of its names, so the user sees the problem without a
+      # vocabulary lookup.
+      def warn_about_reused_codes(db, codes)
+        meanings = dm.code_meanings(db, vocabulary_id, codes)
+        reused = codes.filter_map do |code|
+          names = meanings[code.downcase]
+          "#{code}: #{names.join(' | ')}" if names
+        end
+        add_warning("code(s) with more than one meaning", *reused) unless reused.empty?
       end
 
       def describe_codes(db, codes)

data/lib/conceptql/paths.rb

@@ -37,5 +37,11 @@ module ConceptQL
     def race_file
       ConceptQL.config_dir / "race.yml"
     end
+
+    # The ConceptQL specification, shipped in the gem for applications that
+    # serve it to readers (the Jigsaw diagram editor serves it over MCP).
+    def spec_path
+      root / "doc" / "spec.md"
+    end
   end
 end

data/lib/conceptql/query.rb

@@ -4,7 +4,7 @@ require "json"
 require "forwardable"
 require_relative "scope"
 require_relative "nodifier"
-require_relative "sql_formatters"
+require "sequel/sqlformat"
 
 module ConceptQL
   class Query
@@ -208,8 +208,11 @@ module ConceptQL
       end
     end
 
+    # Formats inside the Ruby process with the sequel-sqlformat gem (the Rust
+    # sqlformat crate). It changes only whitespace and keyword case, and it never
+    # fails on unknown syntax, so DuckDB and Spark SQL format too.
     def format(sql)
-      SqlFormatters.format(sql, rdbms)
+      Sequel::SQLFormat.format(sql, dialect: Sequel::SQLFormat.dialect_for(cdb.database_type), indent: 2, uppercase: true)
     end
   end
 end

data/lib/conceptql/rdbms/duckdb.rb

@@ -12,10 +12,6 @@ module ConceptQL
         Sequel.function(:datediff, "day", to, from)
       end
 
-      def preferred_formatter
-        SqlFormatters::PG_FORMAT
-      end
-
       # DuckDB accepts WITH ... AS MATERIALIZED since v0.8.0.
       def supports_materialized?
         true

data/lib/conceptql/rdbms/generic.rb

@@ -59,10 +59,6 @@ module ConceptQL
         end
       end
 
-      def preferred_formatter
-        SqlFormatters::SQL_FORMATTER
-      end
-
       # Returns from - to in days, NOT to - from, so datediff(later, earlier)
       # is positive. Every adapter keeps this order. This default emits
       # Spark's datediff(endDate, startDate), which returns endDate - startDate,

data/lib/conceptql/rdbms/postgres.rb

@@ -14,10 +14,6 @@ module ConceptQL
         db.vacuum_table(table_name, analyze: true) if analyze_temp_tables?
       end
 
-      def preferred_formatter
-        SqlFormatters::POSTGRES
-      end
-
       # DISTINCT ON is Postgres-only. The other adapters inherit
       # Generic#primary_concepts, which uses ROW_NUMBER() instead.
       def primary_concepts(db, all_primary_ids)

data/lib/conceptql/rdbms/spark.rb

@@ -13,10 +13,6 @@ module ConceptQL
       def null_safe_equal(lhs, rhs)
         Sequel.lit("(? <=> ?)", lhs, rhs)
       end
-
-      def preferred_formatter
-        SqlFormatters::PG_FORMAT
-      end
     end
   end
 end

data/lib/conceptql/utils.rb

@@ -4,8 +4,6 @@ require "active_support/core_ext/array/conversions"
 require "active_support/core_ext/object/blank"
 require "active_support/core_ext/string/inflections"
 require "json"
-require "open3"
-require "timeout"
 
 module ConceptQL
   module Utils
@@ -31,47 +29,6 @@ module ConceptQL
         end
       end
 
-      def timed_capture(*commands)
-        # Heavily adapted from:
-        # https://gist.github.com/lpar/1032297#gistcomment-1738285
-        opts = extract_opts!(commands)
-        timeout = opts.fetch(:timeout)
-
-        stdin, stdout, stderr, wait_thread = Open3.popen3(*commands)
-        wait_thread[:timed_out] = false
-        stdin.puts opts[:stdin_data] if opts[:stdin_data]
-        stdin.close
-
-        # Purposefully NOT using Timeout.rb because in general it is a dangerous API!
-        # http://blog.headius.com/2008/02/rubys-threadraise-threadkill-timeoutrb.html
-        Thread.new do
-          sleep timeout
-          if wait_thread.alive?
-            begin
-              wait_thread[:timed_out] = true
-              # please note: we are assuming the command will create ONE process (not handling subprocs / proc groups)
-              command = "kill -9 #{wait_thread.pid}"
-              system(command)
-            rescue(Errno::ESRCH)
-              # Do nothing!
-            end
-          end
-        end
-
-        if (es = wait_thread.value.exitstatus) && !es.zero?
-          raise stderr.read
-        end
-
-        out = stdout.read
-        stdout.close
-
-        if wait_thread[:timed_out]
-          raise Timeout::Error, "Command #{commands} failed to complete after #{timeout} seconds"
-        end
-
-        out
-      end
-
       def extract_opts!(arr)
         return {} unless arr.last.is_a?(Hash)
 

data/lib/conceptql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ConceptQL
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/conceptql/vocabularies/entry.rb

@@ -9,7 +9,6 @@ module ConceptQL
 
       METHODS = %i[
         id
-        omopv4_vocabulary_id
         omopv5_vocabulary_id
         domain
         vocabulary_short_name
@@ -28,11 +27,6 @@ module ConceptQL
         predominant_domains
       ]
 
-      COST_RELATED_VOCABS = [
-        "Revenue Code",
-        "DRG"
-      ].freeze
-
       def initialize(hash)
         @hash = hash
         @hash[:omopv5_vocabulary_id] ||= @hash[:id]
@@ -54,12 +48,6 @@ module ConceptQL
         self.class.new(hash.merge(other_entry.hash))
       end
 
-      def belongs_in_omopv4_plus?
-        @belongs_in_omopv4_plus ||= (!from_lexicon? || from_csv?) \
-          && has_domain? \
-          && visible?
-      end
-
       def to_hash
         METADATA_METHODS.to_h do |meth|
           [ meth, send(meth) ]
@@ -114,10 +102,6 @@ module ConceptQL
         hash[:domain] =~ /drug_exposure/i
       end
 
-      def is_costish?
-        belongs_in_omopv4_plus? && COST_RELATED_VOCABS.include?(omopv5_vocabulary_id)
-      end
-
       def visible?
         !hidden?
       end
@@ -126,10 +110,6 @@ module ConceptQL
         omopv5_vocabulary_id || id
       end
 
-      def omopv4_id
-        omopv4_vocabulary_id
-      end
-
       def get_klasses
         VocabularyClassFactory.new(self).get_klasses
       end

data/lib/conceptql/vocabularies/vocabulary_class_factory.rb

@@ -3,8 +3,6 @@
 require_relative "../behaviors/drugish"
 require_relative "../behaviors/labish"
 require_relative "behaviors/gdmish"
-require_relative "behaviors/omopish"
-require_relative "behaviors/costish"
 require_relative "../operators/vocabulary"
 
 module ConceptQL
@@ -19,9 +17,8 @@ module ConceptQL
       def get_klasses
         {
           gdm: get_gdm_klass,
-          gdm_wide: get_gdm_klass,
-          omopv4_plus: get_omopv4_plus_klass
-        }.compact
+          gdm_wide: get_gdm_klass
+        }
       end
 
       def get_gdm_klass
@@ -30,16 +27,6 @@ module ConceptQL
         end
       end
 
-      def get_omopv4_plus_klass
-        return nil unless entry.belongs_in_omopv4_plus?
-
-        get_klass do
-          include Behaviors::Omopish
-
-          include Behaviors::Costish if entry.is_costish?
-        end
-      end
-
       def get_klass(&block)
         ventry = entry
         Class.new(ConceptQL::Operators::Vocabulary) do |klass|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: conceptql
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Ryan Duryea
@@ -79,6 +79,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.66'
+- !ruby/object:Gem::Dependency
+  name: sequel-sqlformat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sequelizer
   requirement: !ruby/object:Gem::Requirement
@@ -314,6 +328,7 @@ files:
 - config/provenance.yml
 - config/race.yml
 - config/vocabularies.csv
+- doc/spec.md
 - lib/conceptql.rb
 - lib/conceptql/behaviors/code_lister.rb
 - lib/conceptql/behaviors/drugish.rb
@@ -330,7 +345,6 @@ files:
 - lib/conceptql/data_model/base.rb
 - lib/conceptql/data_model/gdm.rb
 - lib/conceptql/data_model/gdm_wide.rb
-- lib/conceptql/data_model/omopv4_plus.rb
 - lib/conceptql/database.rb
 - lib/conceptql/date_adjuster.rb
 - lib/conceptql/diagram.rb
@@ -404,11 +418,6 @@ files:
 - lib/conceptql/query_modifiers/gdm_wide/lab_query_modifier.rb
 - lib/conceptql/query_modifiers/gdm_wide/pos_query_modifier.rb
 - lib/conceptql/query_modifiers/gdm_wide/provenance_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/admission_date_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/drug_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/pos_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/provenance_query_modifier.rb
-- lib/conceptql/query_modifiers/omopv4_plus/provider_query_modifier.rb
 - lib/conceptql/query_modifiers/query_modifier.rb
 - lib/conceptql/rdbms.rb
 - lib/conceptql/rdbms/duckdb.rb
@@ -416,15 +425,11 @@ files:
 - lib/conceptql/rdbms/postgres.rb
 - lib/conceptql/rdbms/spark.rb
 - lib/conceptql/scope.rb
-- lib/conceptql/sql_formatters.rb
-- lib/conceptql/sql_formatters/formatter.rb
 - lib/conceptql/utils.rb
 - lib/conceptql/utils_cli.rb
 - lib/conceptql/version.rb
 - lib/conceptql/visitors/metadata_extractor.rb
-- lib/conceptql/vocabularies/behaviors/costish.rb
 - lib/conceptql/vocabularies/behaviors/gdmish.rb
-- lib/conceptql/vocabularies/behaviors/omopish.rb
 - lib/conceptql/vocabularies/dynamic_vocabularies.rb
 - lib/conceptql/vocabularies/entry.rb
 - lib/conceptql/vocabularies/vocabulary_class_factory.rb
@@ -437,8 +442,6 @@ files:
 - schemas/gdm.yml
 - schemas/gdm_wide.yml
 - schemas/ohdsi_vocabs.yml
-- schemas/omopv4_plus.yml
-- schemas/omopv4_plus_more_ignorables.tsv
 homepage: https://github.com/outcomesinsights/conceptql
 licenses:
 - MIT

data/lib/conceptql/data_model/omopv4_plus.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base"
-module ConceptQL
-  module DataModel
-    class Omopv4Plus < Base
-      def base
-        :omop
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/admission_date_query_modifier.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class AdmissionDateQueryModifier < QueryModifier
-        def self.provided_columns
-          %i[
-            admission_date
-            discharge_date
-          ]
-        end
-
-        def self.has_required_columns?(cols)
-          cols.map(&:to_s).any? { |col| col =~ /start_date/i }
-        end
-
-        def modified_query
-          cols = dm.table_cols(source_table).map(&:to_s)
-          return query unless cols.any? { |col| col =~ /start_date/i }
-
-          start_col = cols.find { |col| col =~ /start_date/i }.to_sym
-          end_col = cols.find { |col| col =~ /end_date/i }.to_sym
-
-          query.from_self(alias: :t)
-               .select_all(:t)
-               .select_append(Sequel[:t][start_col].as(:admission_date))
-               .select_append(Sequel.function(:coalesce, Sequel[:t][end_col],
-                                              Sequel[:t][start_col]).as(:discharge_date))
-               .from_self
-        end
-
-        private
-
-        def domain
-          op.domain
-        rescue StandardError
-          nil
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/drug_query_modifier.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class DrugQueryModifier < QueryModifier
-        attr_reader :db
-
-        def self.provided_columns
-          %i[
-            drug_name
-            drug_amount
-            drug_amount_units
-            drug_quantity
-            drug_days_supply
-          ]
-        end
-
-        def self.has_required_columns?(cols)
-          needed = [ :drug_concept_id ]
-          found = needed & cols
-          needed == found
-        end
-
-        def initialize(*args)
-          super
-          @db = query.db
-        end
-
-        def modified_query
-          return query unless dm.table_cols(source_table).include?(:drug_concept_id)
-
-          query.db
-          drug_concept_ids = query.select(:drug_concept_id).distinct
-          query.from_self(alias: :de)
-               .left_join(micro_table(drug_concept_ids).as(:mt), drug_concept_id: :drug_concept_id)
-               .select_all(:de)
-               .select_append(Sequel[:de][:quantity].as(:drug_quantity))
-               .select_append(Sequel[:de][:days_supply].as(:drug_days_supply))
-               .select_append(Sequel[:mt][:amount_value].as(:drug_amount))
-               .select_append(Sequel[:mt][:amount_unit].as(:drug_amount_units))
-               .select_append(Sequel[:mt][:drug_name].as(:drug_name))
-               .from_self
-        end
-
-        private
-
-        def micro_table(drug_concept_ids)
-          # TODO: Does drug_strength only have RXNORM concept_ids?
-          # TODO: What is vocabulary for units?  Can we shrink concept table to just that vocab before joining?
-          collapsed_strengths = db[:drug_strength]
-                                .where(drug_concept_id: drug_concept_ids)
-                                .select_group(:drug_concept_id)
-                                .select_append(
-                                  Sequel.function(:min, :amount_value).as(:amount_value),
-                                  Sequel.function(:min, :amount_unit).as(:amount_unit),
-                                  Sequel.function(:count, 1).as(:dcount)
-                                )
-
-          value_case = Sequel.case({ 1 => :amount_value }, nil, :dcount).as(:amount_value)
-          unit_case = Sequel.case({ 1 => :amount_unit }, nil, :dcount).as(:amount_unit)
-
-          collapsed_strengths = collapsed_strengths.from_self.select(:drug_concept_id, value_case, unit_case)
-
-          db.from(Sequel[:concept].as(:dc))
-            .left_join(collapsed_strengths, { drug_concept_id: :concept_id }, { table_alias: :ds })
-            .where(Sequel[:dc][:concept_id] => drug_concept_ids)
-            .select(
-              Sequel[:dc][:concept_id].as(:drug_concept_id),
-              Sequel[:dc][:concept_name].as(:drug_name),
-              Sequel[:ds][:amount_value].as(:amount_value),
-              Sequel[:ds][:amount_unit].as(:amount_unit)
-            )
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/pos_query_modifier.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class PoSQueryModifier < QueryModifier
-        attr_reader :join_id, :table, :source_column, :column
-
-        def self.provided_columns
-          [ :visit_source_concept_id ]
-        end
-
-        def self.has_required_columns?(cols)
-          needed = [ :visit_occurrence_id ]
-          found = needed & cols
-          needed == found
-        end
-
-        def initialize(*args)
-          super
-          @column = :visit_source_concept_id
-          @join_id = :visit_occurrence_id
-          @table = :visit_occurrence
-          @source_column = :visit_source_concept_id
-        end
-
-        def modified_query
-          return query unless dm.table_cols(source_table).include?(join_id)
-          return query if dm.table_cols(source_table).include?(column)
-
-          left_alias = "tab1".to_sym
-          right_alias = "tab2".to_sym
-
-          extra_table = query.db.from(table).select(Sequel.as(source_column, column), join_id)
-          ds = query.from_self(alias: left_alias)
-          ds.left_join(extra_table.as(right_alias),
-                       Sequel.qualify(left_alias, join_id) => Sequel.qualify(right_alias, join_id))
-            .select_all(left_alias)
-            .select_append(Sequel.qualify(right_alias, column).as(column))
-            .from_self
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/provenance_query_modifier.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class ProvenanceQueryModifier < QueryModifier
-        RELATED_COLUMNS = %w[
-          condition_type_concept_id
-          procedure_type_concept_id
-          death_type_concept_id
-          observation_type_concept_id
-          drug_type_concept_id
-        ].sort.map(&:to_sym)
-
-        attr_reader :db
-
-        def self.provided_columns
-          %i[
-            file_provenance_type
-            code_provenance_type
-          ]
-        end
-
-        def self.has_required_columns?(cols)
-          !(RELATED_COLUMNS & cols).empty?
-        end
-
-        def initialize(*args)
-          super
-          @db = query.db
-        end
-
-        def modified_query
-          return query unless self.class.has_required_columns?(dm.table_cols(source_table))
-
-          query.select_append(Sequel[dm.file_provenance_type_column(query, source_table)].as(:file_provenance_type),
-                              Sequel[dm.code_provenance_type_column(query,
-                                                                    source_table)].as(:code_provenance_type)).from_self
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/query_modifiers/omopv4_plus/provider_query_modifier.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../query_modifier"
-
-module ConceptQL
-  module QueryModifiers
-    module Omopv4Plus
-      class ProviderQueryModifier < QueryModifier
-        def self.provided_columns
-          [ :provider_id ]
-        end
-
-        def self.has_required_columns?(cols)
-          cols.find { |col| col.to_s =~ /provider_id/ }
-        end
-
-        def modified_query
-          col = self.class.has_required_columns?(dm.table_cols(source_table))
-          return query if col.nil? || col.to_sym == :provider_id
-
-          query.select_append(Sequel[col].as(:provider_id)).from_self
-        end
-      end
-    end
-  end
-end

data/lib/conceptql/sql_formatters/formatter.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module ConceptQL
-  module SqlFormatters
-    # An external SQL formatter, given as a command array that reads SQL on
-    # stdin, e.g. Formatter.new(%w[sql-formatter -l postgresql]).
-    class Formatter
-      attr_reader :command
-
-      def initialize(command)
-        @command = command
-      end
-
-      def available?
-        system("which", command.first, out: File::NULL, err: File::NULL)
-      end
-
-      def format(sql)
-        ConceptQL::Utils.timed_capture(*command, stdin_data: sql, timeout: 10)
-      rescue Timeout::Error
-        sql
-      end
-    end
-  end
-end

data/lib/conceptql/sql_formatters.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "sql_formatters/formatter"
-
-module ConceptQL
-  module SqlFormatters
-    SQL_FORMATTER = %w[sql-formatter].freeze
-    POSTGRES = %w[sql-formatter -l postgresql].freeze
-    SQLFORMAT = %w[sqlformat --reindent --use_space_around_operators --identifiers lower --keywords upper -].freeze
-    PG_FORMAT = %w[pg_format].freeze
-
-    # Formats with the rdbms's preferred formatter, else the first of the
-    # fallbacks that is installed, else returns the SQL untouched.
-    def self.format(sql, rdbms)
-      formatter = [ rdbms.preferred_formatter, SQL_FORMATTER, SQLFORMAT, PG_FORMAT ]
-                  .compact
-                  .map { |command| Formatter.new(command) }
-                  .detect(&:available?)
-      formatter ? formatter.format(sql) : sql
-    end
-  end
-end