conceptql 0.3.0 → 0.4.0

data/lib/conceptql/behaviors/utilizable.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module ConceptQL
+  module Behaviors
+    module Utilizable
+      extend ActiveSupport::Concern
+      include ConceptQL::Behaviors::Windowable
+
+      included do
+        domains :condition_occurrence
+        category "Select by Property"
+        basic_type :selection
+        validate_no_upstreams
+        validate_no_arguments
+
+        require_column :admission_date
+        require_column :discharge_date
+        require_column :length_of_stay
+        require_column :admission_source_value
+        require_column :admission_source_description
+        require_column :discharge_location_source_value
+        require_column :discharge_location_source_description
+        require_column :primary_concept_id
+      end
+
+      def column_family
+        "inpatient"
+      end
+
+      # The operators that include this register for gdm and gdm_wide only.
+      def query(db)
+        gdm_it(db)
+      end
+
+      def collection_type
+        raise NotImplementedError
+      end
+
+      # With the Database's share_utilization option, the list is read from a
+      # table that every statement on the Database shares. The WITH entry only
+      # carries the list to Scope#with_ctes, which takes it out of the SQL and
+      # creates the table when the query first reads rows.
+      def evaluate(db)
+        cdb = nodifier.cdb
+        return super unless cdb.opts[:share_utilization]
+
+        list = select_it(query(db))
+        name = cdb.shared_utilization_table(list, scope)
+        db[name].with(name, list, Scope::SHARED_TABLE => true)
+      end
+
+      def gdm_it(db)
+        source_type_id = dm.concept_ids(db, dm.file_provenance_types_vocab, collection_type)
+        all_source_type_ids = dm.descendants_of(db, source_type_id)
+        primary_id = dm.concept_ids(db, dm.code_provenance_types_vocab, "primary")
+        all_primary_ids = dm.descendants_of(db, primary_id)
+        # condition_vocabularies = dm.vocabularies_query.where(domain: 'condition_occurrence').select_map(:id)
+
+        return gdm_wide_it(db, all_primary_ids, all_source_type_ids) if wide?
+
+        # Get primary diagnosis codes
+        primary_concepts = rdbms.primary_concepts(db, all_primary_ids)
+
+        relevant_contexts = db[Sequel[:contexts].as(:cn)]
+                            .where(Sequel[:cn][:source_type_concept_id] => all_source_type_ids)
+                            .select(:collection_id)
+
+        db[:collections].from_self(alias: :cl)
+                        .then { |cl| restrict_to_window_persons(cl, Sequel[:cl][:patient_id]) }
+                        .join(:admission_details, { Sequel[:ad][:id] => Sequel[:cl][:admission_detail_id] }, table_alias: :ad)
+                        .left_join(dm.name_concepts_table(db), { Sequel[:ad][:admit_source_concept_id] => Sequel[:asc][:id] }, table_alias: :asc)
+                        .left_join(dm.name_concepts_table(db), { Sequel[:ad][:discharge_location_concept_id] => Sequel[:dlc][:id] }, table_alias: :dlc)
+                        .left_join(primary_concepts, { Sequel[:pcon][:collection_id] => Sequel[:cl][:id] }, table_alias: :pcon)
+                        .where(Sequel[:cl][:id] => relevant_contexts)
+                        .select(
+                          Sequel[:cl][:start_date].as(:collection_start_date),
+                          Sequel[:cl][:end_date].as(:collection_end_date),
+                          Sequel[:ad][:admission_date].as(:admission_date),
+                          Sequel[:ad][:discharge_date].as(:discharge_date),
+                          Sequel[:asc][:concept_code].as(:admission_source_value),
+                          Sequel[:asc][:concept_text].as(:admission_source_description),
+                          Sequel[:dlc][:concept_code].as(:discharge_location_source_value),
+                          Sequel[:dlc][:concept_text].as(:discharge_location_source_description),
+                          Sequel[:pcon][:concept_id].as(:primary_concept_id),
+                          Sequel[:pcon][:concept_code].as(:source_value),
+                          Sequel[:pcon][:vocabulary_id].as(:source_vocabulary_id),
+                          Sequel[:cl][:patient_id].as(:patient_id),
+                          Sequel[:cl][:id].as(:collection_id)
+                        )
+                        .select_append(Sequel[:ROW_NUMBER].function.over(
+                          partition: [ :admission_detail_id ],
+                          order: [
+                            Sequel[:pcon][:is_primary].desc,
+                            Sequel[:cl][:id],
+                            Sequel[:pcon][:concept_code]
+                          ]
+                        ).as(:nummy))
+                        .from_self
+                        .where(nummy: 1)
+      end
+
+      def gdm_wide_it(db, all_primary_ids, all_source_type_ids)
+        # Emit `CASE WHEN provenance_concept_id IN (...) THEN 1 ELSE 0 END`.
+        # The previous multi-WHEN form (`Sequel.case({{prov=>id1}=>1, {prov=>id2}=>1, ...}, 0)`)
+        # is mis-evaluated by Spark 3.3.2 when its result feeds the outer
+        # `CASE is_primary WHEN 1 THEN col ELSE NULL END` below. See conceptql-pk7.
+        is_primary =
+          if all_primary_ids.empty?
+            Sequel[0]
+          else
+            Sequel.case(
+              [ [ { Sequel[:provenance_concept_id] => all_primary_ids }, 1 ] ],
+              0
+            )
+          end
+
+        db[:observations]
+          .then { |observations| restrict_to_window_persons(observations, :patient_id) }
+          .where(source_type_concept_id: all_source_type_ids)
+          .exclude(admit_admission_date: nil)
+          .select_append(
+            is_primary.as(:is_primary)
+          )
+          .from_self
+          # One admission has many rows with the same concept on different
+          # line dates. The unique id ends the ORDER BY so the kept row is the
+          # same on every run, and admission_end_date takes the latest end_date
+          # over all of the admission's rows, not the kept row's. See conceptql-evk.
+          .select_append(
+            Sequel[:ROW_NUMBER].function.over(
+              partition: [ :admission_detail_id ],
+              order: [ Sequel[:is_primary].desc, Sequel[:collection_id], Sequel[:clinical_code_concept_id], Sequel[:id] ]
+            ).as(:nummy),
+            Sequel.function(:max, :end_date).over(partition: [ :admission_detail_id ]).as(:admission_end_date)
+          )
+          .from_self
+          .where(nummy: 1)
+          .from_self
+          .left_join(dm.name_concepts_table(db), { Sequel[:admit_source_concept_id] => Sequel[:asc][:id] }, table_alias: :asc)
+          .left_join(dm.name_concepts_table(db), { Sequel[:discharge_location_concept_id] => Sequel[:dlc][:id] }, table_alias: :dlc)
+          .select_append(
+            Sequel[:asc][:concept_code].as(:admission_source_value),
+            Sequel[:asc][:concept_text].as(:admission_source_description),
+            Sequel[:dlc][:concept_code].as(:discharge_location_source_value),
+            Sequel[:dlc][:concept_text].as(:discharge_location_source_description),
+            Sequel.case({ 1 => :clinical_code_concept_id }, nil, :is_primary).as(:primary_concept_id),
+            Sequel.case({ 1 => :clinical_code_source_value }, nil, :is_primary).as(:source_value),
+            Sequel.case({ 1 => :clinical_code_vocabulary_id }, nil, :is_primary).as(:source_vocabulary_id)
+          )
+          .from_self
+      end
+
+      # Keep only the persons that the scope's table windows cover, before the
+      # ROW_NUMBER over admission_detail_id. Without this, Spark numbers every
+      # inpatient row in the database and applies the window join afterwards.
+      # Output is unchanged because one admission never belongs to two persons
+      # (Ryan, 2026-10-14), so each ROW_NUMBER partition is kept or dropped whole.
+      def restrict_to_window_persons(ds, person_column)
+        return ds if skip_windows?
+
+        scope.windows.grep(Window::Table).inject(ds) do |restricted, window|
+          restricted.where(person_column => window.person_ids(restricted.db))
+        end
+      end
+
+      def table
+        :collections
+      end
+
+      def override_columns
+        return unless gdm?
+
+        {
+          length_of_stay: (rdbms.days_between(Sequel[:admission_date],
+                                              discharge_date_column) + 1).as(:length_of_stay),
+          criterion_table: Sequel.cast_string("collections").as(:criterion_table),
+          criterion_domain: Sequel.cast_string("condition_occurrence").as(:criterion_domain),
+          admission_source_value: Sequel[:admission_source_value],
+          admission_source_description: Sequel[:admission_source_description],
+          discharge_location_source_value: Sequel[:discharge_location_source_value],
+          discharge_location_source_description: Sequel[:discharge_location_source_description],
+          admission_date: Sequel[:admission_date],
+          discharge_date: discharge_date_column.as(:discharge_date),
+          person_id: Sequel[:patient_id].as(:person_id),
+          criterion_id: Sequel[:collection_id].as(:criterion_id),
+          start_date: Sequel[:collection_start_date].as(:start_date),
+          end_date: Sequel[:collection_end_date].as(:end_date),
+          primary_concept_id: Sequel[:primary_concept_id],
+          source_value: Sequel[:source_value].as(:source_value),
+          source_vocabulary_id: Sequel[:source_vocabulary_id].as(:source_vocabulary_id)
+        }
+      end
+
+      # gdm_wide's AdmissionDateQueryModifier falls back to the kept row's own
+      # end_date when the admission has no discharge date. An admission falls
+      # back to its latest end_date instead (Ryan, 2026-10-16). See conceptql-evk.
+      def discharge_date_column
+        return Sequel[:discharge_date] unless wide?
+
+        Sequel.function(:coalesce, :admit_discharge_date, :admission_end_date, :discharge_date)
+      end
+    end
+  end
+end

data/lib/conceptql/behaviors/windowable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module Behaviors
+    # Applies scope windows to Operators
+    module Windowable
+      def select_it(query, opts = {})
+        ds = super
+        return ds if skip_windows?
+
+        scope.windows.each do |window|
+          ds = window.call(self, ds, timeless: timeless?)
+        end
+
+        ds
+      end
+
+      def skip_windows?
+        false
+      end
+
+      def timeless?
+        false
+      end
+    end
+  end
+end

data/lib/conceptql/cli.rb

@@ -1,43 +1,46 @@
-$: << "lib"
+# frozen_string_literal: true
 
-require 'thor'
-require 'sequelizer'
-require 'json'
-require 'pp'
-require 'csv'
-require_relative 'query'
-require_relative 'knitter'
+$LOAD_PATH << "lib"
+
+require "thor"
+require "sequelizer"
+require "json"
+require "csv"
+require "logger"
+require "conceptql"
+require_relative "query"
 
 module ConceptQL
   class CLI < Thor
     include Sequelizer
 
     class_option :adapter,
-      aliases: :a,
-      desc: 'adapter for database'
+                 aliases: :a,
+                 desc: "adapter for database"
     class_option :host,
-      aliases: :h,
-      banner: 'localhost',
-      desc: 'host for database'
+                 aliases: :h,
+                 banner: "localhost",
+                 desc: "host for database"
     class_option :username,
-      aliases: :u,
-      desc: 'username for database'
+                 aliases: :u,
+                 desc: "username for database"
     class_option :password,
-      aliases: :P,
-      desc: 'password for database'
+                 aliases: :P,
+                 desc: "password for database"
     class_option :port,
-      aliases: :p,
-      type: :numeric,
-      banner: '5432',
-      desc: 'port for database'
+                 aliases: :p,
+                 type: :numeric,
+                 banner: "5432",
+                 desc: "port for database"
     class_option :database,
-      aliases: :d,
-      desc: 'database for database'
+                 aliases: :d,
+                 desc: "database for database"
     class_option :search_path,
-      aliases: :s,
-      desc: 'schema for database (PostgreSQL only)'
+                 aliases: :s,
+                 desc: "schema for database (PostgreSQL only)"
 
-    desc 'run_statement statement_file', 'Reads the ConceptQL statement from the statement file and executes it against the DB'
+    desc "run_statement statement_file",
+         "Reads the ConceptQL statement from the statement file and executes it against the DB"
     def run_statement(statement_file)
       q = cdb(options).query(criteria_from_file(statement_file))
       puts q.sql
@@ -45,79 +48,85 @@ module ConceptQL
       pp q.query.all
     end
 
-    desc "annotate_statement", "Reads in a statement and annotates it"
-    def annotate_statement(statement_file)
-      q = ConceptQL::Query.new(db(options), criteria_from_file(statement_file))
-      pp q.annotate
-    end
-
-    desc 'show_graph statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
-    option :watch_file
-    def show_graph(file)
-      graph_it(criteria_from_file(file))
-      system('open /tmp/graph.pdf')
+    desc "analyze statement_file",
+         "Reads the ConceptQL statement from the statement file and executes EXPLAIN ANALYZE against the DB"
+    def analyze(statement_file)
+      q = cdb(options).query(criteria_from_file(statement_file))
+      puts JSON.pretty_generate(q.statement)
+      puts q.sql(:formatted, :create_tables)
+      q.db.logger = db.loggers + [ Logger.new($stdout).tap { |l| l.level = Logger::INFO } ]
+      puts q.analyze
     end
 
-    desc 'show_and_tell_file statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph, then executes the statement against the DB'
-    option :full
-    option :watch_file
-    def show_and_tell_file(file)
-      show_and_tell(criteria_from_file(file), options)
+    desc "sql statement_file", "Reads the ConceptQL statement from the statement file and prints the SQL out"
+    def sql(statement_file)
+      q = cdb(options).query(criteria_from_file(statement_file))
+      puts q.sql(:formatted, :create_tables)
     end
 
-    desc 'fake_graph file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
-    def fake_graph(file)
-      require_relative 'fake_annotater'
-      annotated = ConceptQL::FakeAnnotater.new(criteria_from_file(file)).annotate
-      pp annotated
-      ConceptQL::AnnotateGrapher.new.graph_it(annotated, '/tmp/graph.pdf')
-      system('open /tmp/graph.pdf')
+    desc "code_list statement_file",
+         "Reads the ConceptQL statement from the statement file and prints the code list out"
+    def code_list(statement_file)
+      q = cdb(options).query(criteria_from_file(statement_file))
+      puts q.code_list
     end
 
-    desc 'metadata', 'Generates the metadata.js file for the JAM'
-    def metadata
-      File.write('metadata.js', "$metadata = #{ConceptQL.metadata(warn: true).to_json};")
-      File.write('metadata.json', ConceptQL.metadata.to_json)
-      if system("which", "json_pp", out: File::NULL, err: File::NULL)
-        system("cat metadata.json | json_pp", out: "metadata_pp.json")
+    desc "annotate_statement", "Reads in a statement and annotates it"
+    def annotate_statement(statement_file)
+      q = ConceptQL::Query.new(cdb(options), criteria_from_file(statement_file))
+      pp q.annotate(skip_counts: true)
+    end
+
+    desc "scope_annotate", "Reads in a statement and annotates it"
+    def scope_annotate(statement_file)
+      q = ConceptQL::Query.new(cdb(options), criteria_from_file(statement_file))
+      pp q.scope_annotate(skip_counts: true)
+    end
+
+    desc "render_json STATEMENT_FILE",
+         "Prints the statement(s) as a #{ConceptQL::Diagram::FORMAT} render tree (JSON) for the diagram renderer"
+    long_desc <<~DESC
+      STATEMENT_FILE holds one ConceptQL statement or an array of them (.json, or Ruby for any other extension).
+      Prints {"format": "#{ConceptQL::Diagram::FORMAT}", "statements": [<node>, ...]}, one node per statement.
+
+      Without --counts no claims database is used. With --counts each node gains
+      "counts": {<domain>: {"rows": Integer, "n": Integer}}, computed against the database
+      given by the connection options (or sequelizer's configuration).
+    DESC
+    option :counts, type: :boolean, default: false, desc: "annotate with per-domain row/person counts (needs a DB)"
+    option :data_model, desc: "data model: gdm, gdm_wide, omopv4_plus (default: $CONCEPTQL_DATA_MODEL or gdm)"
+    def render_json(statement_file)
+      cdb = if options[:counts]
+              ConceptQL::Database.new(db(options.except("counts", "data_model")),
+                                      { data_model: options[:data_model] }.compact)
+      else
+              ConceptQL::Diagram.default_cdb(options[:data_model])
       end
+      doc = ConceptQL::Diagram.render(criteria_from_file(statement_file), cdb: cdb, counts: options[:counts])
+      puts JSON.pretty_generate(doc)
     end
 
-    desc 'selection_operators', 'Generates a TSV of all the selection operators'
-    def selection_operators
-      require 'csv'
-      CSV.open('selection_operators.tsv', 'w', col_sep: "\t") do |csv|
-        csv << ["name"]
-        ConceptQL.metadata[:operators].values.select { |v| v[:basic_type] == :selection }.map { |v| v[:preferred_name] }.each do |name|
-          csv << [name]
-        end
-      end
+    desc "metadata", "Generates the metadata.js file for the JAM"
+    def metadata
+      json = ConceptQL.metadata(cdb(options), warn: true).to_json
+      File.write("metadata.js", "$metadata = #{json};")
+      File.write("metadata.json", json)
+      return unless system("which", "json_pp", out: File::NULL, err: File::NULL)
 
-      CSV.open('domains.tsv', 'w', col_sep: "\t") do |csv|
-        csv << ["name"]
-        ConceptQL::Operators::TABLE_COLUMNS.each do |domain, columns|
-          next unless columns.include?(:person_id)
-          next if domain.to_s =~ /era\Z/
-          csv << [domain]
-        end
-      end
+      system("cat metadata.json | json_pp", out: "metadata_pp.json")
     end
 
-    desc 'knit', 'Processes ConceptQL fenced code segments and produces a Markdown file'
-    option :ignore_cache, type: :boolean
-    def knit(file)
-      opts = {}
-      if options[:ignore_cache]
-        opts[:cache_options] = { ignore: true }
-      end
-      ConceptQL::Knitter.new(ConceptQL::Database.new(db), file, opts).knit
+    desc "simple_visit STATEMENT_FILE",
+         "Prints the operators, depth and selection-node count of the statement in STATEMENT_FILE"
+    def simple_visit(statement_file)
+      pp cdb(options).query(criteria_from_file(statement_file)).extract_metadata
     end
 
-    desc 'dumpit', 'Dumps out test data into CSVs'
+    desc "dumpit", "Dumps out test data into CSVs"
     def dumpit(path)
       path = Pathname.new(path)
       path.mkpath unless path.exist?
-      headers_path = path + 'headers'
+      headers_path = path / "headers"
       headers_path.mkpath unless headers_path.exist?
       db.tables.each do |table|
         puts "Dumping #{table}..."
@@ -133,68 +142,24 @@ module ConceptQL
     end
 
     private
-    desc 'show_and_tell_db conceptql_id', 'Fetches the ConceptQL from a DB and shows the contents as a ConceptQL graph, then executes the statement against our test database'
-    option :full
-    option :watch_file
-    def show_and_tell_db(conceptql_id)
-      result = fetch_conceptql(conceptql_id, options)
-      puts "Concept: #{result[:label]}"
-      show_and_tell(result[:statement].to_hash, options, result[:label])
-    end
-
-    desc 'show_db_graph conceptql_id', 'Shows a graph for the conceptql statement represented by conceptql_id in the db specified by db_url'
-    def show_db_graph(conceptql_id)
-      result = fetch_conceptql(conceptql_id, options)
-      graph_it(result[:statement], db, result[:label])
-    end
-
-    def fetch_conceptql(conceptql_id)
-      my_db = db(options)
-      my_db.extension(:pg_array, :pg_json)
-      my_db[:concepts].where(concept_id: conceptql_id).select(:statement, :label).first
-    end
-
-    def show_and_tell(statement, options, title = nil)
-      my_db = db(options)
-      q = ConceptQL::Query.new(my_db, statement)
-      statement = q.annotate
-      puts 'JSON'
-      puts JSON.pretty_generate(statement)
-      graph_it(statement, title)
-      STDIN.gets
-      puts q.sql
-      STDIN.gets
-      results = q.all
-      if options[:full]
-        pp results
-      else
-        pp filtered(results)
-      end
-      puts results.length
-    end
-
-    def graph_it(statement, title = nil)
-      my_db = db(options)
-      q = ConceptQL::Query.new(my_db, statement).annotate
-      ConceptQL::AnnotateGrapher.new.graph_it(q, '/tmp/graph.pdf')
-    end
 
     def criteria_from_file(file)
       case File.extname(file)
-      when '.json'
+      when ".json"
         JSON.parse(File.read(file))
       else
         eval(File.read(file))
       end
     end
 
-    def filtered(results)
-      results.each { |r| r.delete_if { |k,v| v.nil? } }
-    end
-
     def cdb(options)
-      ConceptQL::Database.new(db(options))
+      data_db = db(options)
+      cdb = ConceptQL::Database.new(data_db)
+      # A glob of directories, in search path order: "/data/{wide,slim}".
+      if (parquet_dirs = ENV["CONCEPTQL_PARQUET_TEST_DIR"])
+        data_db.make_ready(search_path: Dir[parquet_dirs].map { |dir| Pathname.new(dir).glob("*.parquet") })
+      end
+      cdb
     end
   end
 end
-

data/lib/conceptql/columnizer.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  class Columnizer
+    class ColumnDefinition
+      attr_reader :name, :options
+
+      def initialize(name, options)
+        @name = name
+        @options = options
+      end
+
+      def grouped?
+        options[:grouped]
+      end
+
+      def sequelized
+        col = name
+        if (q = options[:qualifier])
+          col = Sequel.qualify(q, col)
+        end
+
+        if (fun = options[:function])
+          col = Sequel.function(fun, col)
+        end
+
+        unless options[:no_alias]
+          n = extract_name(name)
+          col = Sequel.expr(col).as(n) if n.present?
+        end
+
+        col
+      end
+
+      def extract_name(name)
+        return name if name.is_a?(Symbol)
+
+        n = nil
+        n = name.column if name.respond_to?(:column)
+        return n unless n.blank?
+
+        n = name.table if name.respond_to?(:table)
+        return n unless n.blank?
+
+        extract_name(name.expr) if name.respond_to?(:expr)
+      end
+    end
+
+    attr_reader :columns
+
+    def initialize
+      @columns = {}
+    end
+
+    def add_columns(*column_names)
+      opts = ConceptQL::Utils.extract_opts!(column_names)
+      column_names.each do |column_name|
+        columns[column_name] = ColumnDefinition.new(column_name, opts)
+      end
+    end
+
+    def apply(query)
+      grouped, ungrouped = columns.values.partition(&:grouped?)
+      select_method = :select
+      unless grouped.empty?
+        query = query.select_group(*grouped.map(&:sequelized))
+        select_method = :select_append
+      end
+
+      return query if ungrouped.empty?
+
+      query.send(select_method, *ungrouped.map(&:sequelized))
+    end
+  end
+end