conceptql 0.2.0 → 0.4.0

data/lib/conceptql/cli.rb

@@ -1,156 +1,165 @@
-require 'thor'
-require 'sequelizer'
-require 'psych'
-require 'json'
-require 'pp'
-require_relative 'query'
+# frozen_string_literal: true
+
+$LOAD_PATH << "lib"
+
+require "thor"
+require "sequelizer"
+require "json"
+require "csv"
+require "logger"
+require "conceptql"
+require_relative "query"
 
 module ConceptQL
   class CLI < Thor
     include Sequelizer
 
     class_option :adapter,
-      aliases: :a,
-      desc: 'adapter for database'
+                 aliases: :a,
+                 desc: "adapter for database"
     class_option :host,
-      aliases: :h,
-      banner: 'localhost',
-      desc: 'host for database'
+                 aliases: :h,
+                 banner: "localhost",
+                 desc: "host for database"
     class_option :username,
-      aliases: :u,
-      desc: 'username for database'
+                 aliases: :u,
+                 desc: "username for database"
     class_option :password,
-      aliases: :P,
-      desc: 'password for database'
+                 aliases: :P,
+                 desc: "password for database"
     class_option :port,
-      aliases: :p,
-      type: :numeric,
-      banner: '5432',
-      desc: 'port for database'
+                 aliases: :p,
+                 type: :numeric,
+                 banner: "5432",
+                 desc: "port for database"
     class_option :database,
-      aliases: :d,
-      desc: 'database for database'
+                 aliases: :d,
+                 desc: "database for database"
     class_option :search_path,
-      aliases: :s,
-      desc: 'schema for database (PostgreSQL only)'
+                 aliases: :s,
+                 desc: "schema for database (PostgreSQL only)"
 
-    desc 'run_statement statement_file', 'Reads the ConceptQL statement from the statement file and executes it against the DB'
+    desc "run_statement statement_file",
+         "Reads the ConceptQL statement from the statement file and executes it against the DB"
     def run_statement(statement_file)
-      q = ConceptQL::Query.new(db(options), criteria_from_file(statement_file))
+      q = cdb(options).query(criteria_from_file(statement_file))
       puts q.sql
-      puts q.statement.to_yaml
-      pp q.all
+      puts JSON.pretty_generate(q.statement)
+      pp q.query.all
     end
 
-    desc 'show_graph statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
-    option :watch_file
-    def show_graph(file)
-      graph_it(criteria_from_file(file))
+    desc "analyze statement_file",
+         "Reads the ConceptQL statement from the statement file and executes EXPLAIN ANALYZE against the DB"
+    def analyze(statement_file)
+      q = cdb(options).query(criteria_from_file(statement_file))
+      puts JSON.pretty_generate(q.statement)
+      puts q.sql(:formatted, :create_tables)
+      q.db.logger = db.loggers + [ Logger.new($stdout).tap { |l| l.level = Logger::INFO } ]
+      puts q.analyze
     end
 
-    desc 'show_and_tell_file statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph, then executes the statement against the DB'
-    option :full
-    option :watch_file
-    def show_and_tell_file(file)
-      show_and_tell(criteria_from_file(file), options)
+    desc "sql statement_file", "Reads the ConceptQL statement from the statement file and prints the SQL out"
+    def sql(statement_file)
+      q = cdb(options).query(criteria_from_file(statement_file))
+      puts q.sql(:formatted, :create_tables)
     end
 
-    desc 'fake_graph file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
-    def fake_graph(file)
-      require_relative 'fake_grapher'
-      ConceptQL::FakeGrapher.new.graph_it(criteria_from_file(file), '/tmp/graph')
-      system('open /tmp/graph.pdf')
+    desc "code_list statement_file",
+         "Reads the ConceptQL statement from the statement file and prints the code list out"
+    def code_list(statement_file)
+      q = cdb(options).query(criteria_from_file(statement_file))
+      puts q.code_list
     end
 
-    desc 'metadata', 'Generates the metadata.js file for the JAM'
-    def metadata
-      File.write('/tmp/metadata.js', "var metadata = #{ConceptQL::Nodifier.new.to_metadata.to_json};")
+    desc "annotate_statement", "Reads in a statement and annotates it"
+    def annotate_statement(statement_file)
+      q = ConceptQL::Query.new(cdb(options), criteria_from_file(statement_file))
+      pp q.annotate(skip_counts: true)
     end
 
-    desc 'convert', 'Converts from hash-based syntax to list-based syntax'
-    def convert(file)
-      require 'conceptql/converter'
-      require 'json'
-      begin
-        puts JSON.pretty_generate(ConceptQL::Converter.new.convert(criteria_from_file(file)))
-      rescue
-        puts "Couldn't convert #{file}"
-        puts $!.message
-        puts $!.backtrace.join("\n")
-      end
+    desc "scope_annotate", "Reads in a statement and annotates it"
+    def scope_annotate(statement_file)
+      q = ConceptQL::Query.new(cdb(options), criteria_from_file(statement_file))
+      pp q.scope_annotate(skip_counts: true)
     end
 
-    private
-    desc 'show_and_tell_db conceptql_id', 'Fetches the ConceptQL from a DB and shows the contents as a ConceptQL graph, then executes the statement against our test database'
-    option :full
-    option :watch_file
-    def show_and_tell_db(conceptql_id)
-      result = fetch_conceptql(conceptql_id, options)
-      puts "Concept: #{result[:label]}"
-      show_and_tell(result[:statement].to_hash, options, result[:label])
+    desc "render_json STATEMENT_FILE",
+         "Prints the statement(s) as a #{ConceptQL::Diagram::FORMAT} render tree (JSON) for the diagram renderer"
+    long_desc <<~DESC
+      STATEMENT_FILE holds one ConceptQL statement or an array of them (.json, or Ruby for any other extension).
+      Prints {"format": "#{ConceptQL::Diagram::FORMAT}", "statements": [<node>, ...]}, one node per statement.
+
+      Without --counts no claims database is used. With --counts each node gains
+      "counts": {<domain>: {"rows": Integer, "n": Integer}}, computed against the database
+      given by the connection options (or sequelizer's configuration).
+    DESC
+    option :counts, type: :boolean, default: false, desc: "annotate with per-domain row/person counts (needs a DB)"
+    option :data_model, desc: "data model: gdm, gdm_wide, omopv4_plus (default: $CONCEPTQL_DATA_MODEL or gdm)"
+    def render_json(statement_file)
+      cdb = if options[:counts]
+              ConceptQL::Database.new(db(options.except("counts", "data_model")),
+                                      { data_model: options[:data_model] }.compact)
+      else
+              ConceptQL::Diagram.default_cdb(options[:data_model])
+      end
+      doc = ConceptQL::Diagram.render(criteria_from_file(statement_file), cdb: cdb, counts: options[:counts])
+      puts JSON.pretty_generate(doc)
     end
 
-    desc 'show_db_graph conceptql_id', 'Shows a graph for the conceptql statement represented by conceptql_id in the db specified by db_url'
-    def show_db_graph(conceptql_id)
-      result = fetch_conceptql(conceptql_id, options)
-      graph_it(result[:statement].to_hash, db, result[:label])
-    end
+    desc "metadata", "Generates the metadata.js file for the JAM"
+    def metadata
+      json = ConceptQL.metadata(cdb(options), warn: true).to_json
+      File.write("metadata.js", "$metadata = #{json};")
+      File.write("metadata.json", json)
+      return unless system("which", "json_pp", out: File::NULL, err: File::NULL)
 
-    def fetch_conceptql(conceptql_id)
-      my_db = db(options)
-      my_db.extension(:pg_array, :pg_json)
-      my_db[:concepts].where(concept_id: conceptql_id).select(:statement, :label).first
+      system("cat metadata.json | json_pp", out: "metadata_pp.json")
     end
 
-    def show_and_tell(statement, options, title = nil)
-      my_db = db(options)
-      q = ConceptQL::Query.new(my_db, statement)
-      puts 'YAML'
-      puts q.statement.to_yaml
-      puts 'JSON'
-      puts JSON.pretty_generate(q.statement)
-      graph_it(statement, title)
-      STDIN.gets
-      puts q.sql
-      STDIN.gets
-      results = q.all
-      if options[:full]
-        pp results
-      else
-        pp filtered(results)
-      end
-      puts results.length
+    desc "simple_visit STATEMENT_FILE",
+         "Prints the operators, depth and selection-node count of the statement in STATEMENT_FILE"
+    def simple_visit(statement_file)
+      pp cdb(options).query(criteria_from_file(statement_file)).extract_metadata
     end
 
-    def graph_it(statement, title = nil)
-      require_relative 'graph'
-      require_relative 'tree'
-      conn = db(options)
-      ConceptQL::Graph.new(statement,
-                           dangler: true,
-                           title: title,
-                           db: conn
-                          ).graph_it('/tmp/graph')
-      system('open /tmp/graph.pdf')
-      if options[:watch_file]
-        require_relative 'debugger'
-        debugger = ConceptQL::Debugger.new(statement, db: conn, watch_ids: File.readlines(options[:watch_file]).map(&:to_i))
-        debugger.capture_results('/tmp/debug.xlsx')
+    desc "dumpit", "Dumps out test data into CSVs"
+    def dumpit(path)
+      path = Pathname.new(path)
+      path.mkpath unless path.exist?
+      headers_path = path / "headers"
+      headers_path.mkpath unless headers_path.exist?
+      db.tables.each do |table|
+        puts "Dumping #{table}..."
+        ds = db[table]
+        rows = ds.select_map(ds.columns)
+        CSV.open(path + "#{table}.csv", "wb") do |csv|
+          rows.each { |row| csv << row }
+        end
+        CSV.open(headers_path + "#{table}.csv", "wb") do |csv|
+          csv << ds.columns
+        end
       end
     end
 
+    private
+
     def criteria_from_file(file)
       case File.extname(file)
-      when '.json'
+      when ".json"
         JSON.parse(File.read(file))
       else
         eval(File.read(file))
       end
     end
 
-    def filtered(results)
-      results.each { |r| r.delete_if { |k,v| v.nil? } }
+    def cdb(options)
+      data_db = db(options)
+      cdb = ConceptQL::Database.new(data_db)
+      # A glob of directories, in search path order: "/data/{wide,slim}".
+      if (parquet_dirs = ENV["CONCEPTQL_PARQUET_TEST_DIR"])
+        data_db.make_ready(search_path: Dir[parquet_dirs].map { |dir| Pathname.new(dir).glob("*.parquet") })
+      end
+      cdb
     end
   end
 end
-

data/lib/conceptql/columnizer.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  class Columnizer
+    class ColumnDefinition
+      attr_reader :name, :options
+
+      def initialize(name, options)
+        @name = name
+        @options = options
+      end
+
+      def grouped?
+        options[:grouped]
+      end
+
+      def sequelized
+        col = name
+        if (q = options[:qualifier])
+          col = Sequel.qualify(q, col)
+        end
+
+        if (fun = options[:function])
+          col = Sequel.function(fun, col)
+        end
+
+        unless options[:no_alias]
+          n = extract_name(name)
+          col = Sequel.expr(col).as(n) if n.present?
+        end
+
+        col
+      end
+
+      def extract_name(name)
+        return name if name.is_a?(Symbol)
+
+        n = nil
+        n = name.column if name.respond_to?(:column)
+        return n unless n.blank?
+
+        n = name.table if name.respond_to?(:table)
+        return n unless n.blank?
+
+        extract_name(name.expr) if name.respond_to?(:expr)
+      end
+    end
+
+    attr_reader :columns
+
+    def initialize
+      @columns = {}
+    end
+
+    def add_columns(*column_names)
+      opts = ConceptQL::Utils.extract_opts!(column_names)
+      column_names.each do |column_name|
+        columns[column_name] = ColumnDefinition.new(column_name, opts)
+      end
+    end
+
+    def apply(query)
+      grouped, ungrouped = columns.values.partition(&:grouped?)
+      select_method = :select
+      unless grouped.empty?
+        query = query.select_group(*grouped.map(&:sequelized))
+        select_method = :select_append
+      end
+
+      return query if ungrouped.empty?
+
+      query.send(select_method, *ungrouped.map(&:sequelized))
+    end
+  end
+end