conceptql 0.6.0 → 0.7.0

data/docs/spec/examples/vocabulary-operator.json

@@ -0,0 +1,232 @@
+{
+  "spec_id": "vocabulary-operator",
+  "examples": [
+    {
+      "title": "All records with the ICD-9-CM diagnosis code 412 (old myocardial infarction)",
+      "statement": ["icd9cm", "412"],
+      "help": true,
+      "columns": [
+        "person_id",
+        "criterion_id",
+        "criterion_table",
+        "criterion_domain",
+        "start_date",
+        "end_date",
+        "source_value"
+      ],
+      "rows": [
+        [
+          11,
+          82614,
+          "observations",
+          "condition_occurrence",
+          "2008-10-29",
+          "2008-10-29",
+          "412"
+        ],
+        [
+          17,
+          27269,
+          "observations",
+          "condition_occurrence",
+          "2008-08-25",
+          "2008-08-25",
+          "412"
+        ],
+        [
+          17,
+          46165,
+          "observations",
+          "condition_occurrence",
+          "2009-04-30",
+          "2009-04-30",
+          "412"
+        ],
+        [
+          38,
+          40556,
+          "observations",
+          "condition_occurrence",
+          "2010-02-12",
+          "2010-02-12",
+          "412"
+        ],
+        [
+          54,
+          74606,
+          "observations",
+          "condition_occurrence",
+          "2008-06-05",
+          "2008-06-05",
+          "412"
+        ],
+        [
+          60,
+          986,
+          "observations",
+          "condition_occurrence",
+          "2009-07-19",
+          "2009-07-22",
+          "412"
+        ],
+        [
+          66,
+          16171,
+          "observations",
+          "condition_occurrence",
+          "2009-07-25",
+          "2009-07-25",
+          "412"
+        ],
+        [
+          73,
+          52843,
+          "observations",
+          "condition_occurrence",
+          "2008-11-16",
+          "2008-11-16",
+          "412"
+        ],
+        [
+          77,
+          58165,
+          "observations",
+          "condition_occurrence",
+          "2010-10-06",
+          "2010-10-06",
+          "412"
+        ],
+        [
+          81,
+          1405,
+          "observations",
+          "condition_occurrence",
+          "2009-01-28",
+          "2009-01-30",
+          "412"
+        ]
+      ],
+      "tree": {
+        "format": "conceptql-diagram/v1",
+        "statements": [
+          {
+            "name": "icd9cm",
+            "base": "selection",
+            "humanName": "ICD-9 CM",
+            "vocabularyId": "ICD9CM",
+            "outputTypes": ["condition_occurrence"],
+            "parameters": {},
+            "values": ["412"],
+            "children": [],
+            "counts": {
+              "condition_occurrence": {
+                "rows": 48,
+                "n": 37
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "title": "All MS-DRG 205 records: other respiratory system diagnoses with a major complication or comorbidity",
+      "statement": ["ms_drg", "205"],
+      "help": true,
+      "columns": [
+        "person_id",
+        "criterion_id",
+        "criterion_table",
+        "criterion_domain",
+        "start_date",
+        "end_date",
+        "source_value"
+      ],
+      "rows": [
+        [
+          17,
+          1365,
+          "observations",
+          "procedure_occurrence",
+          "2010-06-02",
+          "2010-06-06",
+          "205"
+        ],
+        [
+          68,
+          18,
+          "observations",
+          "procedure_occurrence",
+          "2008-04-19",
+          "2008-04-23",
+          "205"
+        ],
+        [
+          95,
+          603,
+          "observations",
+          "procedure_occurrence",
+          "2008-08-27",
+          "2008-09-01",
+          "205"
+        ]
+      ],
+      "tree": {
+        "format": "conceptql-diagram/v1",
+        "statements": [
+          {
+            "name": "ms_drg",
+            "base": "selection",
+            "humanName": "MS-DRG",
+            "vocabularyId": "MS-DRG",
+            "outputTypes": ["procedure_occurrence"],
+            "parameters": {},
+            "values": ["205"],
+            "children": [],
+            "counts": {
+              "procedure_occurrence": {
+                "rows": 3,
+                "n": 3
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "title": "All CMS-DRG 205 records (a liver disorder)",
+      "statement": ["cms_drg", "205"],
+      "help": false,
+      "columns": [
+        "person_id",
+        "criterion_id",
+        "criterion_table",
+        "criterion_domain",
+        "start_date",
+        "end_date",
+        "source_value"
+      ],
+      "rows": [],
+      "tree": {
+        "format": "conceptql-diagram/v1",
+        "statements": [
+          {
+            "name": "cms_drg",
+            "base": "selection",
+            "humanName": "CMS-DRG",
+            "vocabularyId": "CMS-DRG",
+            "outputTypes": ["procedure_occurrence"],
+            "parameters": {},
+            "values": ["205"],
+            "children": [],
+            "counts": {
+              "procedure_occurrence": {
+                "rows": 0,
+                "n": 0
+              }
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

data/lib/conceptql/behaviors/metadatable.rb

@@ -47,22 +47,71 @@ module ConceptQL
       @deprecated = message
     end
 
+    # Marks an operator as hidden. It still runs, but ConceptQL.metadata
+    # (Nodifier#to_metadata) leaves it out, so an editor does not offer it.
+    # An operator is not hidden unless it says so. A subclass does not
+    # inherit the mark.
+    def hidden
+      @hidden = true
+    end
+
+    def hidden?
+      @hidden || false
+    end
+
     def predominant_domains(*values)
       return @predominant_domains if values.empty?
 
       @predominant_domains = values
     end
 
+    # An argument or option can carry two texts. instructions: is the form
+    # hint for typing a value. meaning: says what the argument or option
+    # does. Both are plain text, because the diagram editor escapes them.
     def argument(name, options = {})
       (@arguments ||= [])
       @arguments << [ name, auto_label(name, options) ]
     end
 
+    # The options that one argument was declared with
+    def argument_options(name)
+      (@arguments || []).to_h.fetch(name)
+    end
+
     def option(name, options = {})
       @options ||= {}
       @options[name] = auto_label(name, options)
     end
 
+    # Replaces the meaning: text of an option, usually one that this class
+    # inherited. The option hash is shared with the parent class (see
+    # #inherited), so this stores a copy and leaves the parent's text alone.
+    # An option that the class does not have raises KeyError.
+    def option_meaning(name, text)
+      @options[name] = @options.fetch(name).merge(meaning: text)
+    end
+
+    # The plain text that says which records the operator returns. A
+    # subclass does not inherit it.
+    def returns(value = nil)
+      return @returns unless value
+
+      @returns = value
+    end
+
+    # Adds an example: a title and a ConceptQL statement. help: false keeps
+    # the example out of an editor's help, for example because it returns no
+    # rows on the test data. The spec build generates the results and the
+    # diagram, so they are never written here. A subclass does not inherit
+    # the examples, because they would show the wrong operator.
+    def example(title, statement, help: true)
+      (@examples ||= []) << { title: title, statement: statement, help: help }
+    end
+
+    def examples
+      @examples || []
+    end
+
     # Takes an option that a subclass inherited out of its metadata
     def remove_option(name)
       @options.delete(name)
@@ -93,11 +142,16 @@ module ConceptQL
       @basic_type = value
     end
 
+    # min_upstreams and max_upstreams say how many positional upstreams pass
+    # validation (99 means no limit). A validate_*_upstream check sets them
+    # again when to_metadata runs (derive_metadata_from_validations).
     def allows_many_upstreams
+      @min_upstreams = 0
       @max_upstreams = 99
     end
 
     def allows_one_upstream
+      @min_upstreams = 0
       @max_upstreams = 1
     end
 
@@ -155,7 +209,7 @@ module ConceptQL
         short_name: short_name,
         long_name: long_name,
         operation: just_class_name.underscore,
-        min_upstreams: @max_upstreams || 0,
+        min_upstreams: @min_upstreams || 0,
         max_upstreams: @max_upstreams || 0,
         arguments: @arguments || [],
         options: process_options(@options),
@@ -167,10 +221,22 @@ module ConceptQL
         deprecated: (true if deprecated), # nil for an operator that is not deprecated
         arity: arity,
         aliases: @aliases,
-        spec_id: spec_id
+        spec_id: spec_id,
+        returns: get_returns,
+        examples: examples_for(name)
       }
     end
 
+    # The returns text that the metadata serves
+    def get_returns
+      returns
+    end
+
+    # The examples that the metadata of the operator with this name serves
+    def examples_for(_name)
+      examples
+    end
+
     def arity
       case basic_type
       when :set

data/lib/conceptql/behaviors/provenanceable.rb

@@ -76,8 +76,11 @@ module ConceptQL
       [ dm.file_provenance_types_vocab, dm.code_provenance_types_vocab ].flatten
     end
 
+    # The keywords come from the statement, whose validation reports a bad
+    # one (find_bad_keywords), so the lookup does not raise
+    # CodeCaseMismatchError.
     def prov_of(db, vocabularies, ancestors)
-      dm.descendants_of(db, dm.concept_ids(db, vocabularies, ancestors))
+      dm.descendants_of(db, dm.concept_ids(db, vocabularies, ancestors, check_case: false))
     end
 
     def plain_condition(db, codes)

data/lib/conceptql/behaviors/utilizable.rb

@@ -55,6 +55,9 @@ module ConceptQL
         db[name].with(name, list, Scope::SHARED_TABLE => true)
       end
 
+      # collection_type and "primary" are ConceptQL's own codes, so their
+      # lookups check case: a wrong case raises CodeCaseMismatchError instead
+      # of selecting nothing (conceptql-9okk, bead conceptql-rz8s).
       def gdm_it(db)
         source_type_id = dm.concept_ids(db, dm.file_provenance_types_vocab, collection_type)
         all_source_type_ids = dm.descendants_of(db, source_type_id)

data/lib/conceptql/code_case_mismatch_error.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  # A code lookup got codes that match no stored code exactly, but match a
+  # stored code of the same vocabulary when case is ignored (bead
+  # conceptql-rz8s). ConceptQL compares codes case-exactly (seed conceptql-oji
+  # ruling "b"), so without this error the lookup returns no concept for those
+  # codes, and the caller loses rows without a word. t_shank asked GDM_PROV for
+  # "line" when the vocabulary stores "LINE".
+  #
+  # The public lookups of ConceptQL::Lexicon (and of the DataModel, which
+  # delegates to it) raise it. A vocabulary operator in a statement does not:
+  # it passes check_case: false, and warns about the code instead.
+  class CodeCaseMismatchError < ArgumentError
+    # One code of one vocabulary. +stored+ is every spelling of the code that
+    # the vocabulary stores, sorted.
+    Mismatch = Data.define(:vocabulary_id, :code, :stored) do
+      def to_s
+        "The vocabulary #{vocabulary_id} stores #{code.inspect} as #{stored.map(&:inspect).join(" or ")}."
+      end
+    end
+
+    attr_reader :mismatches
+
+    def initialize(mismatches)
+      @mismatches = mismatches
+      super([
+        "A code lookup got codes that their vocabulary stores only in another case.",
+        "ConceptQL compares codes case-exactly, so these codes would match no concept:",
+        *mismatches.map { |mismatch| "  - #{mismatch}" },
+        "Pass each code as its vocabulary stores it."
+      ].join("\n"))
+    end
+
+    # The mismatches among +codes+, given +rows+: the [vocabulary_id,
+    # concept_code] pairs of the vocabularies' codes that equal one of the
+    # codes when case is ignored. A code that some vocabulary stores exactly
+    # is no mismatch. Neither is a code that no vocabulary stores in any case.
+    def self.mismatches(codes, rows)
+      codes.map(&:to_s).uniq.flat_map do |code|
+        same = rows.select { |_, stored| stored.casecmp?(code) }
+        next [] if same.empty? || same.any? { |_, stored| stored == code }
+
+        same.group_by(&:first).sort.map do |vocabulary_id, pairs|
+          Mismatch.new(vocabulary_id, code, pairs.map(&:last).uniq.sort)
+        end
+      end
+    end
+  end
+end

data/lib/conceptql/data_model/gdm.rb

@@ -127,10 +127,31 @@ module ConceptQL
         end
       end
 
-      def information_period_where_clause(arguments)
+      # Each argument is an information type concept ID or a concept code of
+      # information_types_vocab. A statement may mix both. A code that names
+      # no concept adds no ID, so it matches no period: it does not become
+      # concept ID 0. With only unknown codes, the clause matches nothing.
+      # Validation warns about an unknown code, so the lookup does not raise
+      # CodeCaseMismatchError.
+      def information_period_where_clause(db, arguments)
         return if arguments.empty?
 
-        { information_type_concept_id: arguments.map(&:to_i) }
+        codes = information_type_codes(arguments)
+        ids = (arguments - codes).map(&:to_i)
+        ids += concept_ids(db, information_types_vocab, codes, check_case: false) unless codes.empty?
+        { information_type_concept_id: ids }
+      end
+
+      # The arguments that are not concept IDs, as strings. A concept ID is
+      # an Integer or a string of digits.
+      def information_type_codes(arguments)
+        arguments.reject { |arg| arg.is_a?(Integer) || arg.to_s.match?(/\A\d+\z/) }.map(&:to_s)
+      end
+
+      # The vocabulary of the information type concepts, such as PARTD
+      # (Medicare Part D). The GDM and the OHDSI vocabulary tables both name it so.
+      def information_types_vocab
+        "GDM_INFORMATION_TYPE"
       end
 
       def code_provenance_type(_query, _domain)

data/lib/conceptql/diagram_renderer/app.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module DiagramRenderer
+    # The in-app SVG of a diagram: the <svg class="vis"> element that
+    # frontend/lib/conceptql-diagram/src/renderer.js builds (its constructor,
+    # then update() at duration 0), for a page to show and script. The root
+    # svg, the injected <style>, the hidden copy of the icon sprite, the
+    # gradients and the Drawing with hooks (data-node-id, data-insert-before,
+    # data-diagram-key; see Drawing).
+    #
+    # Fit-to-view stays in the browser: renderer.js measures the drawing with
+    # getBBox and sets the .vis-algorithm transform, so this SVG has none.
+    #
+    #   ConceptQL::DiagramRenderer::App.new(configuration).render(tree, lock_nodes: true) # => SVG text
+    class App
+      # XMLSerializer leaves out a namespace declaration that the element
+      # already inherits. The sprite <div> is in the SVG namespace (d3 appends
+      # it to the svg), so the browser's copy of the sprite has no
+      # xmlns="http://www.w3.org/2000/svg".
+      INHERITED_XMLNS = /\s+xmlns\s*=\s*(?:"#{Regexp.escape(Static::SVG_NS)}"|'#{Regexp.escape(Static::SVG_NS)}')/
+      START_TAG = /<[^\s!?\/>][^>]*>/
+
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      # tree is a diagram tree whose nodes have ids, isSelected and
+      # isAttachable (what Ember passes to the renderer, or DiagramData.from's
+      # output). It is drawn as it is. styles and icons are the renderer's
+      # options of those names: the CSS and the sprite markup (default: the
+      # configuration's).
+      def render(tree, styles: @configuration.styles, icons: @configuration.icons, lock_nodes: false, summary: false,
+        show_counts: false, show_options: false, css_transforms: false)
+        drawing = Drawing.new(tree, show_counts:, show_options:, hooks: true, lock_nodes:, css_transforms:)
+        Svg.element("svg", {
+          "xmlns" => Static::SVG_NS,
+          "xmlns:xlink" => Static::XLINK_NS,
+          "class" => lock_nodes ? "vis is-new" : "vis",
+          "height" => "100%",
+          "width" => "100%",
+          "viewBox" => summary ? "0 0 800 500" : nil
+        }, [
+          Svg.element("style", {}, styles.nil? ? [] : [ styles ]),
+          Svg.element("div", { "style" => "display: none;" }, icons.nil? || icons.empty? ? [] : [ Svg::Raw.new(serialized(icons)) ]),
+          Svg.element("defs", {}, drawing.gradients),
+          drawing.algorithm
+        ]).to_s
+      end
+
+      private
+
+      # The sprite markup as the browser serializes the <div> after its
+      # innerHTML is set. The markup must otherwise be in serialized form, as
+      # the sprite in frontend/index.html is.
+      def serialized(icons) = icons.gsub(START_TAG) { _1.gsub(INHERITED_XMLNS, "") }
+    end
+  end
+end

data/lib/conceptql/diagram_renderer/configuration.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module ConceptQL
+  module DiagramRenderer
+    # Where the renderer's assets are. Each path defaults to the copy that the
+    # gem ships under assets/diagram/ (ASSETS), so a host needs no configure
+    # call. A host that has its own copy of an asset sets that path:
+    #
+    #   ConceptQL::DiagramRenderer.configure do |config|
+    #     config.styles_path = "my_diagram.css"
+    #   end
+    #
+    # - sprite_path: a file that holds the operator icon <symbol>s (Sprite):
+    #   assets/diagram/operators.svg.
+    # - styles_path: the diagram CSS: assets/diagram/diagram.css.
+    # - webfonts_dir: the WEBFONTS woff2 files, embedded in the SVG's <style>:
+    #   assets/diagram/fonts.
+    # - resvg_fonts_dir: the Resvg::FONT_FILES TTF files for resvg:
+    #   assets/diagram/fonts.
+    # - resvg_program: the resvg executable, "resvg" (on PATH) by default.
+    #
+    # The files are read once, when first needed.
+    class Configuration
+      # The gem's asset directory.
+      ASSETS = ConceptQL.root.join("assets", "diagram")
+
+      # The Roboto webfonts and their font-style (loadAssets in
+      # cli/src/render-static.js).
+      WEBFONTS = [ [ "Roboto-Normal-Latin.woff2", "normal" ], [ "Roboto-Italic-Latin.woff2", "italic" ] ].freeze
+
+      # The rule for the root svg after the diagram CSS. animation: none is
+      # required: the diagram CSS fades .vis in after a delay, and an SVG shown
+      # as an image runs its CSS animations, so anything that snapshots the
+      # file early (thumbnailers, PDF converters) gets a blank.
+      ROOT_RULE = %(svg.vis { font: normal 15px/1.5 "Roboto", sans-serif; animation: none; cursor: default; }\n)
+
+      # An asset path is not set.
+      class Missing < StandardError; end
+
+      attr_accessor :sprite_path, :styles_path, :webfonts_dir, :resvg_fonts_dir, :resvg_program
+
+      def initialize
+        @sprite_path = ASSETS.join("operators.svg").to_s
+        @styles_path = ASSETS.join("diagram.css").to_s
+        @webfonts_dir = ASSETS.join("fonts").to_s
+        @resvg_fonts_dir = ASSETS.join("fonts").to_s
+        @resvg_program = "resvg"
+      end
+
+      # The sprite block in an HTML page (loadAssets in cli/src/render-static.js).
+      OPERATORS = %r{<div class="operators"[^>]*>(.*?)</div>\s*(?:<|\z)}m
+
+      def sprite = @sprite ||= Sprite.new(sprite_markup)
+
+      def styles = @styles ||= File.read(path(:styles_path))
+
+      # The icon markup that the in-app SVG copies (App): the content of the
+      # <div class="operators"> block when the sprite file has one, else the
+      # whole file.
+      def icons = @icons ||= sprite_markup[OPERATORS, 1] || sprite_markup
+
+      # The text of the standalone SVG's <style>: the embedded Roboto faces, the
+      # diagram CSS and ROOT_RULE (standaloneStyleText in
+      # cli/src/render-static.js).
+      def standalone_style_text
+        @standalone_style_text ||= "#{font_faces}\n#{styles}\n#{ROOT_RULE}"
+      end
+
+      def resvg = Resvg.new(fonts_dir: path(:resvg_fonts_dir), program: resvg_program)
+
+      private
+
+      def sprite_markup = @sprite_markup ||= File.read(path(:sprite_path))
+
+      def font_faces
+        WEBFONTS.map do |file, style|
+          data = [ File.binread(File.join(path(:webfonts_dir), file)) ].pack("m0")
+          %(@font-face { font-family: "Roboto"; font-style: #{style}; font-weight: 100 900; src: url(data:font/woff2;base64,#{data}) format("woff2"); })
+        end.join("\n")
+      end
+
+      def path(name)
+        public_send(name)&.to_s or raise Missing, "ConceptQL::DiagramRenderer: #{name} is not set (see ConceptQL::DiagramRenderer.configure)"
+      end
+    end
+  end
+end

data/lib/conceptql/diagram_renderer/diagram_data.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ConceptQL
+  module DiagramRenderer
+    # The input handling of the renderer: toDiagramData in
+    # frontend/lib/conceptql-diagram/cli/src/render-static.js.
+    #
+    # The input is parsed JSON (hashes with string keys) in one of three shapes:
+    #
+    # - a bare node: {"name" => "icd9", ...}
+    # - an array of statements: [{"name" => ...}, ...]
+    # - {"format" => "conceptql-diagram/v1", "statements" => [...]}, which is
+    #   what `conceptql render_json` prints
+    #
+    # The statements go under a synthetic root with path ids (n0, n0.0, ...).
+    # Each node gets outputTypes ["unknown"] when it has none, parameters {}
+    # when it has none, and isSelected and isAttachable false. Other keys stay
+    # as they are. A bad input raises InputError.
+    #
+    # The input may also have symbol keys, as ConceptQL::Diagram.render's
+    # output has: from writes the input as JSON and reads it back first, so
+    # it reads every input as parsed JSON.
+    module DiagramData
+      FORMAT = "conceptql-diagram/v1"
+
+      module_function
+
+      def from(input)
+        statements = statements_of(JSON.parse(JSON.generate(input)))
+        raise InputError, "no statements to render" unless statements.is_a?(Array) && !statements.empty?
+
+        children = statements.each_with_index.map { |statement, i| assign(statement, "n#{i}") }
+        {
+          "name" => "root",
+          "id" => "root",
+          "isSelected" => false,
+          "isAttachable" => false,
+          "outputTypes" => children.flat_map { _1["outputTypes"] }.uniq,
+          "children" => children
+        }
+      end
+
+      def statements_of(input)
+        if input.is_a?(Array)
+          input
+        elsif input.is_a?(Hash) && input.key?("statements")
+          unless input["format"] == FORMAT
+            format = input.key?("format") ? JSON.generate(input["format"]) : "undefined"
+            raise InputError, "unsupported format #{format}; expected \"#{FORMAT}\""
+          end
+          input["statements"]
+        elsif input.is_a?(Hash) && input["name"].is_a?(String)
+          [ input ]
+        else
+          raise InputError, 'input must be a diagram node, an array of nodes, or {"format": "conceptql-diagram/v1", "statements": [...]}'
+        end
+      end
+
+      def assign(node, id)
+        raise InputError, "node #{id} has no \"name\"" unless node.is_a?(Hash) && node["name"].is_a?(String)
+
+        output_types = node["outputTypes"]
+        parameters = node["parameters"]
+        children = node["children"]
+        node.merge(
+          "id" => id,
+          "outputTypes" => JsFormat.truthy?(output_types) && JsFormat.truthy?(JsFormat.length(output_types)) ? output_types : [ "unknown" ],
+          "parameters" => JsFormat.truthy?(parameters) ? parameters : {},
+          "isSelected" => false,
+          "isAttachable" => false,
+          "children" => (JsFormat.truthy?(children) ? children : []).each_with_index.map { |child, i| assign(child, "#{id}.#{i}") }
+        )
+      end
+
+      private_class_method :statements_of, :assign
+    end
+  end
+end