conceptql 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +498 -0
- data/config/vocabularies.csv +85 -87
- data/config/vocabulary_formats.sha +1 -0
- data/config/vocabulary_formats.tsv +39 -0
- data/doc/spec.md +2091 -0
- data/lib/conceptql/behaviors/metadatable.rb +15 -3
- data/lib/conceptql/behaviors/provenanceable.rb +82 -8
- data/lib/conceptql/behaviors/utilizable.rb +5 -3
- data/lib/conceptql/cli.rb +1 -1
- data/lib/conceptql/data_model/base.rb +3 -171
- data/lib/conceptql/data_model/gdm.rb +3 -1
- data/lib/conceptql/data_model.rb +20 -4
- data/lib/conceptql/database.rb +9 -2
- data/lib/conceptql/date_adjuster.rb +12 -5
- data/lib/conceptql/lexicon/lexicon_gdm.rb +4 -1
- data/lib/conceptql/lexicon/lexicon_ohdsi.rb +19 -1
- data/lib/conceptql/lexicon/lexicon_strategy.rb +58 -4
- data/lib/conceptql/lexicon.rb +218 -2
- data/lib/conceptql/nodifier.rb +7 -1
- data/lib/conceptql/one_query_counts.rb +120 -0
- data/lib/conceptql/operators/co_reported.rb +4 -31
- data/lib/conceptql/operators/count.rb +4 -0
- data/lib/conceptql/operators/death.rb +1 -1
- data/lib/conceptql/operators/dedup.rb +1 -1
- data/lib/conceptql/operators/episode.rb +10 -2
- data/lib/conceptql/operators/ethnicity.rb +0 -4
- data/lib/conceptql/operators/except.rb +5 -0
- data/lib/conceptql/operators/filter.rb +12 -2
- data/lib/conceptql/operators/hospice.rb +2 -3
- data/lib/conceptql/operators/hospitalization.rb +2 -3
- data/lib/conceptql/operators/information_periods.rb +5 -4
- data/lib/conceptql/operators/invalid.rb +7 -1
- data/lib/conceptql/operators/multiple_vocabularies.rb +19 -6
- data/lib/conceptql/operators/numeric.rb +51 -14
- data/lib/conceptql/operators/occurrence.rb +16 -4
- data/lib/conceptql/operators/one_in_two_out.rb +1 -2
- data/lib/conceptql/operators/operator.rb +66 -30
- data/lib/conceptql/operators/person.rb +1 -1
- data/lib/conceptql/operators/person_filter.rb +5 -0
- data/lib/conceptql/operators/place_of_service_filter.rb +38 -10
- data/lib/conceptql/operators/provenance.rb +13 -4
- data/lib/conceptql/operators/provider_filter.rb +5 -1
- data/lib/conceptql/operators/race.rb +2 -10
- data/lib/conceptql/operators/read.rb +1 -100
- data/lib/conceptql/operators/recall.rb +1 -1
- data/lib/conceptql/operators/snf.rb +3 -4
- data/lib/conceptql/operators/sum.rb +46 -5
- data/lib/conceptql/operators/temporal_operator.rb +6 -2
- data/lib/conceptql/operators/trim_date.rb +52 -6
- data/lib/conceptql/operators/trim_date_end.rb +11 -1
- data/lib/conceptql/operators/trim_date_start.rb +11 -1
- data/lib/conceptql/operators/union.rb +45 -1
- data/lib/conceptql/operators/vocabulary.rb +21 -4
- data/lib/conceptql/paths.rb +16 -0
- data/lib/conceptql/query.rb +78 -24
- data/lib/conceptql/query_modifiers/gdm/provider_query_modifier.rb +7 -7
- data/lib/conceptql/rdbms/generic.rb +21 -1
- data/lib/conceptql/rdbms/spark.rb +12 -0
- data/lib/conceptql/scope.rb +187 -24
- data/lib/conceptql/version.rb +1 -1
- data/lib/conceptql/vocabularies/drg_systems.rb +118 -0
- data/lib/conceptql/vocabularies/dynamic_vocabularies.rb +10 -4
- data/lib/conceptql/vocabularies/entry.rb +15 -23
- data/lib/conceptql/vocabularies/formats.rb +59 -0
- data/lib/conceptql/vocabularies/vocabulary_class_factory.rb +8 -16
- data/lib/conceptql/window/table.rb +12 -3
- metadata +9 -14
- data/lib/conceptql/data_model/omopv4_plus.rb +0 -12
- data/lib/conceptql/operators/concurrent_within.rb +0 -69
- data/lib/conceptql/query_modifiers/omopv4_plus/admission_date_query_modifier.rb +0 -45
- data/lib/conceptql/query_modifiers/omopv4_plus/drug_query_modifier.rb +0 -80
- data/lib/conceptql/query_modifiers/omopv4_plus/pos_query_modifier.rb +0 -47
- data/lib/conceptql/query_modifiers/omopv4_plus/provenance_query_modifier.rb +0 -45
- data/lib/conceptql/query_modifiers/omopv4_plus/provider_query_modifier.rb +0 -26
- data/lib/conceptql/vocabularies/behaviors/costish.rb +0 -24
- data/lib/conceptql/vocabularies/behaviors/omopish.rb +0 -32
- data/schemas/omopv4_plus.yml +0 -729
- data/schemas/omopv4_plus_more_ignorables.tsv +0 -13
|
@@ -5,11 +5,10 @@ require_relative "operator"
|
|
|
5
5
|
module ConceptQL
|
|
6
6
|
module Operators
|
|
7
7
|
class Hospice < Operator
|
|
8
|
-
|
|
9
|
-
register __FILE__, :gdm, :gdm_wide
|
|
8
|
+
register __FILE__
|
|
10
9
|
include ConceptQL::Behaviors::Utilizable
|
|
11
10
|
|
|
12
|
-
desc "Selects admission records
|
|
11
|
+
desc "Selects hospice admission records."
|
|
13
12
|
|
|
14
13
|
def collection_type
|
|
15
14
|
"hospice"
|
|
@@ -5,11 +5,10 @@ require_relative "operator"
|
|
|
5
5
|
module ConceptQL
|
|
6
6
|
module Operators
|
|
7
7
|
class Hospitalization < Operator
|
|
8
|
-
|
|
9
|
-
register __FILE__, :gdm, :gdm_wide
|
|
8
|
+
register __FILE__
|
|
10
9
|
include ConceptQL::Behaviors::Utilizable
|
|
11
10
|
|
|
12
|
-
desc "Selects admission records
|
|
11
|
+
desc "Selects inpatient admission records."
|
|
13
12
|
|
|
14
13
|
def collection_type
|
|
15
14
|
"inpatient"
|
|
@@ -10,7 +10,10 @@ module ConceptQL
|
|
|
10
10
|
include ConceptQL::Behaviors::Windowable
|
|
11
11
|
include ConceptQL::Behaviors::Timeless
|
|
12
12
|
|
|
13
|
-
desc "
|
|
13
|
+
desc "Selects all information period records, which have the observation_period domain. " \
|
|
14
|
+
"The operator takes no upstream stream. " \
|
|
15
|
+
"Each argument is an information type concept ID, and with arguments, " \
|
|
16
|
+
"the operator selects only the information periods of those types."
|
|
14
17
|
domains :observation_period
|
|
15
18
|
category "Get Related Data"
|
|
16
19
|
basic_type :selection
|
|
@@ -23,9 +26,7 @@ module ConceptQL
|
|
|
23
26
|
end
|
|
24
27
|
|
|
25
28
|
def source_table
|
|
26
|
-
|
|
27
|
-
tab = :payer_plan_period if tab == :observation_period && arguments.present?
|
|
28
|
-
tab
|
|
29
|
+
dm.table_by_domain(domain)
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
def where_clause
|
|
@@ -8,13 +8,19 @@ module ConceptQL
|
|
|
8
8
|
no_desc
|
|
9
9
|
default_query_columns
|
|
10
10
|
|
|
11
|
+
# The statement [nil] or [""] has no operator name. It is named
|
|
12
|
+
# "invalid", as an empty statement is, so it does not report its error
|
|
13
|
+
# under the key "" (bead conceptql-1ci).
|
|
11
14
|
def initialize(*args)
|
|
12
15
|
super
|
|
16
|
+
@op_name = "invalid" if op_name.to_s.empty?
|
|
13
17
|
@errors = @options.delete(:errors)
|
|
14
18
|
end
|
|
15
19
|
|
|
20
|
+
# The name in the statement. Operator#operator_name would give the class
|
|
21
|
+
# name, "invalid", for every unknown operator.
|
|
16
22
|
def operator_name
|
|
17
|
-
|
|
23
|
+
op_name
|
|
18
24
|
end
|
|
19
25
|
|
|
20
26
|
def query(_db)
|
|
@@ -59,11 +59,25 @@ module ConceptQL
|
|
|
59
59
|
# sharing a common table and feed them into a single single query,
|
|
60
60
|
# but I think this would require some revamping of Vocabulary, and I'm
|
|
61
61
|
# just not interested in taking that on right now.
|
|
62
|
+
all = disjoint_vocabularies?
|
|
62
63
|
vocab_ops.map { |vo| vo.evaluate(db) }.inject do |union, q|
|
|
63
|
-
union.union(q.from_self)
|
|
64
|
+
union.union(q.from_self, all: all)
|
|
64
65
|
end
|
|
65
66
|
end
|
|
66
67
|
|
|
68
|
+
# Each vocabulary operator selects only the rows whose vocabulary column
|
|
69
|
+
# holds its own vocabulary. A row holds one vocabulary, and each output
|
|
70
|
+
# row carries its source row's id as criterion_id, so operators with
|
|
71
|
+
# different vocabularies on one column never produce the same row. Then
|
|
72
|
+
# UNION ALL skips a duplicate removal that is expensive on large data
|
|
73
|
+
# (conceptql-5pz). A repeated row inside one branch now passes through,
|
|
74
|
+
# as it does for a single vocabulary operator and for union. A custom
|
|
75
|
+
# file can list one vocabulary twice, so those branches keep UNION.
|
|
76
|
+
def disjoint_vocabularies?
|
|
77
|
+
conditions = vocab_ops.map(&:vocabulary_condition)
|
|
78
|
+
conditions.map(&:keys).uniq.length == 1 && conditions.uniq.length == conditions.length
|
|
79
|
+
end
|
|
80
|
+
|
|
67
81
|
def domains(_db)
|
|
68
82
|
vocab_ops.map(&:domain).uniq
|
|
69
83
|
end
|
|
@@ -92,10 +106,9 @@ module ConceptQL
|
|
|
92
106
|
end
|
|
93
107
|
|
|
94
108
|
# The per-code warnings a Vocabulary operator gives, from most to least
|
|
95
|
-
# specific. See
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
CODE_WARNINGS = [ BAD_FORMAT, UNKNOWN_CODE ].freeze
|
|
109
|
+
# specific (Operator::BAD_FORMATS, Operator::UNKNOWN_CODE). See
|
|
110
|
+
# #code_warnings.
|
|
111
|
+
CODE_WARNINGS = [ *BAD_FORMATS, UNKNOWN_CODE ].freeze
|
|
99
112
|
|
|
100
113
|
def additional_validation(db, opts = {})
|
|
101
114
|
vocab_ops.each { |vo| vo.valid?(db, opts) }
|
|
@@ -118,7 +131,7 @@ module ConceptQL
|
|
|
118
131
|
end
|
|
119
132
|
|
|
120
133
|
rejected = arguments.uniq.select { |code| verdicts.all? { |verdict| verdict.key?(code) } }
|
|
121
|
-
bad_format, unknown = rejected.partition { |code| verdicts.all? { |verdict| verdict[code]
|
|
134
|
+
bad_format, unknown = rejected.partition { |code| verdicts.all? { |verdict| BAD_FORMATS.include?(verdict[code]) } }
|
|
122
135
|
|
|
123
136
|
[ [ BAD_FORMAT, *bad_format ], [ UNKNOWN_CODE, *unknown ] ].reject { |warning| warning.length == 1 }
|
|
124
137
|
end
|
|
@@ -16,17 +16,11 @@ module ConceptQL
|
|
|
16
16
|
class Numeric < PassThru
|
|
17
17
|
register __FILE__
|
|
18
18
|
|
|
19
|
-
desc
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
- Or a value from a column in the origin row
|
|
25
|
-
|
|
26
|
-
Accepts two params:
|
|
27
|
-
- Either a number value or a symbol representing a column name
|
|
28
|
-
- An optional stream
|
|
29
|
-
EOF
|
|
19
|
+
desc "Sets the value_as_number of each record of its upstream stream. " \
|
|
20
|
+
"The argument is a number, or the name of a column whose value is copied from the same record. " \
|
|
21
|
+
"Without an upstream stream, the operator selects one person record for each person, " \
|
|
22
|
+
"with value_as_number set to the argument. " \
|
|
23
|
+
"A column name must then name a column of the person table."
|
|
30
24
|
argument :value, type: :float
|
|
31
25
|
allows_one_upstream
|
|
32
26
|
validate_at_most_one_upstream
|
|
@@ -54,6 +48,39 @@ module ConceptQL
|
|
|
54
48
|
|
|
55
49
|
private
|
|
56
50
|
|
|
51
|
+
# A column name in the argument must be a column of the rows it is read
|
|
52
|
+
# from, or the statement fails at run time. The match ignores case,
|
|
53
|
+
# because DuckDB and Spark resolve an identifier without regard to case,
|
|
54
|
+
# and such a statement runs on them today.
|
|
55
|
+
def additional_validation(_db, _opts = {})
|
|
56
|
+
name = column_argument
|
|
57
|
+
return if name.nil? || matched_column
|
|
58
|
+
|
|
59
|
+
add_error("unknown column", name)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The column that the argument names, spelled as the rows spell it, or
|
|
63
|
+
# nil when the argument names no readable column. The SQL quotes the
|
|
64
|
+
# name, and Postgres folds only unquoted names, so the SQL must use this
|
|
65
|
+
# spelling for a match in another case to run there too.
|
|
66
|
+
def matched_column
|
|
67
|
+
name = column_argument
|
|
68
|
+
name && readable_columns.find { |column| column.to_s.casecmp?(name) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# With an upstream, a column name is read from the upstream's rows
|
|
72
|
+
# (SELECT ... FROM (<upstream>)), not from the tables the upstream
|
|
73
|
+
# reads. Every stream emits the same columns, the scope's query columns,
|
|
74
|
+
# so one check covers an upstream that mixes tables, such as a union of
|
|
75
|
+
# two domains. Without an upstream, the name is read from the person
|
|
76
|
+
# table that as_criterion selects from, so it must be one of that
|
|
77
|
+
# table's columns in the data model's schema.
|
|
78
|
+
def readable_columns
|
|
79
|
+
return scope.query_columns if stream
|
|
80
|
+
|
|
81
|
+
dm.table_cols(dm.table_by_domain(:person))
|
|
82
|
+
end
|
|
83
|
+
|
|
57
84
|
def with_kids(db)
|
|
58
85
|
dm.selectify(db.from(stream.evaluate(db)), replace: { value_as_number: first_argument })
|
|
59
86
|
end
|
|
@@ -79,15 +106,25 @@ module ConceptQL
|
|
|
79
106
|
end
|
|
80
107
|
|
|
81
108
|
def first_argument
|
|
82
|
-
value =
|
|
83
|
-
value = nil if value.blank?
|
|
109
|
+
value = argument_text
|
|
84
110
|
if !value || is_a_number?(value)
|
|
85
111
|
Sequel.cast_numeric(value, Float)
|
|
86
112
|
else
|
|
87
|
-
Sequel.identifier(value)
|
|
113
|
+
Sequel.identifier(matched_column || value)
|
|
88
114
|
end
|
|
89
115
|
end
|
|
90
116
|
|
|
117
|
+
def argument_text
|
|
118
|
+
value = arguments.first.to_s.strip
|
|
119
|
+
value.blank? ? nil : value
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The argument when it names a column, or nil when it is a number or blank
|
|
123
|
+
def column_argument
|
|
124
|
+
value = argument_text
|
|
125
|
+
value if value && !is_a_number?(value)
|
|
126
|
+
end
|
|
127
|
+
|
|
91
128
|
def is_a_number?(value)
|
|
92
129
|
value =~ /\A[-+]?[0-9]*\.?[0-9]+\Z/
|
|
93
130
|
end
|
|
@@ -33,7 +33,9 @@ module ConceptQL
|
|
|
33
33
|
|
|
34
34
|
preferred_name "Nth Occurrence"
|
|
35
35
|
|
|
36
|
-
desc "
|
|
36
|
+
desc "For each person, orders the records by start_date and passes along the record at the given position. " \
|
|
37
|
+
"A positive number counts from the earliest record, so 2 means the second record. " \
|
|
38
|
+
"A negative number counts from the most recent record, so -3 means the third record from the last."
|
|
37
39
|
|
|
38
40
|
argument :occurrence, type: :integer
|
|
39
41
|
category "Filter Single Stream"
|
|
@@ -73,7 +75,10 @@ module ConceptQL
|
|
|
73
75
|
additional_order_columns = []
|
|
74
76
|
|
|
75
77
|
if grouped?
|
|
76
|
-
|
|
78
|
+
# Name the columns to keep. select_remove(:rn) asked the database
|
|
79
|
+
# for them, and the CTE of a labeled upstream does not exist while
|
|
80
|
+
# the SQL is built (bead conceptql-6k8).
|
|
81
|
+
one_date_ds = ds.from_self.where(rn: 1).from_self.select(*dynamic_columns)
|
|
77
82
|
ds = one_date_ds.select_append(
|
|
78
83
|
Sequel[:ROW_NUMBER].function.over(
|
|
79
84
|
partition: matching_columns_plus,
|
|
@@ -114,10 +119,17 @@ module ConceptQL
|
|
|
114
119
|
).as(:timed_rn)
|
|
115
120
|
)
|
|
116
121
|
|
|
117
|
-
|
|
122
|
+
# Each first row nominates its nth rest row. With at_least, several
|
|
123
|
+
# first rows can nominate the same rest row: all rest rows that start
|
|
124
|
+
# after the same gap share one ranking. The join gives one row per
|
|
125
|
+
# nomination, so take the nominated rows from the input instead, and
|
|
126
|
+
# each comes out once (every stream is a set, seed conceptql-0eb).
|
|
127
|
+
nth_ds = db[joined_name].where(timed_rn: occurrence.abs)
|
|
128
|
+
keys = matching_columns_plus(:global_rn).map { |c| Sequel[:l][c] =~ Sequel[:r][c] }
|
|
129
|
+
|
|
130
|
+
rdbms.semi_join(db[input_name], nth_ds, *keys)
|
|
118
131
|
.with(input_name, input_ds)
|
|
119
132
|
.with(joined_name, joined_ds)
|
|
120
|
-
.where(timed_rn: occurrence.abs)
|
|
121
133
|
end
|
|
122
134
|
|
|
123
135
|
def grouped?
|
|
@@ -10,8 +10,7 @@ module ConceptQL
|
|
|
10
10
|
class OneInTwoOut < Operator
|
|
11
11
|
include ConceptQL::Behaviors::SingleEvent
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
register __FILE__, :gdm, :gdm_wide
|
|
13
|
+
register __FILE__
|
|
15
14
|
|
|
16
15
|
desc "Identifies an event that appears either once in an inpatient setting or twice within a specified interval in an outpatient setting."
|
|
17
16
|
|
|
@@ -9,7 +9,7 @@ module ConceptQL
|
|
|
9
9
|
# Built-in operators only: the classes that call Operator.register when
|
|
10
10
|
# their file loads. Vocabulary operators are not in here; each
|
|
11
11
|
# ConceptQL::Database builds its own from its lexicon (Database#operators).
|
|
12
|
-
STATIC_OPERATORS = {
|
|
12
|
+
STATIC_OPERATORS = { gdm: {}, gdm_wide: {} }.freeze
|
|
13
13
|
|
|
14
14
|
SELECTED_COLUMNS = %i[person_id
|
|
15
15
|
criterion_id
|
|
@@ -33,6 +33,22 @@ module ConceptQL
|
|
|
33
33
|
|
|
34
34
|
attr_reader :nodifier, :values, :options, :arguments, :upstreams, :op_name, :errors, :warnings
|
|
35
35
|
|
|
36
|
+
# The per-code warnings of a Vocabulary operator. A code that does not
|
|
37
|
+
# match its vocabulary's format is a warning, and the statement stays
|
|
38
|
+
# valid (Ryan ruled 2026-10-30, seeds conceptql-0eb and conceptql-ysx).
|
|
39
|
+
# The malformed code matches no rows. The formats come from
|
|
40
|
+
# vocabulary_formats (Vocabularies::Formats); a vocabulary without one
|
|
41
|
+
# is not checked. A well-formed code that the vocabulary does not know
|
|
42
|
+
# is also a warning.
|
|
43
|
+
BAD_FORMAT = "improperly formatted code"
|
|
44
|
+
# Claims carry the 11-digit package NDC (jde found no 9-digit product
|
|
45
|
+
# code in 601.9M claim NDCs, seed conceptql-ysx), so the NDC warning
|
|
46
|
+
# says why the code matches nothing. It starts with BAD_FORMAT, which
|
|
47
|
+
# the JDE algorithm spec looks for (bead conceptql-28h).
|
|
48
|
+
NDC_BAD_FORMAT = "#{BAD_FORMAT} (claims use 11-digit package NDCs, so it matches no rows)"
|
|
49
|
+
BAD_FORMATS = [ BAD_FORMAT, NDC_BAD_FORMAT ].freeze
|
|
50
|
+
UNKNOWN_CODE = "unknown code(s)"
|
|
51
|
+
|
|
36
52
|
option :label, type: :string
|
|
37
53
|
|
|
38
54
|
@validations = []
|
|
@@ -48,12 +64,6 @@ module ConceptQL
|
|
|
48
64
|
end
|
|
49
65
|
end
|
|
50
66
|
|
|
51
|
-
def query_columns(*tables)
|
|
52
|
-
define_method(:query_cols) do
|
|
53
|
-
dm.table_columns(*tables)
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
|
|
57
67
|
def default_query_columns
|
|
58
68
|
define_method(:query_cols) do
|
|
59
69
|
dynamic_columns
|
|
@@ -86,9 +96,9 @@ module ConceptQL
|
|
|
86
96
|
end
|
|
87
97
|
end
|
|
88
98
|
|
|
89
|
-
# Declares
|
|
90
|
-
#
|
|
91
|
-
# fails validation.
|
|
99
|
+
# Declares options that this operator does not support, inherited or
|
|
100
|
+
# not. An inherited one leaves the operator's metadata, and a
|
|
101
|
+
# statement that sets any of them fails validation.
|
|
92
102
|
def unsupported_options(*names)
|
|
93
103
|
names.each { |name| remove_option(name) }
|
|
94
104
|
validations << [ :validate_unsupported_options, *names ]
|
|
@@ -170,15 +180,8 @@ module ConceptQL
|
|
|
170
180
|
end
|
|
171
181
|
res = [ op_name, *annotate_values(db, opts) ]
|
|
172
182
|
|
|
173
|
-
if
|
|
174
|
-
scope.
|
|
175
|
-
.from_self
|
|
176
|
-
.select_group(:criterion_domain)
|
|
177
|
-
.select_append { Sequel.function(:count, 1).as(:rows) }
|
|
178
|
-
.select_append { count(:person_id).distinct.as(:n) }
|
|
179
|
-
.each do |h|
|
|
180
|
-
counts[h.delete(:criterion_domain).to_sym] = h
|
|
181
|
-
end
|
|
183
|
+
if countable?(db, opts)
|
|
184
|
+
counts.merge!(scope.one_query_counts&.for(self) || domain_counts(db))
|
|
182
185
|
elsif !errors.empty?
|
|
183
186
|
annotation[:errors] = errors
|
|
184
187
|
scope.add_errors(scope_key, errors)
|
|
@@ -188,6 +191,9 @@ module ConceptQL
|
|
|
188
191
|
cur_counts = counts[domain] ||= { rows: 0, n: 0 }
|
|
189
192
|
scope.add_counts(scope_key, domain, cur_counts)
|
|
190
193
|
end
|
|
194
|
+
# The database returns its groups in no fixed order, and the zero
|
|
195
|
+
# counts above come last. Sort them, so each run gives the same order.
|
|
196
|
+
annotation[:counts] = counts.sort.to_h
|
|
191
197
|
|
|
192
198
|
if defined?(@warnings) && !warnings.empty?
|
|
193
199
|
annotation[:warnings] = warnings
|
|
@@ -211,9 +217,15 @@ module ConceptQL
|
|
|
211
217
|
"<##{self.class} upstreams=[#{upstreams.map(&:inspect).join(', ')}] arguments=[#{arguments.map(&:inspect).join(', ')}]>"
|
|
212
218
|
end
|
|
213
219
|
|
|
220
|
+
# The operator's rows, read from a CTE that Scope#with_ctes moves to the
|
|
221
|
+
# statement's WITH list. With CONCEPTQL_AVOID_CTES, the rows are a
|
|
222
|
+
# subquery instead, because with_ctes then makes no WITH list.
|
|
214
223
|
def evaluate(db)
|
|
215
224
|
q = select_it(query(db))
|
|
225
|
+
return q.from_self if ConceptQL.avoid_ctes?
|
|
226
|
+
|
|
216
227
|
name = cte_name(op_name)
|
|
228
|
+
scope.record_cte(self, name)
|
|
217
229
|
opts = {}
|
|
218
230
|
opts[:materialized] = true if rdbms.supports_materialized?
|
|
219
231
|
db[name].with(name, q, opts)
|
|
@@ -322,6 +334,26 @@ module ConceptQL
|
|
|
322
334
|
valid?(db, opts) && upstreams.all? { |u| u.upstreams_valid?(db, opts) }
|
|
323
335
|
end
|
|
324
336
|
|
|
337
|
+
# Whether #annotate counts this operator's rows and people.
|
|
338
|
+
def countable?(db, opts = {})
|
|
339
|
+
upstreams_valid?(db, opts) && scope.valid? && include_counts?(db, opts)
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
# Rows and distinct people of each criterion_domain in this operator's
|
|
343
|
+
# output, with a query for this operator alone.
|
|
344
|
+
def domain_counts(db)
|
|
345
|
+
counts = {}
|
|
346
|
+
scope.with_ctes(self, db)
|
|
347
|
+
.from_self
|
|
348
|
+
.select_group(:criterion_domain)
|
|
349
|
+
.select_append { Sequel.function(:count, 1).as(:rows) }
|
|
350
|
+
.select_append { count(:person_id).distinct.as(:n) }
|
|
351
|
+
.each do |h|
|
|
352
|
+
counts[h.delete(:criterion_domain).to_sym] = h
|
|
353
|
+
end
|
|
354
|
+
counts
|
|
355
|
+
end
|
|
356
|
+
|
|
325
357
|
def scope
|
|
326
358
|
nodifier.scope
|
|
327
359
|
end
|
|
@@ -348,14 +380,6 @@ module ConceptQL
|
|
|
348
380
|
end
|
|
349
381
|
end
|
|
350
382
|
|
|
351
|
-
def omopv4_plus?
|
|
352
|
-
data_model == :omopv4_plus
|
|
353
|
-
end
|
|
354
|
-
|
|
355
|
-
def gdm?
|
|
356
|
-
%i[gdm gdm_wide].include?(data_model)
|
|
357
|
-
end
|
|
358
|
-
|
|
359
383
|
def wide?
|
|
360
384
|
%i[gdm_wide].include?(data_model)
|
|
361
385
|
end
|
|
@@ -467,10 +491,10 @@ module ConceptQL
|
|
|
467
491
|
cast_column(:file_provenance_type, dm.provenance_type_column(query, table))
|
|
468
492
|
end
|
|
469
493
|
|
|
470
|
-
def provider_id(query,
|
|
494
|
+
def provider_id(query, _table)
|
|
471
495
|
return :provider_id if query_columns(query).include?(:provider_id)
|
|
472
496
|
|
|
473
|
-
cast_column(:provider_id
|
|
497
|
+
cast_column(:provider_id)
|
|
474
498
|
end
|
|
475
499
|
|
|
476
500
|
def determine_tables
|
|
@@ -511,6 +535,7 @@ module ConceptQL
|
|
|
511
535
|
@errors = [] unless defined?(@errors)
|
|
512
536
|
@warnings = [] unless defined?(@warnings)
|
|
513
537
|
|
|
538
|
+
add_warning("deprecated operator", self.class.deprecated) if self.class.deprecated
|
|
514
539
|
add_error("invalid label") if label && !label.is_a?(String)
|
|
515
540
|
self.class.validations.each do |args|
|
|
516
541
|
send(*args)
|
|
@@ -596,7 +621,12 @@ module ConceptQL
|
|
|
596
621
|
def validate_codes_match
|
|
597
622
|
return if bad_arguments.empty?
|
|
598
623
|
|
|
599
|
-
add_warning(
|
|
624
|
+
add_warning(bad_format_message, *bad_arguments)
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
# The text of the format warning. Vocabulary gives the NDC its reason.
|
|
628
|
+
def bad_format_message
|
|
629
|
+
BAD_FORMAT
|
|
600
630
|
end
|
|
601
631
|
|
|
602
632
|
def additional_validation(_db, _opts = {})
|
|
@@ -642,8 +672,14 @@ module ConceptQL
|
|
|
642
672
|
table = Sequel[table] if table.is_a?(Symbol)
|
|
643
673
|
expr = exprs.inject(&:&)
|
|
644
674
|
if use_inner_join?
|
|
675
|
+
# The join gives one copy of a left row for each matching right row.
|
|
676
|
+
# Every stream is a set (seed conceptql-0eb), so remove the copies.
|
|
677
|
+
# A caller can append right columns to the select list (temporal
|
|
678
|
+
# include_rhs_columns), and DISTINCT covers them: rows that differ
|
|
679
|
+
# by a right column stay (bead conceptql-3pr).
|
|
645
680
|
ds.join(table.as(:r), expr)
|
|
646
681
|
.select(*query_cols.map { |c| Sequel[:l][c] })
|
|
682
|
+
.distinct
|
|
647
683
|
else
|
|
648
684
|
rdbms.semi_join(ds, table, *exprs)
|
|
649
685
|
end
|
|
@@ -10,7 +10,7 @@ module ConceptQL
|
|
|
10
10
|
|
|
11
11
|
register __FILE__
|
|
12
12
|
|
|
13
|
-
desc "Selects all records
|
|
13
|
+
desc "Selects all person records. With an upstream stream, the operator selects the person record of each person in that stream."
|
|
14
14
|
|
|
15
15
|
allows_one_upstream
|
|
16
16
|
domains :person
|
|
@@ -10,6 +10,11 @@ module ConceptQL
|
|
|
10
10
|
desc "Passes along left hand records with a corresponding person_id in the right hand records."
|
|
11
11
|
default_query_columns
|
|
12
12
|
|
|
13
|
+
# PersonFilter has no `as` option, and a statement that sets one fails
|
|
14
|
+
# validation instead of running as a plain person_filter (bead
|
|
15
|
+
# conceptql-dlq).
|
|
16
|
+
unsupported_options :as
|
|
17
|
+
|
|
13
18
|
def query(db)
|
|
14
19
|
db.from(left.evaluate(db))
|
|
15
20
|
.where(person_id: right.evaluate(db).from_self.select_group(:person_id))
|
|
@@ -4,22 +4,37 @@ require_relative "operator"
|
|
|
4
4
|
|
|
5
5
|
module ConceptQL
|
|
6
6
|
module Operators
|
|
7
|
-
# Filters the incoming stream of records to only those
|
|
8
|
-
#
|
|
7
|
+
# Filters the incoming stream of records to only those whose place of
|
|
8
|
+
# service is one of the given codes of the CMS Place of Service
|
|
9
|
+
# vocabulary.
|
|
9
10
|
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
11
|
+
# A record's place of service is pos_concept_id: a column of the record's
|
|
12
|
+
# context in gdm and of observations in gdm_wide. The PoS query modifiers
|
|
13
|
+
# bring it into the stream as visit_source_concept_id.
|
|
12
14
|
#
|
|
13
|
-
# If the record has NULL for
|
|
14
|
-
# out.
|
|
15
|
+
# If the record has NULL for its place of service, it is filtered out.
|
|
15
16
|
#
|
|
16
|
-
# Multiple
|
|
17
|
+
# Multiple places of service can be specified at once. Each code becomes
|
|
18
|
+
# a concept ID through the lexicon, so a code that names no concept can
|
|
19
|
+
# match no record. Such a code is an error, as an unrecognized
|
|
20
|
+
# provenance keyword is.
|
|
21
|
+
#
|
|
22
|
+
# The code "*" keeps a record whose place of service is any concept of
|
|
23
|
+
# the vocabulary, as provenance's "*" keeps any provenance concept. A
|
|
24
|
+
# pos_concept_id of another vocabulary is not a place of service. Because
|
|
25
|
+
# codes are ORed, "*" with other codes is "*". The other codes are still
|
|
26
|
+
# checked.
|
|
17
27
|
class PlaceOfServiceFilter < Operator
|
|
18
28
|
register __FILE__
|
|
19
29
|
|
|
20
|
-
|
|
30
|
+
VOCABULARY_ID = "CMS Place of Service"
|
|
31
|
+
ANY = "*"
|
|
32
|
+
|
|
33
|
+
desc "Passes along the records whose place of service is one of the given CMS Place of Service codes. " \
|
|
34
|
+
"The code * matches a record that has any place of service. " \
|
|
35
|
+
"A record that has no place of service is not passed along."
|
|
21
36
|
|
|
22
|
-
argument :places_of_service, type: :codelist, vocab:
|
|
37
|
+
argument :places_of_service, type: :codelist, vocab: VOCABULARY_ID
|
|
23
38
|
category "Filter Single Stream"
|
|
24
39
|
basic_type :temporal
|
|
25
40
|
allows_one_upstream
|
|
@@ -35,8 +50,21 @@ module ConceptQL
|
|
|
35
50
|
|
|
36
51
|
private
|
|
37
52
|
|
|
53
|
+
# With no codes, the lookup returns every concept of the vocabulary.
|
|
38
54
|
def place_of_service_concept_ids(db)
|
|
39
|
-
|
|
55
|
+
codes = arguments.map(&:to_s)
|
|
56
|
+
dm.concept_ids(db, VOCABULARY_ID, codes.include?(ANY) ? [] : codes)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# With no lexicon there are no codes to check against, and every code
|
|
60
|
+
# would be reported. Provenance skips its check in the same situation.
|
|
61
|
+
# A code is known only in its own case, as the lookup finds it.
|
|
62
|
+
def additional_validation(db, _opts = {})
|
|
63
|
+
return if dm.lexicon.strategy == :no_db
|
|
64
|
+
|
|
65
|
+
known = dm.concept_codes(db, VOCABULARY_ID)
|
|
66
|
+
unknown = arguments.reject { |code| code.to_s == ANY || known.include?(code.to_s) }
|
|
67
|
+
add_error(UNKNOWN_CODE, *unknown) unless unknown.empty?
|
|
40
68
|
end
|
|
41
69
|
end
|
|
42
70
|
end
|
|
@@ -14,12 +14,19 @@ module ConceptQL
|
|
|
14
14
|
# If the record has NULL for the provenance-related field, they are filtered
|
|
15
15
|
# out.
|
|
16
16
|
#
|
|
17
|
-
# Multiple provenances can be specified at once
|
|
17
|
+
# Multiple provenances can be specified at once. "*" asks for any
|
|
18
|
+
# provenance. At least one keyword is required.
|
|
18
19
|
class Provenance < Operator
|
|
19
|
-
|
|
20
|
-
register __FILE__, :gdm, :gdm_wide
|
|
20
|
+
register __FILE__
|
|
21
21
|
|
|
22
|
-
desc "Passes along records
|
|
22
|
+
desc "Passes along the records whose provenance matches at least one of the given keywords. " \
|
|
23
|
+
"Each record has a file provenance, such as inpatient or outpatient, " \
|
|
24
|
+
"and a code provenance, such as primary or first. " \
|
|
25
|
+
"A plain keyword matches a record when either provenance matches it. " \
|
|
26
|
+
"A keyword of the form <file keyword>:<code keyword> matches a record when its file provenance " \
|
|
27
|
+
"matches the file keyword and its code provenance matches the code keyword. " \
|
|
28
|
+
"The keyword * matches a record that has any file or code provenance. " \
|
|
29
|
+
"At least one keyword is required."
|
|
23
30
|
|
|
24
31
|
argument :provenance_types, label: "Provenance Types", type: :codelist
|
|
25
32
|
category "Filter Single Stream"
|
|
@@ -40,6 +47,8 @@ module ConceptQL
|
|
|
40
47
|
private
|
|
41
48
|
|
|
42
49
|
def additional_validation(db, _opts = {})
|
|
50
|
+
return add_error('no provenance keywords; use "*" for any provenance') if arguments.empty?
|
|
51
|
+
|
|
43
52
|
# With no lexicon there are no provenance concepts to check against,
|
|
44
53
|
# and every keyword would be reported as unrecognized. Vocabulary
|
|
45
54
|
# operators skip their unknown-code check in the same situation.
|
|
@@ -9,7 +9,11 @@ module ConceptQL
|
|
|
9
9
|
class ProviderFilter < Operator
|
|
10
10
|
register __FILE__
|
|
11
11
|
|
|
12
|
-
desc "Passes along records
|
|
12
|
+
desc "Passes along the records whose provider matches the given specialties and roles. " \
|
|
13
|
+
"Each option is a comma-separated list of concept IDs, and * matches any value. " \
|
|
14
|
+
"A record matches the specialties when its context specialty or its practitioner specialty " \
|
|
15
|
+
"is in the list. " \
|
|
16
|
+
"A record can have more than one specialty, so the operator can pass along the same record more than once."
|
|
13
17
|
option :specialties, type: :string
|
|
14
18
|
option :roles, type: :string
|
|
15
19
|
category "Filter Single Stream"
|
|
@@ -29,20 +29,12 @@ module ConceptQL
|
|
|
29
29
|
@race_descendents ||= YAML.load_file(ConceptQL.race_file)
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
-
def query_cols
|
|
33
|
-
table_columns(table)
|
|
34
|
-
end
|
|
35
|
-
|
|
36
32
|
def table
|
|
37
33
|
source_table
|
|
38
34
|
end
|
|
39
35
|
|
|
40
36
|
def source_table
|
|
41
|
-
|
|
42
|
-
:patients
|
|
43
|
-
else
|
|
44
|
-
:person
|
|
45
|
-
end
|
|
37
|
+
:patients
|
|
46
38
|
end
|
|
47
39
|
|
|
48
40
|
def query(db)
|
|
@@ -52,7 +44,7 @@ module ConceptQL
|
|
|
52
44
|
|
|
53
45
|
c_ids = (concept_ids + actual_ids).map { |i| [ i, race_descendents[i] ] }.flatten.compact.uniq.sort
|
|
54
46
|
|
|
55
|
-
c_ids = dm.related_concept_ids(db, c_ids)
|
|
47
|
+
c_ids = dm.related_concept_ids(db, c_ids)
|
|
56
48
|
|
|
57
49
|
q = db.from(source_table)
|
|
58
50
|
if words.any? { |w| w.match(/unknown/i) }
|