conceptql 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +141 -0
- data/docs/spec/README.md +402 -337
- data/docs/spec/examples/date-range-operator.json +1 -1
- data/docs/spec/examples/death-operator.json +1 -1
- data/docs/spec/examples/nth-occurrence-operator.json +1 -1
- data/docs/spec/examples/one-in-two-out-operator.json +1 -1
- data/docs/spec/examples/person-filter-operator.json +2 -2
- data/docs/spec/examples/vocabulary-operator.json +3 -3
- data/lib/conceptql/behaviors/metadatable.rb +76 -62
- data/lib/conceptql/behaviors/utilizable.rb +1 -1
- data/lib/conceptql/database.rb +16 -2
- data/lib/conceptql/diagram.rb +2 -2
- data/lib/conceptql/lexicon/lexicon_no_db.rb +4 -4
- data/lib/conceptql/lexicon.rb +7 -4
- data/lib/conceptql/operators/after.rb +5 -4
- data/lib/conceptql/operators/any_overlap.rb +4 -3
- data/lib/conceptql/operators/before.rb +5 -4
- data/lib/conceptql/operators/binary_operator_operator.rb +11 -13
- data/lib/conceptql/operators/casting_operator.rb +1 -1
- data/lib/conceptql/operators/co_reported.rb +9 -7
- data/lib/conceptql/operators/contains.rb +4 -3
- data/lib/conceptql/operators/count.rb +3 -3
- data/lib/conceptql/operators/date_range.rb +9 -8
- data/lib/conceptql/operators/death.rb +4 -4
- data/lib/conceptql/operators/dedup.rb +2 -2
- data/lib/conceptql/operators/during.rb +4 -3
- data/lib/conceptql/operators/episode.rb +9 -9
- data/lib/conceptql/operators/equal.rb +3 -2
- data/lib/conceptql/operators/ethnicity.rb +7 -3
- data/lib/conceptql/operators/except.rb +4 -3
- data/lib/conceptql/operators/filter.rb +4 -3
- data/lib/conceptql/operators/first.rb +4 -3
- data/lib/conceptql/operators/from.rb +2 -1
- data/lib/conceptql/operators/gender.rb +4 -3
- data/lib/conceptql/operators/hospice.rb +4 -3
- data/lib/conceptql/operators/hospitalization.rb +4 -3
- data/lib/conceptql/operators/information_periods.rb +5 -4
- data/lib/conceptql/operators/last.rb +4 -3
- data/lib/conceptql/operators/match.rb +4 -3
- data/lib/conceptql/operators/multiple_vocabularies.rb +4 -3
- data/lib/conceptql/operators/numeric.rb +4 -4
- data/lib/conceptql/operators/numeric_filter.rb +6 -6
- data/lib/conceptql/operators/occurrence.rb +25 -18
- data/lib/conceptql/operators/one_in_two_out.rb +16 -16
- data/lib/conceptql/operators/operator.rb +16 -3
- data/lib/conceptql/operators/pass_thru.rb +1 -1
- data/lib/conceptql/operators/person.rb +3 -3
- data/lib/conceptql/operators/person_filter.rb +6 -5
- data/lib/conceptql/operators/place_of_service_filter.rb +7 -7
- data/lib/conceptql/operators/provenance.rb +9 -9
- data/lib/conceptql/operators/provider_filter.rb +6 -6
- data/lib/conceptql/operators/race.rb +4 -3
- data/lib/conceptql/operators/recall.rb +6 -5
- data/lib/conceptql/operators/snf.rb +4 -3
- data/lib/conceptql/operators/sum.rb +6 -5
- data/lib/conceptql/operators/time_window.rb +6 -6
- data/lib/conceptql/operators/trim_date_end.rb +6 -5
- data/lib/conceptql/operators/trim_date_start.rb +6 -5
- data/lib/conceptql/operators/union.rb +8 -6
- data/lib/conceptql/operators/vocabulary.rb +12 -8
- data/lib/conceptql/query.rb +41 -12
- data/lib/conceptql/scope.rb +10 -2
- data/lib/conceptql/version.rb +1 -1
- data/lib/conceptql/visitors/metadata_extractor.rb +3 -1
- data/lib/conceptql/vocabularies/drg_systems.rb +16 -14
- data/lib/conceptql/vocabularies/read_codes.rb +1 -1
- data/lib/conceptql/window.rb +23 -2
- data/schemas/gdm.yml +0 -66
- data/schemas/gdm_wide.yml +0 -66
- metadata +1 -1
|
@@ -9,14 +9,14 @@ module ConceptQL
|
|
|
9
9
|
include ConceptQL::Behaviors::Timeless
|
|
10
10
|
|
|
11
11
|
register __FILE__
|
|
12
|
+
effects :select, :cast
|
|
12
13
|
|
|
13
|
-
desc "Selects every patient. With one upstream operator, it selects the patients who have a
|
|
14
|
-
returns "One
|
|
14
|
+
desc "Selects every patient. With one upstream operator, it selects the patients who have a result in the upstream stream."
|
|
15
|
+
returns "One result for each selected patient, from the patients table. The type of the results is person, " \
|
|
15
16
|
"and their start_date and end_date are both the patient's birth date."
|
|
16
17
|
example "All patients", [ "person" ]
|
|
17
18
|
example "The patients with an old myocardial infarction (ICD-9-CM 412)", [ "person", [ "icd9cm", "412" ] ]
|
|
18
19
|
|
|
19
|
-
allows_one_upstream
|
|
20
20
|
domains :person
|
|
21
21
|
|
|
22
22
|
def my_domain
|
|
@@ -6,18 +6,19 @@ module ConceptQL
|
|
|
6
6
|
module Operators
|
|
7
7
|
class PersonFilter < BinaryOperatorOperator
|
|
8
8
|
register __FILE__
|
|
9
|
+
effects :keep
|
|
9
10
|
|
|
10
|
-
desc "Keeps each left-hand
|
|
11
|
-
returns "The left-hand
|
|
12
|
-
option_meaning :right, "The right-hand stream. The operator keeps the left-hand
|
|
11
|
+
desc "Keeps each left-hand result whose patient has at least one result in the right-hand stream. The type and the dates of the right-hand results do not matter."
|
|
12
|
+
returns "The left-hand results of every type whose patient has at least one right-hand result. The right-hand results are not returned."
|
|
13
|
+
option_meaning :right, "The right-hand stream. The operator keeps the left-hand results of each patient who has a result in this stream, and does not pass these results on."
|
|
13
14
|
|
|
14
15
|
example "All MI Conditions for people who are male",
|
|
15
16
|
[ "person_filter", { left: [ "icd9", "412" ], right: [ "gender", "Male" ] } ]
|
|
16
|
-
example "All MI Conditions for people who had an office visit at some point in the
|
|
17
|
+
example "All MI Conditions for people who had an office visit at some point in the dataset",
|
|
17
18
|
[ "person_filter", { left: [ "icd9", "412" ], right: [ "cpt", "99214" ] } ]
|
|
18
19
|
example "The same result as the example above, with the office visits converted to patients first",
|
|
19
20
|
[ "person_filter", { left: [ "icd9", "412" ], right: [ "person", [ "cpt", "99214" ] ] } ]
|
|
20
|
-
example "All MI Conditions for people who are Male OR had an office visit at some point in the
|
|
21
|
+
example "All MI Conditions for people who are Male OR had an office visit at some point in the dataset",
|
|
21
22
|
[ "person_filter", { left: [ "icd9", "412" ], right: [ "union", [ "cpt", "99214" ], [ "gender", "Male" ] ] } ]
|
|
22
23
|
example "Yields two types: all MI Conditions for people who are Male, and all office visit Procedures for people who are Male",
|
|
23
24
|
[ "person_filter", { left: [ "union", [ "icd9", "412" ], [ "cpt", "99214" ] ], right: [ "gender", "Male" ] } ]
|
|
@@ -26,27 +26,27 @@ module ConceptQL
|
|
|
26
26
|
# checked.
|
|
27
27
|
class PlaceOfServiceFilter < Operator
|
|
28
28
|
register __FILE__
|
|
29
|
+
effects :keep
|
|
29
30
|
|
|
30
31
|
VOCABULARY_ID = "CMS Place of Service"
|
|
31
32
|
ANY = "*"
|
|
32
33
|
|
|
33
|
-
desc "Passes along the
|
|
34
|
-
"A
|
|
34
|
+
desc "Passes along the results whose place of service is one of the given CMS Place of Service codes. " \
|
|
35
|
+
"A result that has no place of service is not passed along."
|
|
35
36
|
|
|
36
|
-
returns "The upstream
|
|
37
|
+
returns "The upstream results whose place of service is one of the given codes."
|
|
37
38
|
|
|
38
39
|
argument :places_of_service, type: :codelist, vocab: VOCABULARY_ID,
|
|
39
40
|
meaning: "One or more CMS Place of Service codes, such as 21 (inpatient hospital) or 22 (outpatient hospital). " \
|
|
40
|
-
"The code * matches a
|
|
41
|
-
"An unknown code gets a warning and matches no
|
|
41
|
+
"The code * matches a result whose place of service is any code of that vocabulary. " \
|
|
42
|
+
"An unknown code gets a warning and matches no result."
|
|
42
43
|
|
|
43
44
|
# The test data has no place of service, so the example returns no rows
|
|
44
45
|
example "Diabetes diagnoses (ICD-9 250.00) with the place of service outpatient hospital (22)",
|
|
45
46
|
[ "place_of_service_filter", 22, [ "icd9", "250.00" ] ], help: false
|
|
46
47
|
|
|
47
48
|
category "Filter Single Stream"
|
|
48
|
-
|
|
49
|
-
allows_one_upstream
|
|
49
|
+
diagram_base :temporal
|
|
50
50
|
validate_one_upstream
|
|
51
51
|
validate_at_least_one_argument
|
|
52
52
|
require_column :visit_source_concept_id
|
|
@@ -18,29 +18,29 @@ module ConceptQL
|
|
|
18
18
|
# provenance. At least one keyword is required.
|
|
19
19
|
class Provenance < Operator
|
|
20
20
|
register __FILE__
|
|
21
|
+
effects :keep
|
|
21
22
|
|
|
22
|
-
desc "Passes along the
|
|
23
|
-
"The provenance of a
|
|
23
|
+
desc "Passes along the results whose provenance matches at least one of the given keywords. " \
|
|
24
|
+
"The provenance of a result says which kind of file it came from, such as inpatient, " \
|
|
24
25
|
"and where its code was on the claim, such as primary."
|
|
25
26
|
|
|
26
|
-
returns "The upstream
|
|
27
|
+
returns "The upstream results whose provenance matches at least one of the keywords."
|
|
27
28
|
|
|
28
29
|
argument :provenance_types, label: "Provenance Types", type: :codelist,
|
|
29
30
|
meaning: "One or more provenance keywords, in lower case. " \
|
|
30
|
-
"A file keyword, such as inpatient, outpatient or carrier_claim, says which kind of file a
|
|
31
|
+
"A file keyword, such as inpatient, outpatient or carrier_claim, says which kind of file a result came from. " \
|
|
31
32
|
"A code keyword, such as primary or first, says where the code was on the claim. " \
|
|
32
|
-
"A plain keyword matches a
|
|
33
|
+
"A plain keyword matches a result when its file provenance or its code provenance matches it. " \
|
|
33
34
|
"A keyword also matches the provenances under it, so inpatient includes snf. " \
|
|
34
35
|
"A keyword of the form file:code, such as inpatient:primary, needs both the file and the code provenance to match. " \
|
|
35
|
-
"The keyword * matches a
|
|
36
|
-
"A statement with no keyword is invalid. An unknown keyword gets a warning and matches no
|
|
36
|
+
"The keyword * matches a result that has any provenance. " \
|
|
37
|
+
"A statement with no keyword is invalid. An unknown keyword gets a warning and matches no result."
|
|
37
38
|
|
|
38
39
|
example "Diabetes diagnoses (ICD-9 250.00) from inpatient files",
|
|
39
40
|
[ "provenance", "inpatient", [ "icd9", "250.00" ] ]
|
|
40
41
|
|
|
41
42
|
category "Filter Single Stream"
|
|
42
|
-
|
|
43
|
-
allows_one_upstream
|
|
43
|
+
diagram_base :temporal
|
|
44
44
|
validate_one_upstream
|
|
45
45
|
default_query_columns
|
|
46
46
|
|
|
@@ -8,13 +8,14 @@ module ConceptQL
|
|
|
8
8
|
# providers matching the given criteria.
|
|
9
9
|
class ProviderFilter < Operator
|
|
10
10
|
register __FILE__
|
|
11
|
+
effects :keep, :change
|
|
11
12
|
|
|
12
|
-
desc "Passes along the
|
|
13
|
+
desc "Passes along the results whose provider matches the given specialties and roles. " \
|
|
13
14
|
"Each option is a comma-separated list of concept IDs, and * matches any value."
|
|
14
15
|
|
|
15
|
-
returns "The upstream
|
|
16
|
-
"A
|
|
17
|
-
"Each
|
|
16
|
+
returns "The upstream results that have a provider who matches both the specialties and the roles. " \
|
|
17
|
+
"A result can have more than one provider, so a result comes out once for each provider who matches. " \
|
|
18
|
+
"Each result gets the columns provider_id, context_specialty_concept_id, practitioner_specialty_concept_id " \
|
|
18
19
|
"and role_type_concept_id."
|
|
19
20
|
|
|
20
21
|
option :specialties, type: :string,
|
|
@@ -29,8 +30,7 @@ module ConceptQL
|
|
|
29
30
|
[ "provider_filter", [ "icd9", "250.00" ], { "specialties" => "38003868", "roles" => "100000067" } ]
|
|
30
31
|
|
|
31
32
|
category "Filter Single Stream"
|
|
32
|
-
|
|
33
|
-
allows_one_upstream
|
|
33
|
+
diagram_base :temporal
|
|
34
34
|
validate_one_upstream
|
|
35
35
|
validate_required_options :specialties, :roles
|
|
36
36
|
require_column :provider_id
|
|
@@ -13,6 +13,7 @@ module ConceptQL
|
|
|
13
13
|
# you won't get any matches
|
|
14
14
|
class Race < Operator
|
|
15
15
|
register __FILE__
|
|
16
|
+
effects :select
|
|
16
17
|
|
|
17
18
|
include ConceptQL::Behaviors::Windowable
|
|
18
19
|
include ConceptQL::Behaviors::Timeless
|
|
@@ -24,12 +25,12 @@ module ConceptQL
|
|
|
24
25
|
"The operator also selects the more detailed races under each given race, " \
|
|
25
26
|
"and each concept that is a kind of one of the race concepts through an is-a relationship. " \
|
|
26
27
|
"A value that contains Unknown also selects the patients who have no race or race concept ID 0."
|
|
27
|
-
returns "One person
|
|
28
|
-
"The start_date and the end_date of each
|
|
28
|
+
returns "One person result for each patient whose race is one of the given races. " \
|
|
29
|
+
"The start_date and the end_date of each result are the patient's birth date."
|
|
29
30
|
example "All patients recorded as white", [ "race", "White" ]
|
|
30
31
|
domains :person
|
|
31
32
|
category "Select by Property"
|
|
32
|
-
|
|
33
|
+
diagram_base :selection
|
|
33
34
|
validate_no_upstreams
|
|
34
35
|
validate_at_least_one_argument
|
|
35
36
|
|
|
@@ -13,10 +13,11 @@ module ConceptQL
|
|
|
13
13
|
# "define" operator and will fetch the records cached in the corresponding table
|
|
14
14
|
class Recall < Operator
|
|
15
15
|
register __FILE__
|
|
16
|
+
effects :refer
|
|
16
17
|
|
|
17
|
-
desc "Passes on the
|
|
18
|
-
"Use it to break a complex statement into parts, or to use a stream of
|
|
19
|
-
returns "The
|
|
18
|
+
desc "Passes on the results of the operator in the statement that has the given label. " \
|
|
19
|
+
"Use it to break a complex statement into parts, or to use a stream of results more than once."
|
|
20
|
+
returns "The results of the operator that has the given label."
|
|
20
21
|
example 'Label neurological diagnoses as "neuro dxs". Recall them to remove those that were co-reported ' \
|
|
21
22
|
"with a nervous-system MS-DRG. Then apply the one inpatient, two outpatient rule. " \
|
|
22
23
|
"(The code lists are shortened.)",
|
|
@@ -46,11 +47,11 @@ module ConceptQL
|
|
|
46
47
|
]
|
|
47
48
|
|
|
48
49
|
argument :name, type: :string,
|
|
49
|
-
meaning: "The label of the operator whose
|
|
50
|
+
meaning: "The label of the operator whose results to pass on. It must match the label exactly, " \
|
|
50
51
|
"including case. The labeled operator can come before or after the recall in the " \
|
|
51
52
|
"statement, but labeled streams that recall each other in a loop are invalid."
|
|
52
53
|
category "Get Related Data"
|
|
53
|
-
|
|
54
|
+
diagram_base :selection
|
|
54
55
|
validate_no_upstreams
|
|
55
56
|
validate_one_argument
|
|
56
57
|
default_query_columns
|
|
@@ -6,13 +6,14 @@ module ConceptQL
|
|
|
6
6
|
module Operators
|
|
7
7
|
class Snf < Operator
|
|
8
8
|
register __FILE__
|
|
9
|
+
effects :select
|
|
9
10
|
include ConceptQL::Behaviors::Utilizable
|
|
10
11
|
|
|
11
|
-
desc "Selects admissions to skilled nursing facilities (SNF): one
|
|
12
|
-
returns "One
|
|
12
|
+
desc "Selects admissions to skilled nursing facilities (SNF): one result for each admission whose claim came from an SNF file."
|
|
13
|
+
returns "One result for each SNF admission. The type of the results is condition_occurrence, " \
|
|
13
14
|
"and their criterion_table is collections, the table of claims. Their start_date and end_date " \
|
|
14
15
|
"are the dates of the claim, and source_value holds the primary code of the admission when it has one. " \
|
|
15
|
-
"Each
|
|
16
|
+
"Each result also has admission and discharge columns, such as admission_date, discharge_date and length_of_stay."
|
|
16
17
|
# The test data has no SNF claims
|
|
17
18
|
example "All SNF admissions", [ "snf" ], help: false
|
|
18
19
|
|
|
@@ -6,6 +6,7 @@ module ConceptQL
|
|
|
6
6
|
module Operators
|
|
7
7
|
class Sum < PassThru
|
|
8
8
|
register __FILE__
|
|
9
|
+
effects :aggregate
|
|
9
10
|
# "We do not yet support numeric or sum in any meaningful way"
|
|
10
11
|
# (Ryan, 2026-11-03, seed conceptql-mdo.4)
|
|
11
12
|
hidden
|
|
@@ -26,13 +27,13 @@ module ConceptQL
|
|
|
26
27
|
vocabularies.length > 1
|
|
27
28
|
end
|
|
28
29
|
|
|
29
|
-
desc "Groups the
|
|
30
|
-
"end_date and value_as_number. For each group, the operator passes along one
|
|
30
|
+
desc "Groups the results that have the same values in all columns except criterion_id, start_date, " \
|
|
31
|
+
"end_date and value_as_number. For each group, the operator passes along one result " \
|
|
31
32
|
"with the sum of value_as_number, the earliest start_date, the most recent end_date " \
|
|
32
|
-
"and a criterion_id of 0. With group_by_person, the operator groups all of the
|
|
33
|
-
"of each person and passes along one person
|
|
33
|
+
"and a criterion_id of 0. With group_by_person, the operator groups all of the results " \
|
|
34
|
+
"of each person and passes along one person result for each person."
|
|
34
35
|
option :group_by_person, type: :boolean,
|
|
35
|
-
instructions: "Choose whether to sum all of the
|
|
36
|
+
instructions: "Choose whether to sum all of the results of each person into one person result, for example to count the results of each person"
|
|
36
37
|
default_query_columns
|
|
37
38
|
validate_at_least_one_upstream
|
|
38
39
|
validate_no_arguments
|
|
@@ -24,6 +24,7 @@ module ConceptQL
|
|
|
24
24
|
# start: 'd', end: '' # Only adjust start_date by positive 1 day and leave end_date uneffected
|
|
25
25
|
class TimeWindow < PassThru
|
|
26
26
|
register __FILE__
|
|
27
|
+
effects :change
|
|
27
28
|
|
|
28
29
|
# The values that the start and end options take (DateAdjuster)
|
|
29
30
|
DATE_ADJUSTMENT_TEXT = "The value is an amount of time, made of one or more numbers, each with a unit: " \
|
|
@@ -31,17 +32,17 @@ module ConceptQL
|
|
|
31
32
|
"a bare number means days, and a unit without a number means 1 of it. " \
|
|
32
33
|
"For example, -1m is 1 month back, and 10d3m is 3 months and 10 days. " \
|
|
33
34
|
"An empty value or 0 leaves the date unchanged. The value start or end uses the " \
|
|
34
|
-
"
|
|
35
|
+
"result's start_date or end_date, and a date in YYYY-MM-DD format sets the date to that date."
|
|
35
36
|
|
|
36
|
-
desc "Moves the start_date and the end_date of each
|
|
37
|
-
"Use it to shift or widen a
|
|
37
|
+
desc "Moves the start_date and the end_date of each result. " \
|
|
38
|
+
"Use it to shift or widen a result's dates before you compare them with other results."
|
|
38
39
|
option :start, type: :string,
|
|
39
40
|
instructions: 'Enter a number followed by "d", "w", "m" or "y" for days, weeks, months or years, such as 30d. A negative number moves the date earlier. You can also enter start, end or a date in YYYY-MM-DD format.',
|
|
40
41
|
meaning: "How to change the start_date. If you omit it, the start_date does not change. #{DATE_ADJUSTMENT_TEXT}"
|
|
41
42
|
option :end, type: :string,
|
|
42
43
|
instructions: 'Enter a number followed by "d", "w", "m" or "y" for days, weeks, months or years, such as 30d. A negative number moves the date earlier. You can also enter start, end or a date in YYYY-MM-DD format.',
|
|
43
44
|
meaning: "How to change the end_date. If you omit it, the end_date does not change. #{DATE_ADJUSTMENT_TEXT}"
|
|
44
|
-
returns "One
|
|
45
|
+
returns "One result for each upstream result, with the start_date and the end_date that the options give. " \
|
|
45
46
|
"The other columns do not change."
|
|
46
47
|
example "All Diagnoses of Diabetes (ICD-9 250.01) within 30 days of an MI",
|
|
47
48
|
[ "during", { "left" => [ "icd9", "250.01" ], "right" => [ "time_window", [ "icd9", "412" ], { "start" => "-30d", "end" => "30d" } ] } ]
|
|
@@ -52,12 +53,11 @@ module ConceptQL
|
|
|
52
53
|
[ "time_window", [ "icd9", "412" ], { "start" => "", "end" => "start" } ]
|
|
53
54
|
example "Nonsensical, but allowed: swap the start_date and end_date",
|
|
54
55
|
[ "time_window", [ "icd9", "412" ], { "start" => "end", "end" => "start" } ]
|
|
55
|
-
allows_one_upstream
|
|
56
56
|
validate_one_upstream
|
|
57
57
|
validate_no_arguments
|
|
58
58
|
validate_option DateAdjuster::VALID_INPUT, :start, :end
|
|
59
59
|
category "Modify Data"
|
|
60
|
-
|
|
60
|
+
diagram_base :temporal
|
|
61
61
|
|
|
62
62
|
def query(db)
|
|
63
63
|
dm.selectify(db.from(stream.evaluate(db)).from_self,
|
|
@@ -16,12 +16,13 @@ module ConceptQL
|
|
|
16
16
|
# record is passed thru unaffected.
|
|
17
17
|
class TrimDateEnd < TrimDate
|
|
18
18
|
register __FILE__
|
|
19
|
+
effects :keep, :change
|
|
19
20
|
|
|
20
|
-
desc "For each patient, moves the end_date of each left-hand
|
|
21
|
-
returns "The left-hand
|
|
22
|
-
option_meaning :left, "The left-hand stream. The operator moves the end_date of these
|
|
23
|
-
option_meaning :right, "The right-hand stream. For each patient, the earliest start_date of these
|
|
24
|
-
option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is subtracted from the start_date of the right-hand
|
|
21
|
+
desc "For each patient, moves the end_date of each left-hand result back to the start_date of the right-hand result with the earliest start_date. A left-hand result that starts after that start_date is removed, and the left-hand results of a patient without right-hand results pass through unchanged."
|
|
22
|
+
returns "The left-hand results, each with its end_date set to the earlier of its own end_date and the start_date of the patient's right-hand result. Left-hand results that start after that start_date are not returned. The right-hand results are not returned."
|
|
23
|
+
option_meaning :left, "The left-hand stream. The operator moves the end_date of these results and passes them on."
|
|
24
|
+
option_meaning :right, "The right-hand stream. For each patient, the earliest start_date of these results sets the latest end_date of the left-hand results. These results are not passed on."
|
|
25
|
+
option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is subtracted from the start_date of the right-hand result before the comparison. The adjusted date is the one that moves the end_date of the left-hand result."
|
|
25
26
|
|
|
26
27
|
example "Each patient's span from birth to age 50 or to 1970-01-01, whichever is earlier (patients born after 1970 are dropped)",
|
|
27
28
|
[ "trim_date_end", { left: [ "time_window", [ "person" ], { end: "+50y" } ],
|
|
@@ -16,12 +16,13 @@ module ConceptQL
|
|
|
16
16
|
# record is passed thru unaffected.
|
|
17
17
|
class TrimDateStart < TrimDate
|
|
18
18
|
register __FILE__
|
|
19
|
+
effects :keep, :change
|
|
19
20
|
|
|
20
|
-
desc "For each patient, moves the start_date of each left-hand
|
|
21
|
-
returns "The left-hand
|
|
22
|
-
option_meaning :left, "The left-hand stream. The operator moves the start_date of these
|
|
23
|
-
option_meaning :right, "The right-hand stream. For each patient, the end_date of the
|
|
24
|
-
option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is added to the end_date of the right-hand
|
|
21
|
+
desc "For each patient, moves the start_date of each left-hand result forward to the end_date of the right-hand result with the latest start_date. A left-hand result that ends before that end_date is removed, and the left-hand results of a patient without right-hand results pass through unchanged."
|
|
22
|
+
returns "The left-hand results, each with its start_date set to the later of its own start_date and the end_date of the patient's right-hand result. Left-hand results that end before that end_date are not returned. The right-hand results are not returned."
|
|
23
|
+
option_meaning :left, "The left-hand stream. The operator moves the start_date of these results and passes them on."
|
|
24
|
+
option_meaning :right, "The right-hand stream. For each patient, the end_date of the result with the latest start_date sets the earliest start_date of the left-hand results. These results are not passed on."
|
|
25
|
+
option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is added to the end_date of the right-hand result before the comparison. The adjusted date is the one that moves the start_date of the left-hand result."
|
|
25
26
|
|
|
26
27
|
example "Each patient's span from birth to age 50, trimmed to start no earlier than 1980-01-01 (patients who were 50 before 1980 are dropped)",
|
|
27
28
|
[ "trim_date_start", { left: [ "time_window", [ "person" ], { end: "+50y" } ],
|
|
@@ -6,6 +6,7 @@ module ConceptQL
|
|
|
6
6
|
module Operators
|
|
7
7
|
class Union < PassThru
|
|
8
8
|
register __FILE__
|
|
9
|
+
effects :combine
|
|
9
10
|
|
|
10
11
|
class Combiner
|
|
11
12
|
attr_reader :combinables
|
|
@@ -54,8 +55,10 @@ module ConceptQL
|
|
|
54
55
|
end
|
|
55
56
|
end
|
|
56
57
|
|
|
58
|
+
# Each upstream passes along its own records, so with two or more
|
|
59
|
+
# upstreams one person can get one row from each (bead conceptql-8q8h).
|
|
57
60
|
def events_per_patient
|
|
58
|
-
upstreams.
|
|
61
|
+
upstreams.length > 1 ? :multiple : super
|
|
59
62
|
end
|
|
60
63
|
|
|
61
64
|
def multiple_vocabularies
|
|
@@ -64,11 +67,11 @@ module ConceptQL
|
|
|
64
67
|
vocabularies.length > 1
|
|
65
68
|
end
|
|
66
69
|
|
|
67
|
-
desc "Combines the
|
|
68
|
-
"Each
|
|
70
|
+
desc "Combines the results of all its upstream operators into one stream. " \
|
|
71
|
+
"Each result keeps its own type, so one stream can hold several types."
|
|
69
72
|
|
|
70
|
-
returns "Each
|
|
71
|
-
"Two
|
|
73
|
+
returns "Each result of each upstream. A result that comes from more than one upstream comes out once. " \
|
|
74
|
+
"Two results are the same only when every column is equal, including label."
|
|
72
75
|
|
|
73
76
|
example "Two streams of the same type (condition_occurrence) combined into one stream",
|
|
74
77
|
[ "union", [ "icd9", "412" ], [ "icd9", "250.01" ] ]
|
|
@@ -77,7 +80,6 @@ module ConceptQL
|
|
|
77
80
|
example "Two condition_occurrence streams and a procedure_occurrence stream combined by a single union",
|
|
78
81
|
[ "union", [ "icd9", "412" ], [ "icd9", "250.01" ], [ "cpt", "99214" ] ]
|
|
79
82
|
|
|
80
|
-
allows_many_upstreams
|
|
81
83
|
category "Combine Streams"
|
|
82
84
|
default_query_columns
|
|
83
85
|
validate_at_least_one_upstream
|
|
@@ -29,11 +29,15 @@ module ConceptQL
|
|
|
29
29
|
include ConceptQL::Behaviors::CodeLister
|
|
30
30
|
|
|
31
31
|
category "Select by Clinical Codes"
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
diagram_base :selection
|
|
33
|
+
effects :select
|
|
34
|
+
desc "Selects the results whose code is one of the given codes in a vocabulary, such as ICD-9-CM diagnoses or CPT procedures. " \
|
|
34
35
|
"Each vocabulary has its own operator, and a few operators search several vocabularies at once."
|
|
35
36
|
validate_no_upstreams
|
|
36
37
|
validate_at_least_one_argument
|
|
38
|
+
# MultipleVocabularies gets this error from its vocabulary operators
|
|
39
|
+
# (#aggregate_messages), so it does not declare the validation itself.
|
|
40
|
+
validate_string_codes
|
|
37
41
|
validate_codes_match
|
|
38
42
|
|
|
39
43
|
# VocabularyClassFactory gives each vocabulary operator a copy of this
|
|
@@ -41,18 +45,18 @@ module ConceptQL
|
|
|
41
45
|
argument :codes, type: :codelist,
|
|
42
46
|
meaning: "One or more codes of the vocabulary. Write each code as the vocabulary writes it, " \
|
|
43
47
|
"including any dots: 250.01, not 25001. A code matches exactly, including case, " \
|
|
44
|
-
"and it does not select its child codes. A code alone of * selects every
|
|
48
|
+
"and it does not select its child codes. A code alone of * selects every result " \
|
|
45
49
|
"of the vocabulary, and there is no other wildcard. A code that does not have the " \
|
|
46
50
|
"vocabulary's format matches nothing and gives a warning."
|
|
47
51
|
|
|
48
|
-
returns "Every
|
|
49
|
-
"The type of the
|
|
52
|
+
returns "Every result whose code is one of the given codes in the operator's vocabulary. " \
|
|
53
|
+
"The type of the results depends on the vocabulary, for example condition_occurrence for icd9cm " \
|
|
50
54
|
"and procedure_occurrence for cpt4. An operator does not follow mappings to other vocabularies."
|
|
51
|
-
example "All
|
|
52
|
-
example "All MS-DRG 205
|
|
55
|
+
example "All results with the ICD-9-CM diagnosis code 412 (old myocardial infarction)", [ "icd9cm", "412" ]
|
|
56
|
+
example "All MS-DRG 205 results: other respiratory system diagnoses with a major complication or comorbidity",
|
|
53
57
|
[ "ms_drg", "205" ]
|
|
54
58
|
# The test data has no DRG record before 2007-10-01
|
|
55
|
-
example "All CMS-DRG 205
|
|
59
|
+
example "All CMS-DRG 205 results (a liver disorder)", [ "cms_drg", "205" ], help: false
|
|
56
60
|
|
|
57
61
|
def source_table
|
|
58
62
|
domain_map(op_name)
|
data/lib/conceptql/query.rb
CHANGED
|
@@ -67,6 +67,12 @@ module ConceptQL
|
|
|
67
67
|
@cdb = cdb
|
|
68
68
|
@given_statement = statement
|
|
69
69
|
@statement, opts = extract_statement(statement, opts)
|
|
70
|
+
# Window::Table queries a window table statement with this Database.
|
|
71
|
+
# Scope#windows merges the scope options with the window options, so a
|
|
72
|
+
# window table statement gets it from both places that a window table
|
|
73
|
+
# comes from: a window statement and the scope options of the calling
|
|
74
|
+
# program.
|
|
75
|
+
opts[:scope_opts] = (opts[:scope_opts] || {}).merge(cdb: cdb)
|
|
70
76
|
@opts = opts
|
|
71
77
|
opts[:algorithm_fetcher] ||= proc do |alg|
|
|
72
78
|
statement, description = db[:concepts].where(concept_id: alg).get(%i[statement label])
|
|
@@ -247,7 +253,7 @@ module ConceptQL
|
|
|
247
253
|
# around it. It annotates this Query's own operator tree, so only a fresh
|
|
248
254
|
# Query calls it.
|
|
249
255
|
def annotate_without_ids(opts)
|
|
250
|
-
opts = opts.merge(skip_counts: true) unless window_table_errors.empty?
|
|
256
|
+
opts = opts.merge(skip_counts: true) unless window_table_errors.empty? && window_option_errors.empty?
|
|
251
257
|
nodifier.scope.one_query_counts = OneQueryCounts.new(operator, db, opts)
|
|
252
258
|
with_window(operator.annotate(db, opts), opts)
|
|
253
259
|
end
|
|
@@ -283,11 +289,23 @@ module ConceptQL
|
|
|
283
289
|
@valid = true
|
|
284
290
|
end
|
|
285
291
|
|
|
292
|
+
# The window options of the scope, as Scope#windows reads them: the
|
|
293
|
+
# scope options of the calling program, merged with the options of the
|
|
294
|
+
# statement's window
|
|
295
|
+
def window_opts
|
|
296
|
+
nodifier.scope.window_opts
|
|
297
|
+
end
|
|
298
|
+
|
|
286
299
|
# The window table of the scope, as Window::Table reads it: a table name
|
|
287
300
|
# or a ConceptQL statement (an Array)
|
|
288
301
|
def window_table
|
|
289
|
-
|
|
290
|
-
|
|
302
|
+
window_opts[:window_table]
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# The errors of the window options (Window.option_errors), as
|
|
306
|
+
# [message, *details]. The window operator of the annotation has them.
|
|
307
|
+
def window_option_errors
|
|
308
|
+
Window.option_errors(window_opts)
|
|
291
309
|
end
|
|
292
310
|
|
|
293
311
|
# The validation errors of the window table's statement, when the window
|
|
@@ -302,18 +320,28 @@ module ConceptQL
|
|
|
302
320
|
end
|
|
303
321
|
end
|
|
304
322
|
|
|
323
|
+
# The options of the statement's window, as the scope runs them: for a
|
|
324
|
+
# window inside a window, the options of the inner window. Nil when the
|
|
325
|
+
# statement is not a window statement.
|
|
326
|
+
def statement_window_options
|
|
327
|
+
nodifier.scope.opts[:window_opts] if @window_statement
|
|
328
|
+
end
|
|
329
|
+
|
|
305
330
|
# The annotated statement inside its window operator, when the statement
|
|
306
|
-
# is a window statement
|
|
307
|
-
# window operator has the options of the
|
|
308
|
-
#
|
|
309
|
-
#
|
|
331
|
+
# is a window statement, the scope's window table is a statement, or the
|
|
332
|
+
# window options have errors. The window operator has the options of the
|
|
333
|
+
# statement's window (statement_window_options) and an annotation hash,
|
|
334
|
+
# which holds its id and the errors of the window options. A window table
|
|
335
|
+
# statement is annotated in place, with the same opts.
|
|
310
336
|
def with_window(annotated, opts)
|
|
311
337
|
table = window_table
|
|
312
|
-
|
|
338
|
+
errors = window_option_errors
|
|
339
|
+
window_options = statement_window_options
|
|
340
|
+
return annotated unless window_options || table.is_a?(Array) || errors.any?
|
|
313
341
|
|
|
314
|
-
options = (
|
|
342
|
+
options = (window_options || {}).dup
|
|
315
343
|
options[:window_table] = cdb.query(table).annotate_without_ids(opts) if table.is_a?(Array)
|
|
316
|
-
[ "window", annotated, options.merge(annotation: {}) ]
|
|
344
|
+
[ "window", annotated, options.merge(annotation: errors.empty? ? {} : { errors: errors }) ]
|
|
317
345
|
end
|
|
318
346
|
|
|
319
347
|
def extract_statement(stmt, opts)
|
|
@@ -325,9 +353,10 @@ module ConceptQL
|
|
|
325
353
|
raise "window operator needs a ConceptQL statement followed by a hash as the last item of the array."
|
|
326
354
|
end
|
|
327
355
|
|
|
356
|
+
# The options of an inner window replace those of an outer window.
|
|
328
357
|
opts[:scope_opts] = (opts[:scope_opts] || {})
|
|
329
|
-
opts[:scope_opts].merge!(window_opts: stmt.last
|
|
330
|
-
@
|
|
358
|
+
opts[:scope_opts].merge!(window_opts: stmt.last)
|
|
359
|
+
@window_statement = true
|
|
331
360
|
extract_statement(stmt[1], opts)
|
|
332
361
|
elsif stmt.length == 1 && stmt.first.is_a?(Array)
|
|
333
362
|
extract_statement(stmt.first, opts)
|
data/lib/conceptql/scope.rb
CHANGED
|
@@ -100,7 +100,7 @@ module ConceptQL
|
|
|
100
100
|
@operator_cte_names = {}.compare_by_identity
|
|
101
101
|
@opts = opts.dup
|
|
102
102
|
@query_columns = DEFAULT_COLUMNS.keys
|
|
103
|
-
@query_columns << :window_id if
|
|
103
|
+
@query_columns << :window_id if window_opts[:window_table]
|
|
104
104
|
|
|
105
105
|
@i = 0
|
|
106
106
|
@mutex = Mutex.new
|
|
@@ -634,7 +634,15 @@ module ConceptQL
|
|
|
634
634
|
end
|
|
635
635
|
|
|
636
636
|
def windows
|
|
637
|
-
@windows ||= Window.from(
|
|
637
|
+
@windows ||= Window.from(window_opts)
|
|
638
|
+
end
|
|
639
|
+
|
|
640
|
+
# The window options that run: the scope options, merged with the options
|
|
641
|
+
# of the statement's window (window_opts), which win. A window_table at the
|
|
642
|
+
# top of the scope options is a window table, as one in window_opts is
|
|
643
|
+
# (bead conceptql-6t7q).
|
|
644
|
+
def window_opts
|
|
645
|
+
opts.merge(opts[:window_opts] || {})
|
|
638
646
|
end
|
|
639
647
|
|
|
640
648
|
def force_temp_tables?(options = {})
|
data/lib/conceptql/version.rb
CHANGED
|
@@ -27,13 +27,15 @@ module ConceptQL
|
|
|
27
27
|
@operator_names ||= operators.map(&:op_name)
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
+
# True when every operator except union takes no upstreams. t_shank
|
|
31
|
+
# saves this value in its performance statistics.
|
|
30
32
|
def codeset_only?
|
|
31
33
|
return @codeset_only if defined?(@codeset_only)
|
|
32
34
|
|
|
33
35
|
@codeset_only = operators.reject do |op|
|
|
34
36
|
# We skip Unions because they don't count against codesets
|
|
35
37
|
op.op_name == "union"
|
|
36
|
-
end.map(&:class).
|
|
38
|
+
end.map(&:class).all? { |klass| klass.upstreams_facet == "none" }
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
def max_depth
|