conceptql 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +141 -0
  3. data/docs/spec/README.md +402 -337
  4. data/docs/spec/examples/date-range-operator.json +1 -1
  5. data/docs/spec/examples/death-operator.json +1 -1
  6. data/docs/spec/examples/nth-occurrence-operator.json +1 -1
  7. data/docs/spec/examples/one-in-two-out-operator.json +1 -1
  8. data/docs/spec/examples/person-filter-operator.json +2 -2
  9. data/docs/spec/examples/vocabulary-operator.json +3 -3
  10. data/lib/conceptql/behaviors/metadatable.rb +76 -62
  11. data/lib/conceptql/behaviors/utilizable.rb +1 -1
  12. data/lib/conceptql/database.rb +16 -2
  13. data/lib/conceptql/diagram.rb +2 -2
  14. data/lib/conceptql/lexicon/lexicon_no_db.rb +4 -4
  15. data/lib/conceptql/lexicon.rb +7 -4
  16. data/lib/conceptql/operators/after.rb +5 -4
  17. data/lib/conceptql/operators/any_overlap.rb +4 -3
  18. data/lib/conceptql/operators/before.rb +5 -4
  19. data/lib/conceptql/operators/binary_operator_operator.rb +11 -13
  20. data/lib/conceptql/operators/casting_operator.rb +1 -1
  21. data/lib/conceptql/operators/co_reported.rb +9 -7
  22. data/lib/conceptql/operators/contains.rb +4 -3
  23. data/lib/conceptql/operators/count.rb +3 -3
  24. data/lib/conceptql/operators/date_range.rb +9 -8
  25. data/lib/conceptql/operators/death.rb +4 -4
  26. data/lib/conceptql/operators/dedup.rb +2 -2
  27. data/lib/conceptql/operators/during.rb +4 -3
  28. data/lib/conceptql/operators/episode.rb +9 -9
  29. data/lib/conceptql/operators/equal.rb +3 -2
  30. data/lib/conceptql/operators/ethnicity.rb +7 -3
  31. data/lib/conceptql/operators/except.rb +4 -3
  32. data/lib/conceptql/operators/filter.rb +4 -3
  33. data/lib/conceptql/operators/first.rb +4 -3
  34. data/lib/conceptql/operators/from.rb +2 -1
  35. data/lib/conceptql/operators/gender.rb +4 -3
  36. data/lib/conceptql/operators/hospice.rb +4 -3
  37. data/lib/conceptql/operators/hospitalization.rb +4 -3
  38. data/lib/conceptql/operators/information_periods.rb +5 -4
  39. data/lib/conceptql/operators/last.rb +4 -3
  40. data/lib/conceptql/operators/match.rb +4 -3
  41. data/lib/conceptql/operators/multiple_vocabularies.rb +4 -3
  42. data/lib/conceptql/operators/numeric.rb +4 -4
  43. data/lib/conceptql/operators/numeric_filter.rb +6 -6
  44. data/lib/conceptql/operators/occurrence.rb +25 -18
  45. data/lib/conceptql/operators/one_in_two_out.rb +16 -16
  46. data/lib/conceptql/operators/operator.rb +16 -3
  47. data/lib/conceptql/operators/pass_thru.rb +1 -1
  48. data/lib/conceptql/operators/person.rb +3 -3
  49. data/lib/conceptql/operators/person_filter.rb +6 -5
  50. data/lib/conceptql/operators/place_of_service_filter.rb +7 -7
  51. data/lib/conceptql/operators/provenance.rb +9 -9
  52. data/lib/conceptql/operators/provider_filter.rb +6 -6
  53. data/lib/conceptql/operators/race.rb +4 -3
  54. data/lib/conceptql/operators/recall.rb +6 -5
  55. data/lib/conceptql/operators/snf.rb +4 -3
  56. data/lib/conceptql/operators/sum.rb +6 -5
  57. data/lib/conceptql/operators/time_window.rb +6 -6
  58. data/lib/conceptql/operators/trim_date_end.rb +6 -5
  59. data/lib/conceptql/operators/trim_date_start.rb +6 -5
  60. data/lib/conceptql/operators/union.rb +8 -6
  61. data/lib/conceptql/operators/vocabulary.rb +12 -8
  62. data/lib/conceptql/query.rb +41 -12
  63. data/lib/conceptql/scope.rb +10 -2
  64. data/lib/conceptql/version.rb +1 -1
  65. data/lib/conceptql/visitors/metadata_extractor.rb +3 -1
  66. data/lib/conceptql/vocabularies/drg_systems.rb +16 -14
  67. data/lib/conceptql/vocabularies/read_codes.rb +1 -1
  68. data/lib/conceptql/window.rb +23 -2
  69. data/schemas/gdm.yml +0 -66
  70. data/schemas/gdm_wide.yml +0 -66
  71. metadata +1 -1
@@ -9,14 +9,14 @@ module ConceptQL
9
9
  include ConceptQL::Behaviors::Timeless
10
10
 
11
11
  register __FILE__
12
+ effects :select, :cast
12
13
 
13
- desc "Selects every patient. With one upstream operator, it selects the patients who have a record in the upstream stream."
14
- returns "One record for each selected patient, from the patients table. The type of the records is person, " \
14
+ desc "Selects every patient. With one upstream operator, it selects the patients who have a result in the upstream stream."
15
+ returns "One result for each selected patient, from the patients table. The type of the results is person, " \
15
16
  "and their start_date and end_date are both the patient's birth date."
16
17
  example "All patients", [ "person" ]
17
18
  example "The patients with an old myocardial infarction (ICD-9-CM 412)", [ "person", [ "icd9cm", "412" ] ]
18
19
 
19
- allows_one_upstream
20
20
  domains :person
21
21
 
22
22
  def my_domain
@@ -6,18 +6,19 @@ module ConceptQL
6
6
  module Operators
7
7
  class PersonFilter < BinaryOperatorOperator
8
8
  register __FILE__
9
+ effects :keep
9
10
 
10
- desc "Keeps each left-hand record whose patient has at least one record in the right-hand stream. The type and the dates of the right-hand records do not matter."
11
- returns "The left-hand records of every type whose patient has at least one right-hand record. The right-hand records are not returned."
12
- option_meaning :right, "The right-hand stream. The operator keeps the left-hand records of each patient who has a record in this stream, and does not pass these records on."
11
+ desc "Keeps each left-hand result whose patient has at least one result in the right-hand stream. The type and the dates of the right-hand results do not matter."
12
+ returns "The left-hand results of every type whose patient has at least one right-hand result. The right-hand results are not returned."
13
+ option_meaning :right, "The right-hand stream. The operator keeps the left-hand results of each patient who has a result in this stream, and does not pass these results on."
13
14
 
14
15
  example "All MI Conditions for people who are male",
15
16
  [ "person_filter", { left: [ "icd9", "412" ], right: [ "gender", "Male" ] } ]
16
- example "All MI Conditions for people who had an office visit at some point in the data",
17
+ example "All MI Conditions for people who had an office visit at some point in the dataset",
17
18
  [ "person_filter", { left: [ "icd9", "412" ], right: [ "cpt", "99214" ] } ]
18
19
  example "The same result as the example above, with the office visits converted to patients first",
19
20
  [ "person_filter", { left: [ "icd9", "412" ], right: [ "person", [ "cpt", "99214" ] ] } ]
20
- example "All MI Conditions for people who are Male OR had an office visit at some point in the data",
21
+ example "All MI Conditions for people who are Male OR had an office visit at some point in the dataset",
21
22
  [ "person_filter", { left: [ "icd9", "412" ], right: [ "union", [ "cpt", "99214" ], [ "gender", "Male" ] ] } ]
22
23
  example "Yields two types: all MI Conditions for people who are Male, and all office visit Procedures for people who are Male",
23
24
  [ "person_filter", { left: [ "union", [ "icd9", "412" ], [ "cpt", "99214" ] ], right: [ "gender", "Male" ] } ]
@@ -26,27 +26,27 @@ module ConceptQL
26
26
  # checked.
27
27
  class PlaceOfServiceFilter < Operator
28
28
  register __FILE__
29
+ effects :keep
29
30
 
30
31
  VOCABULARY_ID = "CMS Place of Service"
31
32
  ANY = "*"
32
33
 
33
- desc "Passes along the records whose place of service is one of the given CMS Place of Service codes. " \
34
- "A record that has no place of service is not passed along."
34
+ desc "Passes along the results whose place of service is one of the given CMS Place of Service codes. " \
35
+ "A result that has no place of service is not passed along."
35
36
 
36
- returns "The upstream records whose place of service is one of the given codes."
37
+ returns "The upstream results whose place of service is one of the given codes."
37
38
 
38
39
  argument :places_of_service, type: :codelist, vocab: VOCABULARY_ID,
39
40
  meaning: "One or more CMS Place of Service codes, such as 21 (inpatient hospital) or 22 (outpatient hospital). " \
40
- "The code * matches a record whose place of service is any code of that vocabulary. " \
41
- "An unknown code gets a warning and matches no record."
41
+ "The code * matches a result whose place of service is any code of that vocabulary. " \
42
+ "An unknown code gets a warning and matches no result."
42
43
 
43
44
  # The test data has no place of service, so the example returns no rows
44
45
  example "Diabetes diagnoses (ICD-9 250.00) with the place of service outpatient hospital (22)",
45
46
  [ "place_of_service_filter", 22, [ "icd9", "250.00" ] ], help: false
46
47
 
47
48
  category "Filter Single Stream"
48
- basic_type :temporal
49
- allows_one_upstream
49
+ diagram_base :temporal
50
50
  validate_one_upstream
51
51
  validate_at_least_one_argument
52
52
  require_column :visit_source_concept_id
@@ -18,29 +18,29 @@ module ConceptQL
18
18
  # provenance. At least one keyword is required.
19
19
  class Provenance < Operator
20
20
  register __FILE__
21
+ effects :keep
21
22
 
22
- desc "Passes along the records whose provenance matches at least one of the given keywords. " \
23
- "The provenance of a record says which kind of file it came from, such as inpatient, " \
23
+ desc "Passes along the results whose provenance matches at least one of the given keywords. " \
24
+ "The provenance of a result says which kind of file it came from, such as inpatient, " \
24
25
  "and where its code was on the claim, such as primary."
25
26
 
26
- returns "The upstream records whose provenance matches at least one of the keywords."
27
+ returns "The upstream results whose provenance matches at least one of the keywords."
27
28
 
28
29
  argument :provenance_types, label: "Provenance Types", type: :codelist,
29
30
  meaning: "One or more provenance keywords, in lower case. " \
30
- "A file keyword, such as inpatient, outpatient or carrier_claim, says which kind of file a record came from. " \
31
+ "A file keyword, such as inpatient, outpatient or carrier_claim, says which kind of file a result came from. " \
31
32
  "A code keyword, such as primary or first, says where the code was on the claim. " \
32
- "A plain keyword matches a record when its file provenance or its code provenance matches it. " \
33
+ "A plain keyword matches a result when its file provenance or its code provenance matches it. " \
33
34
  "A keyword also matches the provenances under it, so inpatient includes snf. " \
34
35
  "A keyword of the form file:code, such as inpatient:primary, needs both the file and the code provenance to match. " \
35
- "The keyword * matches a record that has any provenance. " \
36
- "A statement with no keyword is invalid. An unknown keyword gets a warning and matches no record."
36
+ "The keyword * matches a result that has any provenance. " \
37
+ "A statement with no keyword is invalid. An unknown keyword gets a warning and matches no result."
37
38
 
38
39
  example "Diabetes diagnoses (ICD-9 250.00) from inpatient files",
39
40
  [ "provenance", "inpatient", [ "icd9", "250.00" ] ]
40
41
 
41
42
  category "Filter Single Stream"
42
- basic_type :temporal
43
- allows_one_upstream
43
+ diagram_base :temporal
44
44
  validate_one_upstream
45
45
  default_query_columns
46
46
 
@@ -8,13 +8,14 @@ module ConceptQL
8
8
  # providers matching the given criteria.
9
9
  class ProviderFilter < Operator
10
10
  register __FILE__
11
+ effects :keep, :change
11
12
 
12
- desc "Passes along the records whose provider matches the given specialties and roles. " \
13
+ desc "Passes along the results whose provider matches the given specialties and roles. " \
13
14
  "Each option is a comma-separated list of concept IDs, and * matches any value."
14
15
 
15
- returns "The upstream records that have a provider who matches both the specialties and the roles. " \
16
- "A record can have more than one provider, so a record comes out once for each provider who matches. " \
17
- "Each record gets the columns provider_id, context_specialty_concept_id, practitioner_specialty_concept_id " \
16
+ returns "The upstream results that have a provider who matches both the specialties and the roles. " \
17
+ "A result can have more than one provider, so a result comes out once for each provider who matches. " \
18
+ "Each result gets the columns provider_id, context_specialty_concept_id, practitioner_specialty_concept_id " \
18
19
  "and role_type_concept_id."
19
20
 
20
21
  option :specialties, type: :string,
@@ -29,8 +30,7 @@ module ConceptQL
29
30
  [ "provider_filter", [ "icd9", "250.00" ], { "specialties" => "38003868", "roles" => "100000067" } ]
30
31
 
31
32
  category "Filter Single Stream"
32
- basic_type :temporal
33
- allows_one_upstream
33
+ diagram_base :temporal
34
34
  validate_one_upstream
35
35
  validate_required_options :specialties, :roles
36
36
  require_column :provider_id
@@ -13,6 +13,7 @@ module ConceptQL
13
13
  # you won't get any matches
14
14
  class Race < Operator
15
15
  register __FILE__
16
+ effects :select
16
17
 
17
18
  include ConceptQL::Behaviors::Windowable
18
19
  include ConceptQL::Behaviors::Timeless
@@ -24,12 +25,12 @@ module ConceptQL
24
25
  "The operator also selects the more detailed races under each given race, " \
25
26
  "and each concept that is a kind of one of the race concepts through an is-a relationship. " \
26
27
  "A value that contains Unknown also selects the patients who have no race or race concept ID 0."
27
- returns "One person record for each patient whose race is one of the given races. " \
28
- "The start_date and the end_date of each record are the patient's birth date."
28
+ returns "One person result for each patient whose race is one of the given races. " \
29
+ "The start_date and the end_date of each result are the patient's birth date."
29
30
  example "All patients recorded as white", [ "race", "White" ]
30
31
  domains :person
31
32
  category "Select by Property"
32
- basic_type :selection
33
+ diagram_base :selection
33
34
  validate_no_upstreams
34
35
  validate_at_least_one_argument
35
36
 
@@ -13,10 +13,11 @@ module ConceptQL
13
13
  # "define" operator and will fetch the records cached in the corresponding table
14
14
  class Recall < Operator
15
15
  register __FILE__
16
+ effects :refer
16
17
 
17
- desc "Passes on the records of the operator in the statement that has the given label. " \
18
- "Use it to break a complex statement into parts, or to use a stream of records more than once."
19
- returns "The records of the operator that has the given label."
18
+ desc "Passes on the results of the operator in the statement that has the given label. " \
19
+ "Use it to break a complex statement into parts, or to use a stream of results more than once."
20
+ returns "The results of the operator that has the given label."
20
21
  example 'Label neurological diagnoses as "neuro dxs". Recall them to remove those that were co-reported ' \
21
22
  "with a nervous-system MS-DRG. Then apply the one inpatient, two outpatient rule. " \
22
23
  "(The code lists are shortened.)",
@@ -46,11 +47,11 @@ module ConceptQL
46
47
  ]
47
48
 
48
49
  argument :name, type: :string,
49
- meaning: "The label of the operator whose records to pass on. It must match the label exactly, " \
50
+ meaning: "The label of the operator whose results to pass on. It must match the label exactly, " \
50
51
  "including case. The labeled operator can come before or after the recall in the " \
51
52
  "statement, but labeled streams that recall each other in a loop are invalid."
52
53
  category "Get Related Data"
53
- basic_type :selection
54
+ diagram_base :selection
54
55
  validate_no_upstreams
55
56
  validate_one_argument
56
57
  default_query_columns
@@ -6,13 +6,14 @@ module ConceptQL
6
6
  module Operators
7
7
  class Snf < Operator
8
8
  register __FILE__
9
+ effects :select
9
10
  include ConceptQL::Behaviors::Utilizable
10
11
 
11
- desc "Selects admissions to skilled nursing facilities (SNF): one record for each admission whose claim came from an SNF file."
12
- returns "One record for each SNF admission. The type of the records is condition_occurrence, " \
12
+ desc "Selects admissions to skilled nursing facilities (SNF): one result for each admission whose claim came from an SNF file."
13
+ returns "One result for each SNF admission. The type of the results is condition_occurrence, " \
13
14
  "and their criterion_table is collections, the table of claims. Their start_date and end_date " \
14
15
  "are the dates of the claim, and source_value holds the primary code of the admission when it has one. " \
15
- "Each record also has admission and discharge columns, such as admission_date, discharge_date and length_of_stay."
16
+ "Each result also has admission and discharge columns, such as admission_date, discharge_date and length_of_stay."
16
17
  # The test data has no SNF claims
17
18
  example "All SNF admissions", [ "snf" ], help: false
18
19
 
@@ -6,6 +6,7 @@ module ConceptQL
6
6
  module Operators
7
7
  class Sum < PassThru
8
8
  register __FILE__
9
+ effects :aggregate
9
10
  # "We do not yet support numeric or sum in any meaningful way"
10
11
  # (Ryan, 2026-11-03, seed conceptql-mdo.4)
11
12
  hidden
@@ -26,13 +27,13 @@ module ConceptQL
26
27
  vocabularies.length > 1
27
28
  end
28
29
 
29
- desc "Groups the records that have the same values in all columns except criterion_id, start_date, " \
30
- "end_date and value_as_number. For each group, the operator passes along one record " \
30
+ desc "Groups the results that have the same values in all columns except criterion_id, start_date, " \
31
+ "end_date and value_as_number. For each group, the operator passes along one result " \
31
32
  "with the sum of value_as_number, the earliest start_date, the most recent end_date " \
32
- "and a criterion_id of 0. With group_by_person, the operator groups all of the records " \
33
- "of each person and passes along one person record for each person."
33
+ "and a criterion_id of 0. With group_by_person, the operator groups all of the results " \
34
+ "of each person and passes along one person result for each person."
34
35
  option :group_by_person, type: :boolean,
35
- instructions: "Choose whether to sum all of the records of each person into one person record, for example to count the events of each person"
36
+ instructions: "Choose whether to sum all of the results of each person into one person result, for example to count the results of each person"
36
37
  default_query_columns
37
38
  validate_at_least_one_upstream
38
39
  validate_no_arguments
@@ -24,6 +24,7 @@ module ConceptQL
24
24
  # start: 'd', end: '' # Only adjust start_date by positive 1 day and leave end_date uneffected
25
25
  class TimeWindow < PassThru
26
26
  register __FILE__
27
+ effects :change
27
28
 
28
29
  # The values that the start and end options take (DateAdjuster)
29
30
  DATE_ADJUSTMENT_TEXT = "The value is an amount of time, made of one or more numbers, each with a unit: " \
@@ -31,17 +32,17 @@ module ConceptQL
31
32
  "a bare number means days, and a unit without a number means 1 of it. " \
32
33
  "For example, -1m is 1 month back, and 10d3m is 3 months and 10 days. " \
33
34
  "An empty value or 0 leaves the date unchanged. The value start or end uses the " \
34
- "record's start_date or end_date, and a date in YYYY-MM-DD format sets the date to that date."
35
+ "result's start_date or end_date, and a date in YYYY-MM-DD format sets the date to that date."
35
36
 
36
- desc "Moves the start_date and the end_date of each record. " \
37
- "Use it to shift or widen a record's dates before you compare them with other records."
37
+ desc "Moves the start_date and the end_date of each result. " \
38
+ "Use it to shift or widen a result's dates before you compare them with other results."
38
39
  option :start, type: :string,
39
40
  instructions: 'Enter a number followed by "d", "w", "m" or "y" for days, weeks, months or years, such as 30d. A negative number moves the date earlier. You can also enter start, end or a date in YYYY-MM-DD format.',
40
41
  meaning: "How to change the start_date. If you omit it, the start_date does not change. #{DATE_ADJUSTMENT_TEXT}"
41
42
  option :end, type: :string,
42
43
  instructions: 'Enter a number followed by "d", "w", "m" or "y" for days, weeks, months or years, such as 30d. A negative number moves the date earlier. You can also enter start, end or a date in YYYY-MM-DD format.',
43
44
  meaning: "How to change the end_date. If you omit it, the end_date does not change. #{DATE_ADJUSTMENT_TEXT}"
44
- returns "One record for each upstream record, with the start_date and the end_date that the options give. " \
45
+ returns "One result for each upstream result, with the start_date and the end_date that the options give. " \
45
46
  "The other columns do not change."
46
47
  example "All Diagnoses of Diabetes (ICD-9 250.01) within 30 days of an MI",
47
48
  [ "during", { "left" => [ "icd9", "250.01" ], "right" => [ "time_window", [ "icd9", "412" ], { "start" => "-30d", "end" => "30d" } ] } ]
@@ -52,12 +53,11 @@ module ConceptQL
52
53
  [ "time_window", [ "icd9", "412" ], { "start" => "", "end" => "start" } ]
53
54
  example "Nonsensical, but allowed: swap the start_date and end_date",
54
55
  [ "time_window", [ "icd9", "412" ], { "start" => "end", "end" => "start" } ]
55
- allows_one_upstream
56
56
  validate_one_upstream
57
57
  validate_no_arguments
58
58
  validate_option DateAdjuster::VALID_INPUT, :start, :end
59
59
  category "Modify Data"
60
- basic_type :temporal
60
+ diagram_base :temporal
61
61
 
62
62
  def query(db)
63
63
  dm.selectify(db.from(stream.evaluate(db)).from_self,
@@ -16,12 +16,13 @@ module ConceptQL
16
16
  # record is passed thru unaffected.
17
17
  class TrimDateEnd < TrimDate
18
18
  register __FILE__
19
+ effects :keep, :change
19
20
 
20
- desc "For each patient, moves the end_date of each left-hand record back to the start_date of the right-hand record with the earliest start_date. A left-hand record that starts after that start_date is removed, and the left-hand records of a patient without right-hand records pass through unchanged."
21
- returns "The left-hand records, each with its end_date set to the earlier of its own end_date and the start_date of the patient's right-hand record. Left-hand records that start after that start_date are not returned. The right-hand records are not returned."
22
- option_meaning :left, "The left-hand stream. The operator moves the end_date of these records and passes them on."
23
- option_meaning :right, "The right-hand stream. For each patient, the earliest start_date of these records sets the latest end_date of the left-hand records. These records are not passed on."
24
- option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is subtracted from the start_date of the right-hand record before the comparison. The adjusted date is the one that moves the end_date of the left-hand record."
21
+ desc "For each patient, moves the end_date of each left-hand result back to the start_date of the right-hand result with the earliest start_date. A left-hand result that starts after that start_date is removed, and the left-hand results of a patient without right-hand results pass through unchanged."
22
+ returns "The left-hand results, each with its end_date set to the earlier of its own end_date and the start_date of the patient's right-hand result. Left-hand results that start after that start_date are not returned. The right-hand results are not returned."
23
+ option_meaning :left, "The left-hand stream. The operator moves the end_date of these results and passes them on."
24
+ option_meaning :right, "The right-hand stream. For each patient, the earliest start_date of these results sets the latest end_date of the left-hand results. These results are not passed on."
25
+ option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is subtracted from the start_date of the right-hand result before the comparison. The adjusted date is the one that moves the end_date of the left-hand result."
25
26
 
26
27
  example "Each patient's span from birth to age 50 or to 1970-01-01, whichever is earlier (patients born after 1970 are dropped)",
27
28
  [ "trim_date_end", { left: [ "time_window", [ "person" ], { end: "+50y" } ],
@@ -16,12 +16,13 @@ module ConceptQL
16
16
  # record is passed thru unaffected.
17
17
  class TrimDateStart < TrimDate
18
18
  register __FILE__
19
+ effects :keep, :change
19
20
 
20
- desc "For each patient, moves the start_date of each left-hand record forward to the end_date of the right-hand record with the latest start_date. A left-hand record that ends before that end_date is removed, and the left-hand records of a patient without right-hand records pass through unchanged."
21
- returns "The left-hand records, each with its start_date set to the later of its own start_date and the end_date of the patient's right-hand record. Left-hand records that end before that end_date are not returned. The right-hand records are not returned."
22
- option_meaning :left, "The left-hand stream. The operator moves the start_date of these records and passes them on."
23
- option_meaning :right, "The right-hand stream. For each patient, the end_date of the record with the latest start_date sets the earliest start_date of the left-hand records. These records are not passed on."
24
- option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is added to the end_date of the right-hand record before the comparison. The adjusted date is the one that moves the start_date of the left-hand record."
21
+ desc "For each patient, moves the start_date of each left-hand result forward to the end_date of the right-hand result with the latest start_date. A left-hand result that ends before that end_date is removed, and the left-hand results of a patient without right-hand results pass through unchanged."
22
+ returns "The left-hand results, each with its start_date set to the later of its own start_date and the end_date of the patient's right-hand result. Left-hand results that end before that end_date are not returned. The right-hand results are not returned."
23
+ option_meaning :left, "The left-hand stream. The operator moves the start_date of these results and passes them on."
24
+ option_meaning :right, "The right-hand stream. For each patient, the end_date of the result with the latest start_date sets the earliest start_date of the left-hand results. These results are not passed on."
25
+ option_meaning :within, "A date adjustment, such as 30d, 2m or 1y-3d, that is added to the end_date of the right-hand result before the comparison. The adjusted date is the one that moves the start_date of the left-hand result."
25
26
 
26
27
  example "Each patient's span from birth to age 50, trimmed to start no earlier than 1980-01-01 (patients who were 50 before 1980 are dropped)",
27
28
  [ "trim_date_start", { left: [ "time_window", [ "person" ], { end: "+50y" } ],
@@ -6,6 +6,7 @@ module ConceptQL
6
6
  module Operators
7
7
  class Union < PassThru
8
8
  register __FILE__
9
+ effects :combine
9
10
 
10
11
  class Combiner
11
12
  attr_reader :combinables
@@ -54,8 +55,10 @@ module ConceptQL
54
55
  end
55
56
  end
56
57
 
58
+ # Each upstream passes along its own records, so with two or more
59
+ # upstreams one person can get one row from each (bead conceptql-8q8h).
57
60
  def events_per_patient
58
- upstreams.all? { |u| u.events_per_patient == :single } ? :single : :multiple
61
+ upstreams.length > 1 ? :multiple : super
59
62
  end
60
63
 
61
64
  def multiple_vocabularies
@@ -64,11 +67,11 @@ module ConceptQL
64
67
  vocabularies.length > 1
65
68
  end
66
69
 
67
- desc "Combines the records of all its upstream operators into one stream. " \
68
- "Each record keeps its own type, so one stream can hold several types."
70
+ desc "Combines the results of all its upstream operators into one stream. " \
71
+ "Each result keeps its own type, so one stream can hold several types."
69
72
 
70
- returns "Each record of each upstream. A record that comes from more than one upstream comes out once. " \
71
- "Two records are the same only when every column is equal, including label."
73
+ returns "Each result of each upstream. A result that comes from more than one upstream comes out once. " \
74
+ "Two results are the same only when every column is equal, including label."
72
75
 
73
76
  example "Two streams of the same type (condition_occurrence) combined into one stream",
74
77
  [ "union", [ "icd9", "412" ], [ "icd9", "250.01" ] ]
@@ -77,7 +80,6 @@ module ConceptQL
77
80
  example "Two condition_occurrence streams and a procedure_occurrence stream combined by a single union",
78
81
  [ "union", [ "icd9", "412" ], [ "icd9", "250.01" ], [ "cpt", "99214" ] ]
79
82
 
80
- allows_many_upstreams
81
83
  category "Combine Streams"
82
84
  default_query_columns
83
85
  validate_at_least_one_upstream
@@ -29,11 +29,15 @@ module ConceptQL
29
29
  include ConceptQL::Behaviors::CodeLister
30
30
 
31
31
  category "Select by Clinical Codes"
32
- basic_type :selection
33
- desc "Selects the records whose code is one of the given codes in a vocabulary, such as ICD-9-CM diagnoses or CPT procedures. " \
32
+ diagram_base :selection
33
+ effects :select
34
+ desc "Selects the results whose code is one of the given codes in a vocabulary, such as ICD-9-CM diagnoses or CPT procedures. " \
34
35
  "Each vocabulary has its own operator, and a few operators search several vocabularies at once."
35
36
  validate_no_upstreams
36
37
  validate_at_least_one_argument
38
+ # MultipleVocabularies gets this error from its vocabulary operators
39
+ # (#aggregate_messages), so it does not declare the validation itself.
40
+ validate_string_codes
37
41
  validate_codes_match
38
42
 
39
43
  # VocabularyClassFactory gives each vocabulary operator a copy of this
@@ -41,18 +45,18 @@ module ConceptQL
41
45
  argument :codes, type: :codelist,
42
46
  meaning: "One or more codes of the vocabulary. Write each code as the vocabulary writes it, " \
43
47
  "including any dots: 250.01, not 25001. A code matches exactly, including case, " \
44
- "and it does not select its child codes. A code alone of * selects every record " \
48
+ "and it does not select its child codes. A code alone of * selects every result " \
45
49
  "of the vocabulary, and there is no other wildcard. A code that does not have the " \
46
50
  "vocabulary's format matches nothing and gives a warning."
47
51
 
48
- returns "Every record whose code is one of the given codes in the operator's vocabulary. " \
49
- "The type of the records depends on the vocabulary, for example condition_occurrence for icd9cm " \
52
+ returns "Every result whose code is one of the given codes in the operator's vocabulary. " \
53
+ "The type of the results depends on the vocabulary, for example condition_occurrence for icd9cm " \
50
54
  "and procedure_occurrence for cpt4. An operator does not follow mappings to other vocabularies."
51
- example "All records with the ICD-9-CM diagnosis code 412 (old myocardial infarction)", [ "icd9cm", "412" ]
52
- example "All MS-DRG 205 records: other respiratory system diagnoses with a major complication or comorbidity",
55
+ example "All results with the ICD-9-CM diagnosis code 412 (old myocardial infarction)", [ "icd9cm", "412" ]
56
+ example "All MS-DRG 205 results: other respiratory system diagnoses with a major complication or comorbidity",
53
57
  [ "ms_drg", "205" ]
54
58
  # The test data has no DRG record before 2007-10-01
55
- example "All CMS-DRG 205 records (a liver disorder)", [ "cms_drg", "205" ], help: false
59
+ example "All CMS-DRG 205 results (a liver disorder)", [ "cms_drg", "205" ], help: false
56
60
 
57
61
  def source_table
58
62
  domain_map(op_name)
@@ -67,6 +67,12 @@ module ConceptQL
67
67
  @cdb = cdb
68
68
  @given_statement = statement
69
69
  @statement, opts = extract_statement(statement, opts)
70
+ # Window::Table queries a window table statement with this Database.
71
+ # Scope#windows merges the scope options with the window options, so a
72
+ # window table statement gets it from both places that a window table
73
+ # comes from: a window statement and the scope options of the calling
74
+ # program.
75
+ opts[:scope_opts] = (opts[:scope_opts] || {}).merge(cdb: cdb)
70
76
  @opts = opts
71
77
  opts[:algorithm_fetcher] ||= proc do |alg|
72
78
  statement, description = db[:concepts].where(concept_id: alg).get(%i[statement label])
@@ -247,7 +253,7 @@ module ConceptQL
247
253
  # around it. It annotates this Query's own operator tree, so only a fresh
248
254
  # Query calls it.
249
255
  def annotate_without_ids(opts)
250
- opts = opts.merge(skip_counts: true) unless window_table_errors.empty?
256
+ opts = opts.merge(skip_counts: true) unless window_table_errors.empty? && window_option_errors.empty?
251
257
  nodifier.scope.one_query_counts = OneQueryCounts.new(operator, db, opts)
252
258
  with_window(operator.annotate(db, opts), opts)
253
259
  end
@@ -283,11 +289,23 @@ module ConceptQL
283
289
  @valid = true
284
290
  end
285
291
 
292
+ # The window options of the scope, as Scope#windows reads them: the
293
+ # scope options of the calling program, merged with the options of the
294
+ # statement's window
295
+ def window_opts
296
+ nodifier.scope.window_opts
297
+ end
298
+
286
299
  # The window table of the scope, as Window::Table reads it: a table name
287
300
  # or a ConceptQL statement (an Array)
288
301
  def window_table
289
- scope_opts = nodifier.scope.opts
290
- scope_opts.merge(scope_opts[:window_opts] || {})[:window_table]
302
+ window_opts[:window_table]
303
+ end
304
+
305
+ # The errors of the window options (Window.option_errors), as
306
+ # [message, *details]. The window operator of the annotation has them.
307
+ def window_option_errors
308
+ Window.option_errors(window_opts)
291
309
  end
292
310
 
293
311
  # The validation errors of the window table's statement, when the window
@@ -302,18 +320,28 @@ module ConceptQL
302
320
  end
303
321
  end
304
322
 
323
+ # The options of the statement's window, as the scope runs them: for a
324
+ # window inside a window, the options of the inner window. Nil when the
325
+ # statement is not a window statement.
326
+ def statement_window_options
327
+ nodifier.scope.opts[:window_opts] if @window_statement
328
+ end
329
+
305
330
  # The annotated statement inside its window operator, when the statement
306
- # is a window statement or the scope's window table is a statement. The
307
- # window operator has the options of the statement's window and an
308
- # annotation hash, which holds only its id. A window table statement is
309
- # annotated in place, with the same opts.
331
+ # is a window statement, the scope's window table is a statement, or the
332
+ # window options have errors. The window operator has the options of the
333
+ # statement's window (statement_window_options) and an annotation hash,
334
+ # which holds its id and the errors of the window options. A window table
335
+ # statement is annotated in place, with the same opts.
310
336
  def with_window(annotated, opts)
311
337
  table = window_table
312
- return annotated unless @window_options || table.is_a?(Array)
338
+ errors = window_option_errors
339
+ window_options = statement_window_options
340
+ return annotated unless window_options || table.is_a?(Array) || errors.any?
313
341
 
314
- options = (@window_options || {}).dup
342
+ options = (window_options || {}).dup
315
343
  options[:window_table] = cdb.query(table).annotate_without_ids(opts) if table.is_a?(Array)
316
- [ "window", annotated, options.merge(annotation: {}) ]
344
+ [ "window", annotated, options.merge(annotation: errors.empty? ? {} : { errors: errors }) ]
317
345
  end
318
346
 
319
347
  def extract_statement(stmt, opts)
@@ -325,9 +353,10 @@ module ConceptQL
325
353
  raise "window operator needs a ConceptQL statement followed by a hash as the last item of the array."
326
354
  end
327
355
 
356
+ # The options of an inner window replace those of an outer window.
328
357
  opts[:scope_opts] = (opts[:scope_opts] || {})
329
- opts[:scope_opts].merge!(window_opts: stmt.last.merge(cdb: cdb))
330
- @window_options = (@window_options || {}).merge(stmt.last)
358
+ opts[:scope_opts].merge!(window_opts: stmt.last)
359
+ @window_statement = true
331
360
  extract_statement(stmt[1], opts)
332
361
  elsif stmt.length == 1 && stmt.first.is_a?(Array)
333
362
  extract_statement(stmt.first, opts)
@@ -100,7 +100,7 @@ module ConceptQL
100
100
  @operator_cte_names = {}.compare_by_identity
101
101
  @opts = opts.dup
102
102
  @query_columns = DEFAULT_COLUMNS.keys
103
- @query_columns << :window_id if opts.dig(:window_opts, :window_table)
103
+ @query_columns << :window_id if window_opts[:window_table]
104
104
 
105
105
  @i = 0
106
106
  @mutex = Mutex.new
@@ -634,7 +634,15 @@ module ConceptQL
634
634
  end
635
635
 
636
636
  def windows
637
- @windows ||= Window.from(opts.merge(opts[:window_opts] || {}))
637
+ @windows ||= Window.from(window_opts)
638
+ end
639
+
640
+ # The window options that run: the scope options, merged with the options
641
+ # of the statement's window (window_opts), which win. A window_table at the
642
+ # top of the scope options is a window table, as one in window_opts is
643
+ # (bead conceptql-6t7q).
644
+ def window_opts
645
+ opts.merge(opts[:window_opts] || {})
638
646
  end
639
647
 
640
648
  def force_temp_tables?(options = {})
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ConceptQL
4
- VERSION = "0.8.0"
4
+ VERSION = "0.9.0"
5
5
  end
@@ -27,13 +27,15 @@ module ConceptQL
27
27
  @operator_names ||= operators.map(&:op_name)
28
28
  end
29
29
 
30
+ # True when every operator except union takes no upstreams. t_shank
31
+ # saves this value in its performance statistics.
30
32
  def codeset_only?
31
33
  return @codeset_only if defined?(@codeset_only)
32
34
 
33
35
  @codeset_only = operators.reject do |op|
34
36
  # We skip Unions because they don't count against codesets
35
37
  op.op_name == "union"
36
- end.map(&:class).map(&:basic_type).all? { |type| type == :selection }
38
+ end.map(&:class).all? { |klass| klass.upstreams_facet == "none" }
37
39
  end
38
40
 
39
41
  def max_depth