conceptql 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7346b1988f7233add68cf70f2f5617a1b2df86fc
-  data.tar.gz: 893e14fdc17bc4dfed78868017e7cebd5739ddc5
+  metadata.gz: bb925d74153217fcb29ee4c4e5874827005246be
+  data.tar.gz: 41ae6d964b8a6ae82b37098adcda8548917543bd
 SHA512:
-  metadata.gz: a7618751466878c7a78d2c0ef738201ed3612eff584a811e0ebf42f4c3cbeb662cea01a60ce8c6b452d37842aada37923103b5559bd71e34db65f812cad212a8
-  data.tar.gz: 7f26dee3ab73028146901d121c6fc54cc08bad65e49a890e7a1ff3f97a48d5f11e7c7014984e6de2c2ef0c18f1bd983e2953a8f8b7138b69f4f80052d93808cd
+  metadata.gz: f62d0381dbd1ceabed67e9fff58d95975bf4cff27a957f03aa0ba495ca7726b22af3727d26acd4f683ca048a17dac858dffd2bef7f2e0707cae7e5cfd29e6677
+  data.tar.gz: 004743713781e5ea7cf7281ea071597ac8fa0959714571639d97d69a9fdd67b03692b10b9e1c77802230f57149212e9daf5077459d02db86814759051b44cc62

data/.gitignore

@@ -5,6 +5,7 @@
 .bundle
 .config
 .yardoc
+.idea
 Gemfile.lock
 InstalledFiles
 _yardoc

data/CHANGELOG.md

@@ -1,6 +1,58 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## Unreleased
+
+### Added
+- Optimized After/Before nodes when multi-person results are in the right stream.
+- Set end_date as coalesce(end_date, start_date) to make range if end_date missing.
+- Nodes
+    - AnyOverlap
+    - Contains
+    - Filter
+    - Ndc
+    - ObservationPeriod
+    - OneInTwoOut
+    - OverlappedBy
+    - Overlaps
+    - TrimDateEnd
+    - TrimDateStart
+- Nodes for CPRD
+    - Medcode
+    - MedcodeProcedure
+    - ObservationByEnttype
+    - Prodcode
+- Nodes for SEER
+    - FromSeerVisits
+    - ToSeerVisits
+- Ability to limit results to a set of patients by setting Tree#person_ids
+- "units_source_value" and "source_value" columns in results
+- TimeWindow supports date literals
+- ConditionType supports search for "primary"
+- Nodifier#to_metadata
+
+### Changed
+- Except now allows :ignore_date option
+    - Comparison is only done on criterion_id/type
+- DateRange START/END use observation_period instead of visit_occurrence
+- Operator#columns value_as_numeric => value_as_number
+- Syntax is now more "lispy"
+- Recall now uses any labeled operator as if that operator was fed to "Define" operator
+- Nodes now called Operators
+
+### Deprecated
+- Nothing.
+
+### Removed
+- Let/Define operators
+
+### Fixed
+- Many broken specs.
+- Union calls #from_self on incoming streams to avoid column issues
+- Concept node now works again
+- Define ensures tables are built
+
+
 ## 0.1.1 - 2014-09-18
 
 ### Added
@@ -13,7 +65,7 @@ All notable changes to this project will be documented in this file.
 - Nothing.
 
 ### Fixed
-- Calling Query#sql no longer creates a bunch of temporary tables
+- Calling Query#sql no longer creates a bunch of temporary tables.
 
 
 ## 0.1.0 - 2014-09-04

data/Guardfile

@@ -1,28 +1,56 @@
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 
-guard :minitest do
-  # with Minitest::Unit
-  watch(%r{^test/(.*)\/?test_(.*)\.rb$})
-  watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/#{m[1]}test_#{m[2]}.rb" }
-  watch(%r{^test/test_helper\.rb$})      { 'test' }
-
-  # with Minitest::Spec
-  watch(%r{^spec/(.*)_spec\.rb$})
-  watch(%r{^lib/(.+)\.rb$})         { |m| "spec/#{m[1]}_spec.rb" }
-  watch(%r{^spec/spec_helper\.rb$}) { 'spec' }
-
-  # Rails 4
-  # watch(%r{^app/(.+)\.rb$})                               { |m| "test/#{m[1]}_test.rb" }
-  # watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' }
-  # watch(%r{^app/controllers/(.+)_controller\.rb$})        { |m| "test/integration/#{m[1]}_test.rb" }
-  # watch(%r{^app/views/(.+)_mailer/.+})                   { |m| "test/mailers/#{m[1]}_mailer_test.rb" }
-  # watch(%r{^lib/(.+)\.rb$})                               { |m| "test/lib/#{m[1]}_test.rb" }
-  # watch(%r{^test/.+_test\.rb$})
-  # watch(%r{^test/test_helper\.rb$}) { 'test' }
-
-  # Rails < 4
-  # watch(%r{^app/controllers/(.*)\.rb$}) { |m| "test/functional/#{m[1]}_test.rb" }
-  # watch(%r{^app/helpers/(.*)\.rb$})     { |m| "test/helpers/#{m[1]}_test.rb" }
-  # watch(%r{^app/models/(.*)\.rb$})      { |m| "test/unit/#{m[1]}_test.rb" }
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+
+  # Rails files
+  rails = dsl.rails(view_extensions: %w(erb haml slim))
+  dsl.watch_spec_files_for(rails.app_files)
+  dsl.watch_spec_files_for(rails.views)
+
+  watch(rails.controllers) do |m|
+    [
+      rspec.spec.("routing/#{m[1]}_routing"),
+      rspec.spec.("controllers/#{m[1]}_controller"),
+      rspec.spec.("acceptance/#{m[1]}")
+    ]
+  end
+
+  # Rails config changes
+  watch(rails.spec_helper)     { rspec.spec_dir }
+  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
+  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
+
+  # Capybara features specs
+  watch(rails.view_dirs)     { |m| rspec.spec.("features/#{m[1]}") }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
+    Dir[File.join("**/#{m[1]}.feature")][0] || "spec/acceptance"
+  end
 end

data/conceptql.gemspec

@@ -18,13 +18,15 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '~> 4.1'
+  spec.add_dependency 'facets', '~> 3.0'
   spec.add_dependency 'sequelizer', '~> 0.0'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'pg', '~> 0.17'
   spec.add_dependency 'ruby-graphviz', '~> 1.2'
+  spec.add_dependency 'csv2xlsx', '~> 0'
   spec.add_development_dependency 'bundler', '~> 1.5'
   spec.add_development_dependency 'rake', '~> 10.3'
-  spec.add_development_dependency 'minitest', '~> 5.4'
-  spec.add_development_dependency 'guard-minitest', '~> 2.3'
+  spec.add_development_dependency 'rspec', '~> 3.2'
+  spec.add_development_dependency 'guard-rspec', '~> 4.5'
+  spec.add_development_dependency 'byebug', '~> 4.0'
 end

data/doc/spec.md

@@ -153,7 +153,7 @@ Virtually all other nodes add, remove, filter, or otherwise alter streams of res
 Because streams represent sets of results, its makes sense to include a nodes that operate on sets
 
 ### Union
-- Takes any number of child nodes and aggregates their streams
+- Takes any number of upstream nodes and aggregates their streams
     - Unions together streams with identical types
         - Think of streams with the same type flowing together into a single stream
         - We're really just gathering the union of all IDs for identically-typed streams
@@ -1333,7 +1333,7 @@ Just like Filter has an :as option, add one to Except node.  This would simplify
 
 
 ### How to Handle fact_relationship Table from CDMv5
-Each relationship type could be a binary node box read as L <relationship> R. E.g. L 'parent of' R would take a L stream and only pass on parents of rows in R stream.
+Each relationship type could be a binary node box read as L <relationship> R. E.g. L 'downstream of' R would take a L stream and only pass on downstreams of rows in R stream.
 
 We could implement a single node that takes a relationship as an argument (on top of the L and R arguments) or we could create a node class for each relationship.  I think it would be better to have a single relationship node class and take the relationship as the argument.
 

data/lib/conceptql/behaviors/debuggable.rb

@@ -0,0 +1,70 @@
+require_relative '../behaviors/dottable'
+require_relative '../operators/operator'
+require_relative '../operators/binary_operator_operator'
+require 'csv'
+module ConceptQL
+  module Behaviors
+    module Debuggable
+      include Dottable
+      class ResultPrinter
+        attr :db, :dir, :type, :watch_ids, :operator
+        def initialize(db, dir, type, watch_ids, operator)
+          @db = db
+          @dir = dir
+          @type = type
+          @watch_ids = watch_ids
+          @operator = operator
+        end
+
+        def make_file
+          CSV.open(file_path, 'w') do |csv|
+            csv << ConceptQL::Operators::Operator::COLUMNS
+            results.each do |result|
+              csv << result
+            end
+          end
+          file_path
+        end
+
+        def file_path
+          @file_path ||= dir + file_name
+        end
+
+        def file_name
+          @file_name ||= [operator.operator_name, abbreviate(type)].join('_')
+        end
+
+        def results
+          q = operator.evaluate(db)
+            .from_self
+            .where(criterion_type: type.to_s)
+          unless watch_ids.empty?
+            q = q.where(person_id: watch_ids)
+          end
+
+          q.order([:person_id, :criterion_type, :start_date, :end_date, :criterion_id])
+            .select_map(ConceptQL::Operators::Operator::COLUMNS)
+        end
+
+        def abbreviate(type)
+          type.to_s.split('_').map(&:chars).map(&:first).join('')
+        end
+      end
+
+      def print_results(db, dir, watch_ids)
+        print_prep(db) if respond_to?(:print_prep)
+        kids = upstreams
+        if self.is_a?(ConceptQL::Operators::BinaryOperatorOperator)
+          kids = [left, right]
+        end
+        files = kids.map do |upstream|
+          upstream.print_results(db, dir, watch_ids)
+        end
+        files += types.map do |type|
+          ResultPrinter.new(db, dir, type, watch_ids, self).make_file
+        end
+      end
+    end
+  end
+end
+

data/lib/conceptql/behaviors/dottable.rb

@@ -1,4 +1,6 @@
-require 'active_support/inflector'
+require 'facets/string/snakecase'
+require 'facets/string/titlecase'
+
 module ConceptQL
   module Behaviors
     module Dottable
@@ -15,13 +17,23 @@ module ConceptQL
         observation: 'magenta',
         misc: 'black'
       }
-      def node_name
-        @__node_name ||= self.class.name.split('::').last.underscore.gsub(/\W/, '_').downcase + '_' + (@@counter += 1).to_s
+
+      def operator_number
+        @__operator_number ||= (@@counter += 1)
+      end
+
+      def reset_operator_number
+        @@counter = 0
+      end
+
+      def operator_name
+        @__operator_name ||= self.class.name.split('::').last.snakecase.gsub(/\W/, '_').downcase + "_#{operator_number}"
       end
 
       def display_name
         @__display_name ||= begin
-          output = self.class.name.split('::').last.titleize
+          output = self.class.name.split('::').last.snakecase.titlecase
+          #output += " #{operator_number}"
           output += ": #{arguments.join(', ')}" unless arguments.empty?
           if output.length > 100
             parts = output.split
@@ -39,9 +51,9 @@ module ConceptQL
         types.length == 1 ? TYPE_COLORS[types.first] || 'black' : 'black'
       end
 
-      def graph_node(g)
-        @__graph_node ||= begin
-          me = g.add_nodes(node_name)
+      def graph_operator(g)
+        @__graph_operator ||= begin
+          me = g.add_nodes(operator_name)
           me[:label] = display_name
           me[:color] = type_color(types)
           me[:shape] = shape if respond_to?(:shape)
@@ -49,7 +61,7 @@ module ConceptQL
         end
       end
 
-      def link_to(g, dest_node, db = nil)
+      def link_to(g, dest_operator, db = nil)
         edge_options = {}
 
         types.each do |type|
@@ -60,23 +72,25 @@ module ConceptQL
             edge_options[:label] = label.join("\n")
             edge_options[:style] = 'dashed' if my_n.zero?
           end
-          e = g.add_edges(graph_node(g), dest_node, edge_options)
+          e = g.add_edges(graph_operator(g), dest_operator, edge_options)
           e[:color] = type_color(type)
         end
       end
 
       def graph_it(g, db)
         graph_prep(db) if respond_to?(:graph_prep)
-        children.each do |child|
-          child.graph_it(g, db)
+        upstreams.each do |upstream|
+          upstream.graph_it(g, db)
         end
-        graph_node(g)
-        children.each do |child|
-          child.link_to(g, graph_node(g), db)
+        operator = graph_operator(g)
+        upstreams.each do |upstream|
+          upstream.link_to(g, graph_operator(g), db)
         end
+        operator
       end
 
       def my_count(db, type)
+        puts "counting #{operator_name} #{type}"
         evaluate(db).from_self.where(criterion_type: type.to_s).count
       end
 

data/lib/conceptql/behaviors/metadatable.rb

@@ -0,0 +1,97 @@
+require 'facets/kernel/meta_def'
+require 'facets/string/snakecase'
+
+module Metadatable
+  def preferred_name(value = nil)
+    return @preferred_name unless value
+    @preferred_name = value
+  end
+
+  def desc(value = nil)
+    return @desc unless value
+    @desc = value
+  end
+
+  def predominant_types(*values)
+    return @predominant_types if values.empty?
+    @predominant_types = values
+  end
+
+  def argument(name, options = {})
+    (@arguments ||= [])
+    @arguments << [name, options]
+  end
+
+  def option(name, options = {})
+    @options ||= {}
+    @options[name] = options
+  end
+
+  def types(*type_list)
+    @types = type_list
+    define_method(:types) do
+      type_list
+    end
+    if type_list.length == 1
+      define_method(:type) do
+        type_list.first
+      end
+    end
+  end
+
+  def allows_many_upstreams
+    @max_upstreams = 99
+  end
+
+  def allows_one_upstream
+    @max_upstreams = 1
+  end
+
+  def just_class_name
+    self.to_s.split('::').last
+  end
+
+  def humanized_class_name
+    just_class_name.gsub(/([A-Z])/, ' \1').lstrip
+  end
+
+  def category(category)
+    (@categories ||= [])
+    @categories << Array(category)
+  end
+
+  def reset_categories
+    @categories = []
+  end
+
+  def inherited(upstream)
+    (@options || {}).each do |name, opt|
+      upstream.option name, opt
+    end
+
+    (@categories || []).each do |cat|
+      upstream.category cat
+    end
+
+    case @max_upstreams
+    when 1
+      upstream.allows_one_upstream
+    when 99
+      upstream.allows_many_upstreams
+    end
+  end
+
+  def to_metadata
+    {
+      preferred_name: @preferred_name || humanized_class_name,
+      operation: just_class_name.snakecase,
+      max_upstreams: @max_upstreams || 0,
+      arguments: @arguments || [],
+      options: @options || {},
+      predominant_types: @types || @predominant_types || [],
+      desc: @desc,
+      categories: @categories || []
+    }
+  end
+end
+

data/lib/conceptql/behaviors/preppable.rb

@@ -0,0 +1,20 @@
+module ConceptQL
+  module Behaviors
+    module Preppable
+      attr_accessor :prep_proc
+      def prep
+        prep_proc.call if prep_proc
+      end
+
+      def all(*args)
+        prep
+        super
+      end
+
+      def count(*args)
+        prep
+        super
+      end
+    end
+  end
+end

data/lib/conceptql/cli.rb

@@ -39,16 +39,18 @@ module ConceptQL
       q = ConceptQL::Query.new(db(options), criteria_from_file(statement_file))
       puts q.sql
       puts q.statement.to_yaml
-      pp q.execute
+      pp q.all
     end
 
     desc 'show_graph statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph'
+    option :watch_file
     def show_graph(file)
       graph_it(criteria_from_file(file))
     end
 
     desc 'show_and_tell_file statement_file', 'Reads the ConceptQL statement from the file and shows the contents as a ConceptQL graph, then executes the statement against the DB'
     option :full
+    option :watch_file
     def show_and_tell_file(file)
       show_and_tell(criteria_from_file(file), options)
     end
@@ -60,9 +62,28 @@ module ConceptQL
       system('open /tmp/graph.pdf')
     end
 
+    desc 'metadata', 'Generates the metadata.js file for the JAM'
+    def metadata
+      File.write('/tmp/metadata.js', "var metadata = #{ConceptQL::Nodifier.new.to_metadata.to_json};")
+    end
+
+    desc 'convert', 'Converts from hash-based syntax to list-based syntax'
+    def convert(file)
+      require 'conceptql/converter'
+      require 'json'
+      begin
+        puts JSON.pretty_generate(ConceptQL::Converter.new.convert(criteria_from_file(file)))
+      rescue
+        puts "Couldn't convert #{file}"
+        puts $!.message
+        puts $!.backtrace.join("\n")
+      end
+    end
+
     private
     desc 'show_and_tell_db conceptql_id', 'Fetches the ConceptQL from a DB and shows the contents as a ConceptQL graph, then executes the statement against our test database'
     option :full
+    option :watch_file
     def show_and_tell_db(conceptql_id)
       result = fetch_conceptql(conceptql_id, options)
       puts "Concept: #{result[:label]}"
@@ -88,12 +109,11 @@ module ConceptQL
       puts q.statement.to_yaml
       puts 'JSON'
       puts JSON.pretty_generate(q.statement)
-      STDIN.gets
       graph_it(statement, title)
       STDIN.gets
-      puts q.query.sql
+      puts q.sql
       STDIN.gets
-      results = q.execute
+      results = q.all
       if options[:full]
         pp results
       else
@@ -105,12 +125,18 @@ module ConceptQL
     def graph_it(statement, title = nil)
       require_relative 'graph'
       require_relative 'tree'
+      conn = db(options)
       ConceptQL::Graph.new(statement,
                            dangler: true,
                            title: title,
-                           db: db(options)
+                           db: conn
                           ).graph_it('/tmp/graph')
       system('open /tmp/graph.pdf')
+      if options[:watch_file]
+        require_relative 'debugger'
+        debugger = ConceptQL::Debugger.new(statement, db: conn, watch_ids: File.readlines(options[:watch_file]).map(&:to_i))
+        debugger.capture_results('/tmp/debug.xlsx')
+      end
     end
 
     def criteria_from_file(file)

data/lib/conceptql/converter.rb

@@ -0,0 +1,65 @@
+require 'facets/array/extract_options'
+require 'facets/hash/deep_rekey'
+require 'facets/hash/update_values'
+
+module ConceptQL
+  class Converter
+    def convert(statement)
+      traverse(statement).to_list_syntax
+    end
+
+  private
+    class Operator
+      attr :type, :values, :options
+      def initialize(type, *values)
+        @type, @values = type, values.flatten
+        @options = @values.extract_options!.deep_rekey
+        @values = @values
+      end
+
+      def args
+        values.select { |s| !s.is_a?(Operator) }
+      end
+
+      def upstreams
+        values.select { |s| s.is_a?(Operator) }
+      end
+
+      def converted_options
+        options.update_values do |value|
+          if value.is_a?(Operator)
+            value.to_list_syntax
+          else
+            value
+          end
+        end
+      end
+
+      def to_list_syntax
+        stmt = [type]
+        stmt += args unless args.empty?
+        stmt += upstreams.map(&:to_list_syntax) unless upstreams.empty?
+        stmt << converted_options unless options.empty?
+        stmt
+      end
+    end
+
+    def traverse(obj)
+      case obj
+      when Hash
+        if obj.keys.length > 1
+          obj = Hash[obj.map { |key, value| [ key, traverse(value) ]}]
+          return obj
+        end
+        type = obj.keys.first
+        values = traverse(obj[type])
+        obj = Operator.new(type, values)
+        obj
+      when Array
+        obj.map { |value| traverse(value) }
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/conceptql/debugger.rb

@@ -0,0 +1,48 @@
+require_relative 'tree'
+require_relative 'operators/operator'
+require_relative 'behaviors/debuggable'
+
+module ConceptQL
+  class Debugger
+    attr :statement, :db, :watch_ids
+    def initialize(statement, opts = {})
+      @statement = statement
+      @db = opts.fetch(:db, nil)
+      @tree = opts.fetch(:tree, Tree.new)
+      ConceptQL::Operators::Operator.send(:include, ConceptQL::Behaviors::Debuggable)
+      @watch_ids = opts.fetch(:watch_ids, [])
+      raise "Please specify one or more person_ids you'd like to debug" unless @watch_ids
+    end
+
+    def capture_results(path)
+      raise "Please specify path for debug file" unless path
+      Dir.mktmpdir do |dir|
+        dir = Pathname.new(dir)
+        operators = tree.root(self)
+        operators.first.reset_operator_number
+        csv_files = operators.map.with_index do |last_operator, index|
+          last_operator.print_results(db, dir, watch_ids)
+        end.flatten
+        system("csv2xlsx #{path} #{csv_files.join(' ')}")
+      end
+    end
+
+    private
+    attr :yaml, :tree, :db
+
+    def build_graph(g)
+      tree.root(self).each.with_index do |last_operator, index|
+        last_operator.graph_it(g, db)
+        if dangler
+          blank_operator = g.add_nodes("_#{index}")
+          blank_operator[:shape] = 'none'
+          blank_operator[:height] = 0
+          blank_operator[:label] = ''
+          blank_operator[:fixedsize] = true
+          last_operator.link_to(g, blank_operator, db)
+        end
+      end
+    end
+  end
+end
+