blue_button_parser 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,14 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+  gem "json"
+end

data/Gemfile.lock

@@ -0,0 +1,22 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    json (1.5.4)
+    rake (0.9.2.2)
+    rcov (0.9.11)
+    shoulda (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  json
+  rcov
+  shoulda

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 PatientsLikeMe
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,114 @@
+= BlueButtonParser
+
+BlueButtonParser parses a BlueButton free-text personal health data file and translates it into a structured hash suitable for computational purposes.
+
+BlueButton[http://www.va.gov/bluebutton/] is the initiative from the U.S. Department of Veterans Affairs
+to allow veterans to download their information from the "My HealtheVet" personal health record
+into a very simple text file.   Because this file was meant to be human readable, not computationally readable, the file contains almost no markup or delimiters for sections, keys, or values.
+
+BlueButtonParser was created by reverse-engineering the one 
+{sample data file provided by the VA}[http://www.va.gov/BLUEBUTTON/docs/VA_My_HealtheVet_Blue_Button_Sample_Version_12_All_Data.txt] 
+and creating some ad-hoc rules for how to parse the document.  BlueButtonParser will attempt to find all the sections in the file,  all the key-value pairs within that section, and even find collections of items within a section when applicable (e.g. the array of facilities in the section "TREATMENT FACILITIES").
+
+<b>Example free text data</b>
+
+  ----------------------------- DEMOGRAPHICS ----------------------------
+
+  Source: Self-Entered
+
+  First Name: ONE
+  Middle Initial: A
+  Last Name: MHVVETERAN
+  Suffix: 
+  Alias: MHVVET
+  Relationship to VA: Patient, Veteran, Employee
+
+  Gender: Male   Blood Type: AB+         Organ Donor: Yes
+
+  Date of Birth: 01 Mar 1948
+  Marital Status: Married
+  Current Occupation: Truck Driver
+
+<b>Example parsed data (JSON)</b>
+  
+  "DEMOGRAPHICS": {
+    "Source": "Self-Entered",
+    "First Name": "ONE",
+    "Middle Initial": "A",
+    "Last Name": "MHVVETERAN",
+    "Suffix": null,
+    "Alias": "MHVVET",
+    "Relationship to VA": "Patient, Veteran, Employee",
+    "Gender": "Male",
+    "Blood Type": "AB+",
+    "Organ Donor": "Yes",
+    "Date of Birth": "01 Mar 1948",
+    "Marital Status": "Married",
+    "Current Occupation": "Truck Driver"
+  }
+
+== Install
+
+  sudo gem install blue_button_parser
+
+== Usage
+  require 'blue_button_parser'
+
+  # parse your downloaded BlueButton data file
+  my_bb_file = File.read("test/data/blue_button_example_data.txt")
+  bbp = BlueButtonParser.new(my_bb_file)
+
+  # access the parsed data as a Hash
+  parsed_data_hash = bbp.data
+
+  # example: to see the list of sections parsed:
+  parsed_data_hash.keys
+  # => ["MY HEALTHEVET ACCOUNT SUMMARY", "VITALS AND READINGS", "HEALTH INSURANCE", ... ]
+
+  # example: to acccess the "MY HEALTHEVET ACCOUNT SUMMARY" section:
+  summary = parsed_data_hash["MY HEALTHEVET ACCOUNT SUMMARY"]
+  # => {"Authentication Facility Name"=>"SLC10 TEST LAB", "Authentication Date"=>"19 Aug 2010", ... }
+    
+= Caveats
+
+BlueButtonParser was reverse engineered based on the single sample data file provided by the VA (latest version: v12, 02 Dec 2011).  Because there are no rules as to how the document should be formatted, 
+
+* A later version of the BlueButton output could break the parser
+* An instance of BlueButton output for a second patient might have slightly different formatting--for example, line-wrapped values for fields that were not wrapped in the current reference file, or new fields that were not applicable to the patient in the current reference file
+* As new sections get added, there is no guarantee that they will conform to the formatting for other sections
+
+To keep the BlueButtonParser up-to-date, the test data file (test/data/blue_button_example_data.txt) and expect parsed output (test/data/expected_json_output.js) should be updated every time a new version of the BlueButtonData file is released.   
+
+Note however that as far as I know, there is no formal notification that a new version of the sample data file has been released, so I guess developers will just need to be vigilant. :)
+
+After updates, make sure the tests still work and any applicable new tests get added.
+
+= Prior work
+
+Somebody took a stab at this in the past:
+http://rest-developer-edition.na8.force.com/BlueConverter
+
+I believe the code for this example is found here:
+https://github.com/joshbirk/BlueConverter
+
+BlueConverter great first pass implementation, but it needs a few corrections:
+
+* it doesn't deal with multiple key-values on single line, e.g. "Gender: Male  Blood Type: AB+  Organ Donor: Yes"
+* it doesn't deal with values that wrap lines
+* it doesn't recognize that some of the key-values in a section should be grouped into a collection of items, e.g. in the "TREATMENT FACILITIES" section is actually comprised of a series of facilities
+* it doesn't parse the tables, e.g. the "VA Treating Facility" in the "MY HEALTHEVET ACCOUNT SUMMARY"
+
+== Contributing to BlueButtonParser
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2012 PatientsLikeMe. See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "blue_button_parser"
+  gem.homepage = "http://github.com/patientslikeme/blue_button_parser"
+  gem.license = "MIT"
+  gem.summary = %Q{Converts a BlueButton free text data file to a structured data Hash}
+  gem.description = %Q{Converts a BlueButton free text data file to a structured data Hash}
+  gem.email = "open_source@patientslikeme.com"
+  gem.authors = ["PatientsLikeMe"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "blue_button_parser #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/blue_button_parser.gemspec

@@ -0,0 +1,64 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{blue_button_parser}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["PatientsLikeMe"]
+  s.date = %q{2011-12-22}
+  s.description = %q{Converts a BlueButton free text data file to a structured data Hash}
+  s.email = %q{open_source@patientslikeme.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "blue_button_parser.gemspec",
+    "lib/blue_button_parser.rb",
+    "test/data/blue_button_example_data.txt",
+    "test/data/expected_json_output.js",
+    "test/helper.rb",
+    "test/test_blue_button_parser.rb"
+  ]
+  s.homepage = %q{http://github.com/patientslikeme/blue_button_parser}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.4.1}
+  s.summary = %q{Converts a BlueButton free text data file to a structured data Hash}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<json>, [">= 0"])
+    else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+  end
+end
+

data/lib/blue_button_parser.rb

@@ -0,0 +1,282 @@
+class BlueButtonParser
+  
+  attr_reader :text, :data, :config
+
+  ALWAYS_SKIP_LINES = ["^[-]+$", "^[-]+[ ]+$", "^[ ]+[-]+", "^[=]+$", "^(- ){5,}", "END OF MY HEALTHEVET"]
+
+  DEFAULT_CONFIG = {
+    "MY HEALTHEVET PERSONAL INFORMATION REPORT" => {
+      :same_line_keys => ["Name", "Date of Birth"],
+    },
+    "DOWNLOAD REQUEST SUMMARY" => {},
+    "MY HEALTHEVET ACCOUNT SUMMARY" => {
+      :collection => {"Facilities" => {:table_columns => ["VA Treating Facility", " Type"]}}
+    },
+    "DEMOGRAPHICS" => {
+      :collection => {"EMERGENCY CONTACTS" => {:item_starts_with => "Contact First Name"}},      
+      :same_line_keys => [["Gender", "Blood Type", "Organ Donor"], ["Work Phone Number", "Extension"]],
+    },    
+    "HEALTH CARE PROVIDERS" => {
+      :collection => {"Providers" => {:item_starts_with => "Provider Name"}},
+      :same_line_keys => ["Phone Number", "Ext"],
+    },
+    "TREATMENT FACILITIES" => {
+      :collection => {"Facilities" => {:item_starts_with => "Facility Name"}},
+      :same_line_keys => [["Facility Type", "VA Home Facility"], ["Phone Number", "Ext"]],      
+    },
+    "HEALTH INSURANCE" => {
+      :collection => {"Companies" => {:item_starts_with => "Health Insurance Company"}},
+      :same_line_keys => [["ID Number", "Group Number"], ["Start Date", "Stop Date"]],
+    },
+    "VA WELLNESS REMINDERS" => {
+      :collection => {"Reminders" => {:table_columns => ["Wellness Reminder", "Due Date", "Last Completed", "Location"]}}
+    },
+    "VA APPOINTMENTS" => {
+      :collection => {"Appointments" => {:item_starts_with => "Date/Time"}},
+      :skip_lines => ["^FUTURE APPOINTMENTS:", "^PAST APPOINTMENTS:"]
+    },
+    "VA MEDICATION HISTORY" => {
+      :collection => {"Medications" => {:item_starts_with => "Medication"}},
+    },
+    "MEDICATIONS AND SUPPLEMENTS" => {
+      :collection => {"Medications" => {:item_starts_with => "Category"}},
+      :same_line_keys => [["Start Date", "Stop Date"], ["Pharmacy Name", "Pharmacy Phone"]],
+    },
+    "VA ALLERGIES" => {
+      :collection => {"Allergies" => {:item_starts_with => "Allergy Name"}},
+    },
+    "ALLERGIES/ADVERSE REACTIONS" => {
+      :collection => {"Allergies" => {:item_starts_with => "Allergy Name"}},
+    },
+    "MEDICAL EVENTS" => {
+      :collection => {"Event" => {:item_starts_with => "Medical Event"}},      
+    },
+    "IMMUNIZATIONS" => {
+      :collection => {"Immunizations" => {:item_starts_with => "Immunization"}},
+    },
+    "VA LABORATORY RESULTS" => {
+      :collection => {"Labs" => {:item_starts_with => "Lab Test"}},
+    },
+    "LABS AND TESTS" => {
+      :collection => {"Labs" => {:item_starts_with => "Test Name"}},
+    },
+    "VITALS AND READINGS" => {
+      :collection => {"Reading" => {:item_starts_with => "Measurement Type"}},      
+    },
+    "FAMILY HEALTH HISTORY" => {
+      :collection => {"Relation" => {:item_starts_with => "Relationship"}},            
+    },
+    "MILITARY HEALTH HISTORY" => {
+      :same_line_keys => [["Service Branch", "Rank"],["Location of Service", "Onboard Ship"]]
+    },    
+    "DOD MILITARY SERVICE INFORMATION" => {
+      :collection => {
+        "Regular Active Service" => {:table_starts_with => "-- Regular Active Service", :table_columns => ["Service", "Begin Date", "End Date", "Character of Service", "Rank"] },
+        "Reserve/Guard Association Periods" => {:table_starts_with => "-- Reserve/Guard Association Periods", :table_columns => ["Service", "Begin Date", "End Date", "Character of Service", "Rank"] },
+        "DoD MOS/Occupation Codes" => {:table_starts_with => "-- Note: Both Service and DoD Generic codes", :table_columns => ["Service", "Begin Date", "Enl/Off", "Type", "Svc Occ Code", "DoD Occ Code"]}
+      },
+      :skip_lines => ["^Translations of Codes Used in this Section"]
+    }
+  }
+  
+  def initialize(bb_data_text, config=DEFAULT_CONFIG, newline="\n")
+    @text = bb_data_text
+    @config= config
+    @data = parse_text(@text, newline)
+  end
+  
+  private
+  
+  def new_section?(line)
+    new_section = line.match(/^[-]+ (.*) [-]+/) 
+    new_section = new_section[1] if new_section     
+    return new_section
+  end
+  
+  def new_collection?(line, last_line, current_section)
+    new_collection = nil
+
+    if collections = sect_config(current_section, :collection)
+      collections.each_pair do |name, collection_config|  
+        if starts_with = collection_config[:item_starts_with]
+          if line.match(Regexp.new("^#{starts_with}:"))
+            new_collection = name
+            break
+          end
+        elsif table_columns = collection_config[:table_columns]
+          new_table = false
+          
+          if table_starts_with = collection_config[:table_starts_with]
+            if last_line.match(Regexp.new("^#{table_starts_with}"))
+              new_table = true
+            end
+          else
+            new_table = true
+          end
+      
+          if new_table
+            regexp_str = ".?#{table_columns.join('.*')}.?"
+            if line.match(Regexp.new(regexp_str))
+              new_collection = name
+              break
+            end
+          end
+          
+        end
+      end
+    end
+    
+    return new_collection
+  end
+    
+  def get_multi_key_values(line, current_section)
+    key_values = nil
+
+    if key_sets = sect_config(current_section, :same_line_keys)
+      unless key_sets.first.is_a?(Array)
+        key_sets = [key_sets]
+      end
+      
+      key_sets.each do |keys|
+        regexp_str = keys.collect{|k| "#{k}: (.*)"}.join
+        regexp = Regexp.new(regexp_str)
+        if keys_match = line.match(regexp)
+          key_values = Hash.new
+          keys.each_with_index do |key, index|
+            key_values[key] = keys_match[index + 1]
+          end
+          break
+        end
+      end
+    end
+    
+    return key_values
+  end
+    
+  def get_single_key_value(line)    
+    if key_match = line.match(/(.*)\: (.*)?/) 
+      key_values = {key_match[1] => key_match[2]}
+    elsif key_match = line.match(/(.*):$/)
+      key_values = {key_match[1] => nil}
+    else
+      key_values = nil      
+    end
+
+    return key_values
+  end  
+  
+  def get_key_values(line, current_section)
+    key_values = get_multi_key_values(line, current_section) 
+    key_values = get_single_key_value(line) if key_values.nil?            
+    return key_values
+  end
+  
+  def key_ended?(line)
+    # either an empty line or a line starting with a key (e.g. "Status:") means we're done with multi-line
+    line.empty? or line.match(/^\S(.*): (\S*)/)
+  end
+  
+  def sect_config(current_section, key)
+    if @config[current_section]
+      @config[current_section][key]
+    else
+      nil
+    end
+  end  
+  
+  def parse_table_line(line, columns, column_widths)
+    row = Hash.new
+    columns.each_with_index do |column, index|
+      start = column_widths[index]
+      finish = if index == column_widths.size - 1
+          line.size
+        else
+          column_widths[index + 1]
+        end
+      val_str = (line[start, finish-start]).strip
+      val_str = val_str.empty? ? nil : val_str
+      row[column] = val_str
+    end
+    return row
+  end
+  
+  def table_columns(current_section, current_collection)
+    sect_config(current_section, :collection)[current_collection][:table_columns]
+  end
+  
+  def column_widths(line, columns)
+    columns.collect{|c| line.index(c)}
+  end
+  
+  def parse_text(text, newline="\n")    
+    # parse text line by line
+    lines = text.split(newline)
+
+    # state variables
+    current_section = nil    
+    current_collection = nil
+    current_key = nil
+    current_table = nil
+    
+    # put parsed data into this hash
+    data = Hash.new    
+    
+    # start parsing
+    lines.each_with_index do |line, index|
+      skip_regexps = (ALWAYS_SKIP_LINES + (sect_config(current_section, :skip_lines) || [])).compact
+      skip_regexps = skip_regexps.collect{|r| Regexp.new(r)}
+      next if skip_regexps.find{|re| re.match(line)}
+      
+      if collection = new_collection?(line, lines[index - 1], current_section)
+        current_collection = collection
+        current_key = nil
+        data[current_section][current_collection] ||= []
+        if columns = table_columns(current_section, current_collection)
+          current_table = {:columns => columns, :widths => column_widths(line, columns)}
+          next
+        else
+          data[current_section][current_collection] << Hash.new
+        end
+      end
+
+      if key_ended?(line)
+        current_key = nil
+      end
+
+      if current_section and current_key
+        value = line.rstrip
+        if current_collection.nil?
+          data[current_section][current_key] = [data[current_section][current_key], value].compact.join(" \n")
+        else
+          (data[current_section][current_collection].last)[current_key] = [(data[current_section][current_collection].last)[current_key], value].compact.join(" \n")
+        end
+      end
+      
+      if section = new_section?(line)
+        current_section = section        
+        current_collection = nil
+        data[current_section] ||= {}
+      elsif current_table
+        if line.empty?
+          current_collection = nil
+          current_table = nil
+        else
+          data[current_section][current_collection] << parse_table_line(line, current_table[:columns], current_table[:widths])
+        end
+      elsif (!current_key and current_section and key_values = get_key_values(line, current_section))
+        key_values.each_pair do |key, value|
+          val = (value.nil? or value.strip.empty?) ? nil : value.strip #empty strings should be converted to nils
+          if current_collection.nil?
+            data[current_section][key] = val
+          else
+            (data[current_section][current_collection].last)[key] = val
+          end
+          current_key = key
+        end
+      end
+    end
+
+    return data
+  end
+
+end