common_core_parser 0.0.1

data/lib/common_core_parser/broken_data_corrector.rb

@@ -0,0 +1,172 @@
+module CommonCoreParser
+  class BrokenDataCorrector
+
+    class << self
+      def run
+        Master.instance.elements.each do |element|
+          self.private_methods.select { |method_name| method_name.to_s.match(/^correct_/) }.each do |corrector_method|
+            self.send(corrector_method,element)
+          end
+        end
+      end
+
+      # There are a some cases where multiple items are sharing the same ref_id.
+      # * Two cases of an ELA Standard and Domain having the same ref_id.
+      # * Three cases of pairs of clusters sharing the same ref_id.
+      # * Two standards that both have the same ref_id
+      # This patch corrects the ref_ids to remove conflucts.
+      # See also patch_duplicated_parent_ref_ids()
+      def patch_duplicated_ref_ids(element,candidate_ref_id)
+        flagged_standard_domain_ref_ids = ['7CB154907B5743c7B97BFCFF452D7977','F053D3437D1E4338A2C18B25DACBED85']
+        flagged_cluster_ref_ids = ['B1AC98EADE4145689E70EEEBD9B8CC18','834B17E279C64263AA83F7625F5D2993','91FABAB899814C55851003A0EE98F8FB']
+        flagged_duplicated_stadard_ref_ids = ['FBCBB7C696FE475695920CA622B1C857']
+        if candidate_ref_id == 'F053D3437D1E4338A2C18B25DACBED85' and element.is_a?(Standard)
+          return "Standard:DUPLICATEDREF_ID:#{candidate_ref_id}"
+        elsif candidate_ref_id == 'F053D3437D1E4338A2C18B25DACBED85' and element.is_a?(Domain)
+          return "Domain:DUPLICATEDREF_ID:#{candidate_ref_id}"
+        elsif flagged_standard_domain_ref_ids.include?(candidate_ref_id) and element.is_a?(Standard)
+          return "Standard:DUPLICATEDREF_ID:#{candidate_ref_id}"
+        elsif flagged_cluster_ref_ids.include?(candidate_ref_id) and element.is_a?(Cluster)
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_ref_id}:#{element.code}"
+        elsif flagged_duplicated_stadard_ref_ids.include?(candidate_ref_id) and element.is_a?(Standard)
+          return "Standard:DUPLICATEDREF_ID:#{candidate_ref_id}:#{element.code}"
+        else
+          return candidate_ref_id
+        end
+      end
+
+      # See comments on patch_duplicated_ref_ids() above.
+      def patch_duplicated_parent_ref_ids(element,candidate_parent_ref_id)
+        flagged_cluster_ref_ids = ['B1AC98EADE4145689E70EEEBD9B8CC18','834B17E279C64263AA83F7625F5D2993','91FABAB899814C55851003A0EE98F8FB']
+        if candidate_parent_ref_id == 'F053D3437D1E4338A2C18B25DACBED85' and element.is_a?(Component)
+          return "Standard:DUPLICATEDREF_ID:#{candidate_parent_ref_id}"
+        elsif candidate_parent_ref_id == 'F053D3437D1E4338A2C18B25DACBED85' and element.is_a?(Standard)
+          return "Domain:DUPLICATEDREF_ID:#{candidate_parent_ref_id}"
+        elsif flagged_cluster_ref_ids.include?(candidate_parent_ref_id) and element.is_a?(Standard) and element.code.match(/\.K\.G\./)
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_parent_ref_id}:Mathematics.K.G.1"
+        elsif flagged_cluster_ref_ids.include?(candidate_parent_ref_id) and element.is_a?(Standard) and element.code == 'CCSS.Math.Content.K.MD.B.3'
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_parent_ref_id}:Mathematics.K.MD.2"
+        elsif flagged_cluster_ref_ids.include?(candidate_parent_ref_id) and element.is_a?(Standard) and element.code.match(/\.3\.G\.A\./)
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_parent_ref_id}:Mathematics.3.G.1"
+        elsif flagged_cluster_ref_ids.include?(candidate_parent_ref_id) and element.is_a?(Standard) and element.code == 'CCSS.Math.Content.6.NS.A.1'
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_parent_ref_id}:Mathematics.6.NS.1"
+        elsif flagged_cluster_ref_ids.include?(candidate_parent_ref_id) and element.is_a?(Standard) and element.code.match(/\.3\.MD\.C\./)
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_parent_ref_id}:Mathematics.3.MD.3"
+        elsif flagged_cluster_ref_ids.include?(candidate_parent_ref_id) and element.is_a?(Standard) and element.code.match(/\.6\.RP\.A\./)
+          return "Cluster:DUPLICATEDREF_ID:#{candidate_parent_ref_id}:Mathematics.6.RP.1"
+        else
+          return candidate_parent_ref_id
+        end
+      end
+
+      #######
+      private
+      #######
+
+      # A few standards in the new xml files are are pointed to cluster ids that
+      # don't exist in the old xml files. Point them to their old clusters for now
+      # This shouldn't be necessary when corestandards.org makes a complete set of
+      # updated xml files available.
+      def correct_standards_pointed_to_missing_ref_ids(element)
+        ref_id_map = {
+          'B1AC98EADE4145689E70EEEBD9B8CC19' => patch_duplicated_parent_ref_ids(element,'B1AC98EADE4145689E70EEEBD9B8CC18'),
+          '834B17E279C64263AA83F7625F5D2994' => patch_duplicated_parent_ref_ids(element,'834B17E279C64263AA83F7625F5D2993'),
+          '91FABAB899814C55851003A0EE98F8FC' => patch_duplicated_parent_ref_ids(element,'91FABAB899814C55851003A0EE98F8FB')
+        }
+        return unless (element.is_a?(Standard) and ref_id_map.keys.include?(element.parent_ref_id))
+        element.instance_variable_set(:@parent_ref_id,ref_id_map[element.parent_ref_id])
+      end
+
+      # The "Standards for Mathematical Practice" point to each other instead of
+      # a cluster or domain.  WTF? Let's just orphanize them for now.
+      def correct_parent_ref_id_for_practice_standards(element)
+        return unless (element.is_a?(Standard) and element.code.match(/CCSS\.Math\.Practice\.MP[0-9]+/))
+        element.instance_variable_set(:@parent_ref_id,'INTENTIONALLYORPHANED')
+      end
+
+      # The ELA "Anchor Standards" don't have a parent in the xml.
+      # Flagging as intentionally orphaned until it gets corrected.
+      def correct_ela_anchor_standards(element)
+        return unless (element.is_a?(Standard) and element.parent_ref_id.blank? and element.code.match(/CCSS\.ELA\-Literacy\.CCRA.[RWSL]+\.[0-9]+/))
+        element.instance_variable_set(:@parent_ref_id,'INTENTIONALLYORPHANED')
+      end
+
+      # The ELA Grade 3 "Language" Standards don't have a parent in the xml.
+      # Point them to the correct Domain
+      def correct_ela_grade_3_language_standards(element)
+        return unless (element.is_a?(Standard) and element.parent_ref_id.blank? and element.code.match(/CCSS\.ELA\-Literacy\.L\.3/))
+        element.instance_variable_set(:@parent_ref_id,patch_duplicated_parent_ref_ids(element,'F053D3437D1E4338A2C18B25DACBED85'))
+      end
+
+      def correct_unnecessarily_unicoded_characters(element)
+        # http://www.htmlescape.net/unicode_chart_general_punctuation.html
+        # http://www.htmlescape.net/unicode_chart_miscellaneous_symbols.html
+        element.instance_variable_set(:@statement,element.statement.gsub(/\u2013/,%q{-})) # &#8211;
+        element.instance_variable_set(:@statement,element.statement.gsub(/\u2019/,%q{'})) # &#8217;
+        element.instance_variable_set(:@statement,element.statement.gsub(/\u2020/,%q{"})) # &#8220;
+        element.instance_variable_set(:@statement,element.statement.gsub(/\u2021/,%q{"})) # &#8221;
+        element.instance_variable_set(:@statement,element.statement.gsub(/\u2605/,%q{*})) # &#9733;
+      end
+
+      # Math.xml contains a lot of unnecessarily escaped tags.
+      def correct_unncessarily_escaped_html_tags(element)
+        statement = element.statement
+        [:sup, :i].each do |tag|
+          statement.gsub!(/&lt;#{tag}&gt;/,%Q{<#{tag}>})
+          statement.gsub!(/&lt;\/#{tag}&gt;/,%Q{</#{tag}>})
+        end
+        element.instance_variable_set(:@statement,statement)
+      end
+
+      # One ELA standard has it's own RefID (currently 32D9C8830A6C4fd5B715A1DFFC4D4BA4) set as it's code.
+      def correct_literacy_rh_11_12_1_code(element)
+        if (element.ref_id == '32D9C8830A6C4fd5B715A1DFFC4D4BA4' or element.parent_ref_id == 'F9877ECD38D8432185F96C5EC5A050AF') and element.statement.match('^Cite specific textual evidence')
+          element.instance_variable_set(:@code,'CCSS.ELA-Literacy.RH.11-12.1')
+        end
+      end
+
+      # Many standards contain html tags that are closed at the beginning of the following standard.
+      def correct_unclosed_html_tags(element)
+        [:i, :sup].each do |tag|
+          strip_stray_close_tags(element,tag)
+          append_missing_close_tags(element,tag)
+        end
+      end
+
+
+
+
+
+
+
+      ######################################################################################
+
+      def strip_stray_close_tags(element,tag)
+        if starts_with_close?(element,tag)
+          element.instance_variable_set(:@statement,element.statement.sub(/^<\/#{tag}>/,''))
+        end
+      end
+
+      def append_missing_close_tags(element,tag)
+        if more_opens_than_closes?(element,tag)
+          element.instance_variable_set(:@statement,"#{element.statement}</#{tag}>")
+        end
+      end
+
+      def starts_with_close?(element,tag)
+        element.statement.match(/^<\/#{tag}>/)
+      end
+
+
+      def more_opens_than_closes?(element,tag)
+        opens = element.statement.match(/<#{tag}>/)
+        closes = element.statement.match(/<\/#{tag}>/)
+        return false if opens.nil?
+        return true if closes.nil? or (opens.size == closes.size + 1)
+        raise "too many unclosed open <#{tag}> tags to deal with" if opens.size > closes.size + 1 # just in case this becomes an issue in the future so it can be identified immediately
+        return false
+      end
+    end
+
+  end
+end

data/lib/common_core_parser/element.rb

@@ -0,0 +1,100 @@
+module CommonCoreParser
+  class Element
+
+    attr_accessor :data
+    attr_reader :children_hash
+
+    def initialize(xmldoc)
+      raise(ArgumentError, "Not a LearningStandardItem\n #{xmldoc}") unless xmldoc.name == 'LearningStandardItem'
+      @data = xmldoc
+      @children_hash = {}
+    end
+
+    def to_s
+      "<#{self.class} ref_id: #{ref_id}, parent_ref_id: #{parent_ref_id}, code: #{code}, statement: #{statement}, grades: #{grades.join(',')}>"
+    end
+
+    def ref_id
+      @ref_id ||= BrokenDataCorrector.patch_duplicated_ref_ids(self,(data.attributes['RefID'] || data.attributes['RefId']).value.strip)
+    end
+
+    def parent
+      @parent ||= Master.instance.elements_hash[parent_ref_id]
+    end
+
+    def children
+      children_hash.values
+    end
+
+    def parent_ref_id
+      return @parent_ref_id if @parent_ref_id
+
+      #new file format
+      data.xpath('./RelatedLearningStandardItems/LearningStandardItemRefId').each do |lsiri|
+        return @parent_ref_id = BrokenDataCorrector.patch_duplicated_parent_ref_ids(self,lsiri.text.strip) if lsiri.attributes['RelationshipType'].value == 'childOf'
+      end
+
+      #old file format
+      data.xpath('./PredecessorItems/LearningStandardItemRefId').each do |lsiri|
+        return @parent_ref_id = BrokenDataCorrector.patch_duplicated_parent_ref_ids(self,lsiri.text.strip)
+      end
+
+      return nil
+    end
+
+    def add_child(element)
+      @children_hash[element.ref_id] = element
+    end
+
+    def code
+      @code ||= data.xpath('./StatementCodes/StatementCode').first.text.strip
+    end
+
+    def statement
+      @statement ||= data.xpath('./Statements/Statement').first.text.strip
+    end
+
+    def grades
+      @grades ||= data.xpath('./GradeLevels/GradeLevel').map(&:text).map {|string| convert_grade_string_to_grades(string) }.flatten
+    end
+
+    def valid_grades?
+      (grades & %w(K HS K-12 01 02 03 04 05 06 07 08 09 10 11 12)) == grades
+    end
+
+    def valid?
+      errors.blank?
+    end
+
+    def errors
+      raise RuntimeError
+    end
+
+    def error_message
+      errors.join(',')
+    end
+
+    def illigit?
+      return false unless ref_id.blank?
+      case
+        when (ref_id.blank? and statement.match(/not applicable/)) then true
+        when (ref_id.blank? and statement.match(/begins in grade [0-9]+/)) then true
+        else false
+      end
+    end
+
+    #######
+    private
+    #######
+
+    def convert_grade_string_to_grades(string)
+      grades = case
+        when string == 'K-12' then ['K',*(1..12)]
+        when string == 'HS' then [*(9..12)]
+        when string.match(/([0-9]+)-([0-9]+)/) then [*($1..$2)]
+        else [string]
+      end
+      grades.map{|string| string == 'K' ? string : sprintf('%02d',string.to_i) }
+    end
+  end
+end

data/lib/common_core_parser/elements/cluster.rb

@@ -0,0 +1,10 @@
+module CommonCoreParser
+  class Cluster < CommonCoreParser::Element
+
+    def errors
+      errors = []
+      return errors
+    end
+
+  end
+end

data/lib/common_core_parser/elements/component.rb

@@ -0,0 +1,10 @@
+module CommonCoreParser
+  class Component < CommonCoreParser::Element
+
+    def errors
+      errors = []
+      return errors
+    end
+
+  end
+end

data/lib/common_core_parser/elements/domain.rb

@@ -0,0 +1,13 @@
+module CommonCoreParser
+  class Domain < CommonCoreParser::Element
+
+    def errors
+      errors = []
+      errors << 'Refid is blank' if ref_id.blank?
+      errors << 'Code is blank' if code.blank?
+      errors << 'Statement is blank' if statement.blank?
+      errors << "invalid grades: #{grades}" unless valid_grades?
+      return errors
+    end
+  end
+end

data/lib/common_core_parser/elements/standard.rb

@@ -0,0 +1,31 @@
+module CommonCoreParser
+  class Standard < CommonCoreParser::Element
+
+    def errors
+      errors = []
+      errors << 'Refid is blank' if ref_id.blank?
+      errors << 'PRI is blank' if parent_ref_id.blank?
+      errors << 'Code is blank' if code.blank?
+      errors << 'Statement is blank' if statement.blank?
+      errors << 'invalid grades' unless valid_grades?
+      return errors
+    end
+
+    def error_message
+      errors.join(',')
+    end
+
+    def cluster
+      @cluster ||= self.parent.is_a?(Cluster) ? self.parent : nil
+    end
+
+    def domain
+      @domain ||= cluster ? cluster.parent : self.parent
+    end
+
+    def subject
+      @subject ||= domain.try(:parent)
+    end
+
+  end
+end

data/lib/common_core_parser/elements/standard_type.rb

@@ -0,0 +1,10 @@
+module CommonCoreParser
+  class StandardType < CommonCoreParser::Element
+
+    def errors
+      errors = []
+      return errors
+    end
+
+  end
+end

data/lib/common_core_parser/elements/subject_grade.rb

@@ -0,0 +1,10 @@
+module CommonCoreParser
+  class SubjectGrade < CommonCoreParser::Element
+
+    def errors
+      errors = []
+      return errors
+    end
+
+  end
+end

data/lib/common_core_parser/master.rb

@@ -0,0 +1,85 @@
+module CommonCoreParser
+  class Master
+    include Singleton
+
+    ELEMENT_NAMES = [:elements, :domains, :clusters, :standards, :components, :subject_grades, :standard_types]
+    attr_reader *( ELEMENT_NAMES.map { |element_name| "#{element_name}_hash" } )
+
+    def initialize
+      ELEMENT_NAMES.each do |element_name|
+        instance_variable_set("@#{element_name}_hash",{})
+      end
+    end
+
+    def load_math
+      load_elements_from_paths(DATA_PATH+'/Math.xml',DATA_PATH+'/Mathematics/**/*.xml')
+    end
+
+    def load_ela
+      load_elements_from_paths(DATA_PATH+'/ELA-Literacy.xml',DATA_PATH+'/ELA/**/*.xml')
+    end
+
+    def load_elements_from_paths(*paths)
+      self.send(:initialize)
+      paths.flatten.each do |path|
+        Dir.glob(path).map do |filename|
+          xml = Nokogiri::XML(File.open(filename))
+          xml.xpath('//LearningStandardItem').each do |learning_standard_item|
+            klass = get_class_from_element(learning_standard_item)
+            add_element(klass.new(learning_standard_item))
+          end
+        end
+      end
+      correct_bad_data
+      reunite_children_with_parents
+      return standards
+    end
+
+    def add_element(element)
+      return if element.illigit?
+      raise "#{element} has same ref_id as #{@elements_hash[element.ref_id]}" if @elements_hash[element.ref_id]
+      @elements_hash[element.ref_id] = element
+      instance_variable_get(instance_variable_name_for_element(element))[element.ref_id] = element
+    end
+
+    def delete_element(element)
+      @elements_hash.delete(element.classified_ref_id)
+      instance_variable_get(instance_variable_name_for_element(element)).delete(element.ref_id)
+    end
+
+    # Create methods for each element type that returns
+    # and array of elements instead of a hash
+    ELEMENT_NAMES.each do |element_name|
+      define_method(element_name) do
+        instance_variable_get(instance_variable_name_for_element_name(element_name)).values
+      end
+    end
+
+    #######
+    private
+    #######
+
+    def correct_bad_data
+      BrokenDataCorrector.run
+    end
+
+    def reunite_children_with_parents
+      elements.each do |element|
+        elements_hash[element.parent_ref_id].add_child(element) unless elements_hash[element.parent_ref_id].nil?
+      end
+    end
+
+    def get_class_from_element(xml_element)
+      hierarcy_level_description = xml_element.xpath('./StandardHierarchyLevel/description').first || xml_element.xpath('./StandardHierarchyLevel/Description').first
+      return "CommonCoreParser::#{hierarcy_level_description.text}".gsub(/\s/,'').constantize
+    end
+
+    def instance_variable_name_for_element(element)
+      instance_variable_name_for_element_name(element.class.name.sub(/^.*::/, '').underscore.pluralize)
+    end
+
+    def instance_variable_name_for_element_name(element_name)
+      "@#{element_name}_hash"
+    end
+  end
+end

data/lib/common_core_parser/version.rb

@@ -0,0 +1,3 @@
+module CommonCoreParser
+  VERSION = '0.0.1'
+end

data/lib/common_core_parser.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'rubygems'
+require 'bundler/setup'
+require 'nokogiri'
+require 'singleton'
+require 'active_support/core_ext'
+
+require File.expand_path('../common_core_parser', __FILE__)
+require File.expand_path('../common_core_parser/master',  __FILE__)
+require File.expand_path('../common_core_parser/broken_data_corrector',  __FILE__)
+
+require File.expand_path('../common_core_parser/element',  __FILE__)
+require File.expand_path('../common_core_parser/elements/component',  __FILE__)
+require File.expand_path('../common_core_parser/elements/standard',  __FILE__)
+require File.expand_path('../common_core_parser/elements/cluster',  __FILE__)
+require File.expand_path('../common_core_parser/elements/domain',  __FILE__)
+require File.expand_path('../common_core_parser/elements/subject_grade',  __FILE__)
+require File.expand_path('../common_core_parser/elements/standard_type',  __FILE__)
+
+module CommonCoreParser
+  DATA_PATH = File.expand_path('../../data', __FILE__)
+end
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+end

data/test/jenkins.bash

@@ -0,0 +1,3 @@
+#!/bin/bash
+bundle install
+bundle exec rake test

data/test/test_helper.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'simplecov'
+SimpleCov.start
+
+require File.expand_path('../../lib/common_core_parser', __FILE__)
+require 'active_support'
+require 'active_support/test_case'
+require 'test/unit'
+
+class ActiveSupport::TestCase
+  class << self
+    # test "verify something" do
+    #   ...
+    # end
+    def test(name, &block)
+      test_name = "test_#{name.gsub(/\s+/,'_')}".to_sym
+      defined = instance_method(test_name) rescue false
+      raise "#{test_name} is already defined in #{self}" if defined
+      if block_given?
+        define_method(test_name, &block)
+      else
+        define_method(test_name) do
+          flunk "No implementation provided for #{name}"
+        end
+      end
+    end
+  end
+end
+
+require 'mocha'

data/test/unit/domain_test.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'test_helper'
+
+module CommonCoreParser
+  class DomainTest < ActiveSupport::TestCase
+
+    # to_s
+    test "should return string representation of domain" do
+      assert_equal '<CommonCoreParser::Domain ref_id: 4246B738DA224018B6D20C12F9ED0073, parent_ref_id: C235350E091D437FBE2794CE93FBE949, code: Mathematics.1.G, statement: Geometry, grades: 01>', math_domain.to_s
+    end
+
+    # valid?
+    test "should return true for valid?" do
+      assert_equal true, math_domain.valid?
+    end
+
+    test "should return false for valid? when statement is blank" do
+      saved_math_domain = math_domain
+      saved_math_domain.stubs(:statement).returns('')
+      assert_equal true, saved_math_domain.statement.blank?
+      assert_equal false, saved_math_domain.valid?
+    end
+
+    test 'should return repdecessor ref id' do
+      assert_equal('C235350E091D437FBE2794CE93FBE949',math_domain.parent_ref_id)
+    end
+
+  private
+
+    def math_domain
+      Domain.new(math_domain_xml)
+    end
+
+    def math_domain_xml
+      @math_domain_xml ||= Nokogiri::XML(File.read(DATA_PATH+'/Mathematics/Grade1/Domain/Math_Grade1_G.xml')).xpath('//LearningStandardItem').first
+    end
+  end
+end