udise_school_report_reader 0.1.0

data/lib/udise_school_report_reader/teacher_data_reader.rb

@@ -0,0 +1,204 @@
+require_relative 'data_reader_base'
+
+class TeacherDataReader
+  include DataReaderBase
+
+  FIELD_MAPPINGS = {
+    'Regular' => {
+      key_path: ['teachers', 'count_by_level', 'regular'],
+      value_type: :integer
+    },
+    'Part-time' => {
+      key_path: ['teachers', 'count_by_level', 'part_time'],
+      value_type: :integer
+    },
+    'Contract' => {
+      key_path: ['teachers', 'count_by_level', 'contract'],
+      value_type: :integer
+    },
+    'Male' => {
+      key_path: ['teachers', 'demographics', 'male'],
+      value_type: :integer
+    },
+    'Female' => {
+      key_path: ['teachers', 'demographics', 'female'],
+      value_type: :integer
+    },
+    'Transgender' => {
+      key_path: ['teachers', 'demographics', 'transgender'],
+      value_type: :integer
+    },
+    'Below Graduate' => {
+      key_path: ['teachers', 'qualifications', 'academic', 'below_graduate'],
+      value_type: :integer
+    },
+    'Graduate' => {
+      key_path: ['teachers', 'qualifications', 'academic', 'graduate'],
+      value_type: :integer
+    },
+    'Post Graduate and Above' => {
+      key_path: ['teachers', 'qualifications', 'academic', 'post_graduate_and_above'],
+      value_type: :integer
+    },
+    'B.Ed. or Equivalent' => {
+      key_path: ['teachers', 'qualifications', 'professional', 'bed'],
+      value_type: :integer
+    },
+    'M.Ed. or Equivalent' => {
+      key_path: ['teachers', 'qualifications', 'professional', 'med'],
+      value_type: :integer
+    },
+    'Diploma or Certificate in basic teachers training' => {
+      key_path: ['teachers', 'qualifications', 'professional', 'basic_training'],
+      value_type: :integer
+    },
+    'Bachelor of Elementary Education (B.El.Ed.)' => {
+      key_path: ['teachers', 'qualifications', 'professional', 'beled'],
+      value_type: :integer
+    },
+    'Diploma/degree in special Education' => {
+      key_path: ['teachers', 'qualifications', 'professional', 'special_education'],
+      value_type: :integer
+    },
+    'Teachers Aged above 55' => {
+      key_path: ['teachers', 'age_distribution', 'above_55'],
+      value_type: :integer
+    },
+    'Total Teacher Trained in Computer' => {
+      key_path: ['teachers', 'training', 'computer_trained'],
+      value_type: :integer
+    },
+    'No. of Total Teacher Received Service Training' => {
+      key_path: ['teachers', 'training', 'service', 'total'],
+      value_type: :integer
+    },
+    'Special Training Received' => {
+      key_path: ['teachers', 'training', 'special', 'received'],
+      value_type: :string
+    },
+    'Teaching Hours per Week' => {
+      key_path: ['teachers', 'workload', 'teaching_hours', 'per_week'],
+      value_type: :integer,
+      extract_pattern: /(\d+)/
+    },
+    'Non-Teaching Hours' => {
+      key_path: ['teachers', 'workload', 'non_teaching_hours', 'per_week'],
+      value_type: :integer,
+      extract_pattern: /(\d+)/
+    },
+    'Total Teacher Involve in Non Teaching Assignment' => {
+      key_path: ['teachers', 'assignments', 'non_teaching'],
+      value_type: :integer,
+      extract_pattern: /^(\d+)$/
+    },
+    'Subject:' => {
+      key_path: ['teachers', 'workload', 'by_subject'],
+      value_type: :integer,
+      extract_pattern: /^Subject:\s*(.+?)(?:\s*,\s*Teachers:\s*(\d+))?$/,
+      dynamic_key: true,
+      key_from_match: 1,
+      value_from_match: 2,
+      key_transform: :downcase
+    }
+  }
+
+  def self.read(lines)
+    require 'yaml'
+    template = YAML.load_file('template.yml')
+    data = { 'teachers' => template['teachers'] }
+
+    # Process base module mappings first
+    base_data = super
+    if base_data&.dig('teachers')
+      data['teachers']['count_by_level'] = base_data['teachers']['count_by_level'] if base_data['teachers']['count_by_level']
+      data['teachers']['demographics'] = base_data['teachers']['demographics'] if base_data['teachers']['demographics']
+      data['teachers']['age_distribution'] = base_data['teachers']['age_distribution'] if base_data['teachers']['age_distribution']
+      
+      # Handle nested structures
+      data['teachers']['assignments'] ||= {}
+      data['teachers']['assignments']['non_teaching'] = base_data['teachers']['assignments']['non_teaching'] if base_data['teachers']['assignments']&.dig('non_teaching')
+
+      if base_data['teachers']['qualifications']
+        data['teachers']['qualifications'] ||= {}
+        data['teachers']['qualifications']['academic'] = base_data['teachers']['qualifications']['academic'] if base_data['teachers']['qualifications']['academic']
+        data['teachers']['qualifications']['professional'] = base_data['teachers']['qualifications']['professional'] if base_data['teachers']['qualifications']['professional']
+      end
+
+      if base_data['teachers']['training']
+        data['teachers']['training'] ||= {}
+        data['teachers']['training']['computer_trained'] = base_data['teachers']['training']['computer_trained'] if base_data['teachers']['training']['computer_trained']
+        data['teachers']['training']['service'] ||= {}
+        data['teachers']['training']['service']['total'] = base_data['teachers']['training']['service']['total'] if base_data['teachers']['training']['service']&.dig('total')
+        data['teachers']['training']['special'] ||= {}
+        data['teachers']['training']['special']['received'] = base_data['teachers']['training']['special']['received'] if base_data['teachers']['training']['special']&.dig('received')
+      end
+
+      if base_data['teachers']['workload']
+        data['teachers']['workload'] ||= {}
+        data['teachers']['workload']['teaching_hours'] = base_data['teachers']['workload']['teaching_hours'] if base_data['teachers']['workload']['teaching_hours']
+        data['teachers']['workload']['non_teaching_hours'] = base_data['teachers']['workload']['non_teaching_hours'] if base_data['teachers']['workload']['non_teaching_hours']
+        data['teachers']['workload']['by_subject'] = base_data['teachers']['workload']['by_subject'] if base_data['teachers']['workload']['by_subject']
+      end
+    end
+
+    lines.each_with_index do |line, i|
+      next_line = lines[i + 1]&.strip
+
+      case line
+      when /^(\d+)-(.+)$/
+        category = $2.strip
+        if next_line =~ /^\d+$/
+          key = case category
+          when 'Primary'
+            'primary'
+          when 'Up.Pr.'
+            'upper_primary'
+          when 'Pr. & Up.Pr.'
+            'primary_and_upper_primary'
+          when 'Sec. only'
+            'secondary_only'
+          when 'H Sec only.'
+            'higher_secondary_only'
+          when 'Up pri and Sec.'
+            'upper_primary_and_secondary'
+          when 'Sec and H Sec'
+            'secondary_and_higher_secondary'
+          when 'Pre-Primary Only.'
+            'pre_primary_only'
+          when 'Pre- Pri & Pri'
+            'pre_primary_and_primary'
+          else
+            category.downcase.gsub(/[^a-z0-9]+/, '_').gsub(/^_|_$/, '')
+          end
+
+          data['teachers']['classes_taught'][key] = next_line.to_i
+        end
+
+      when "Other"
+        if next_line =~ /^\d+$/
+          data['teachers']['qualifications']['professional']['other'] = next_line.to_i
+        end
+      when "None"
+        if next_line =~ /^\d+$/
+          data['teachers']['qualifications']['professional']['none'] = next_line.to_i
+        end
+      when "Pursuing any Relevant Professional Course"
+        if next_line =~ /^\d+$/
+          data['teachers']['qualifications']['professional']['pursuing_course'] = next_line.to_i
+        end
+      end
+    end
+
+    # Clean up empty sections
+    data['teachers'].each do |key, section|
+      if section.is_a?(Hash)
+        # Don't clean up assignments section
+        next if key == 'assignments'
+        section.reject! { |_, v| v.nil? || (v.is_a?(Hash) && v.empty?) }
+      end
+    end
+    data['teachers'].reject! { |k, v| v.nil? || (v.is_a?(Hash) && v.empty? && k != 'assignments') }
+
+    data
+  end
+end 

data/lib/udise_school_report_reader/version.rb

@@ -0,0 +1,3 @@
+module UdiseSchoolReportReader
+  VERSION = "0.1.0"
+end 

data/lib/udise_school_report_reader.rb

@@ -0,0 +1,41 @@
+require "udise_school_report_reader/version"
+
+# Core functionality
+require_relative "udise_school_report_reader/pdf_block_extractor"
+require_relative "udise_school_report_reader/pdf_rectangle_extractor"
+require_relative "udise_school_report_reader/pdf_content_compressor"
+require_relative "udise_school_report_reader/block_rectangle_combiner"
+
+# Data readers
+require_relative "udise_school_report_reader/data_reader_base"
+require_relative "udise_school_report_reader/activities_data_reader"
+require_relative "udise_school_report_reader/anganwadi_data_reader"
+require_relative "udise_school_report_reader/basic_info_data_reader"
+require_relative "udise_school_report_reader/building_data_reader"
+require_relative "udise_school_report_reader/characteristics_reader"
+require_relative "udise_school_report_reader/digital_facilities_data_reader"
+require_relative "udise_school_report_reader/enrollment_data_reader"
+require_relative "udise_school_report_reader/ews_data_reader"
+require_relative "udise_school_report_reader/location_data_reader"
+require_relative "udise_school_report_reader/official_data_reader"
+require_relative "udise_school_report_reader/rooms_data_reader"
+require_relative "udise_school_report_reader/rte_data_reader"
+require_relative "udise_school_report_reader/sanitation_data_reader"
+require_relative "udise_school_report_reader/teacher_data_reader"
+
+# Writers
+require_relative "udise_school_report_reader/csv_writer"
+require_relative "udise_school_report_reader/enrollment_html_writer"
+require_relative "udise_school_report_reader/enrollment_yaml_writer"
+require_relative "udise_school_report_reader/ews_html_writer"
+require_relative "udise_school_report_reader/ews_yaml_writer"
+require_relative "udise_school_report_reader/rte_html_writer"
+require_relative "udise_school_report_reader/rte_yaml_writer"
+
+# Main parser
+require_relative "udise_school_report_reader/school_report_parser"
+
+module UdiseSchoolReportReader
+  class Error < StandardError; end
+  # Your code goes here...
+end 

data/test/school_report_parser_test.rb

@@ -0,0 +1,62 @@
+require 'minitest/autorun'
+require 'fileutils'
+require 'yaml'
+require 'udise_school_report_reader'
+
+class SchoolReportParserTest < Minitest::Test
+  def setup
+    @schools = ['carmel-2223', 'jhita-2223', 'kachora-2223', 'sarvo-2223']
+  end
+
+  def test_yaml_generation
+    @schools.each do |school|
+      pdf_path = File.join(File.dirname(__dir__), "samples/#{school}/#{school}.pdf")
+      yaml_path = File.join(File.dirname(__dir__), "samples/#{school}/#{school}.yml")
+      
+      # Skip if PDF doesn't exist
+      next unless File.exist?(pdf_path)
+      next unless File.exist?(yaml_path)  # Skip if no benchmark YAML exists
+      
+      # Generate new YAML data without writing files
+      actual_data = UdiseSchoolReportReader::SchoolReportParser.extract_to_text(pdf_path)
+      
+      # Compare with benchmark
+      expected = YAML.load_file(yaml_path)
+      compare_nested_hashes(expected, actual_data, [], school)
+    end
+    
+    assert true, "All schools processed successfully"
+  end
+
+  private
+
+  def compare_nested_hashes(expected, actual, path = [], school)
+    return if expected == actual
+
+    if expected.is_a?(Hash) && actual.is_a?(Hash)
+      # Check for missing keys
+      (expected.keys - actual.keys).each do |key|
+        flunk "Missing key '#{(path + [key]).join('.')}' in #{school}"
+      end
+
+      # Check for extra keys
+      (actual.keys - expected.keys).each do |key|
+        flunk "Extra key '#{(path + [key]).join('.')}' in #{school}"
+      end
+
+      # Compare values for common keys
+      (expected.keys & actual.keys).each do |key|
+        compare_nested_hashes(expected[key], actual[key], path + [key], school)
+      end
+    elsif expected.is_a?(Array) && actual.is_a?(Array)
+      if expected.length != actual.length
+        flunk "Array length mismatch at '#{path.join('.')}' in #{school}. Expected #{expected.length}, got #{actual.length}"
+      end
+      expected.zip(actual).each_with_index do |(exp_item, act_item), idx|
+        compare_nested_hashes(exp_item, act_item, path + [idx], school)
+      end
+    else
+      assert_equal expected, actual, "Value mismatch at '#{path.join('.')}' in #{school}. Expected #{expected.inspect}, got #{actual.inspect}"
+    end
+  end
+end 

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: udise_school_report_reader
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Syed Fazil Basheer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-01-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: This gem provides functionality to read and parse UDISE school reports,
+  extracting various data points including teacher information, room details, location
+  data, and more.
+email:
+- fazil@fazn.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/udise_school_report_reader.rb
+- lib/udise_school_report_reader/activities_data_reader.rb
+- lib/udise_school_report_reader/anganwadi_data_reader.rb
+- lib/udise_school_report_reader/basic_info_data_reader.rb
+- lib/udise_school_report_reader/block_rectangle_combiner.rb
+- lib/udise_school_report_reader/building_data_reader.rb
+- lib/udise_school_report_reader/characteristics_reader.rb
+- lib/udise_school_report_reader/csv_writer.rb
+- lib/udise_school_report_reader/data_reader_base.rb
+- lib/udise_school_report_reader/digital_facilities_data_reader.rb
+- lib/udise_school_report_reader/enrollment_data_reader.rb
+- lib/udise_school_report_reader/enrollment_html_writer.rb
+- lib/udise_school_report_reader/enrollment_yaml_writer.rb
+- lib/udise_school_report_reader/ews_data_reader.rb
+- lib/udise_school_report_reader/ews_html_writer.rb
+- lib/udise_school_report_reader/ews_yaml_writer.rb
+- lib/udise_school_report_reader/location_data_reader.rb
+- lib/udise_school_report_reader/official_data_reader.rb
+- lib/udise_school_report_reader/pdf_block_extractor.rb
+- lib/udise_school_report_reader/pdf_content_compressor.rb
+- lib/udise_school_report_reader/pdf_rectangle_extractor.rb
+- lib/udise_school_report_reader/rooms_data_reader.rb
+- lib/udise_school_report_reader/rte_data_reader.rb
+- lib/udise_school_report_reader/rte_html_writer.rb
+- lib/udise_school_report_reader/rte_yaml_writer.rb
+- lib/udise_school_report_reader/sanitation_data_reader.rb
+- lib/udise_school_report_reader/school_report_parser.rb
+- lib/udise_school_report_reader/teacher_data_reader.rb
+- lib/udise_school_report_reader/version.rb
+- test/school_report_parser_test.rb
+homepage: https://github.com/UDISE-Plus/udise-school-report-reader
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/UDISE-Plus/udise-school-report-reader
+  source_code_uri: https://github.com/UDISE-Plus/udise-school-report-reader
+  changelog_uri: https://github.com/UDISE-Plus/udise-school-report-reader/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: A Ruby gem to parse and extract data from UDISE school reports
+test_files: []