udise_school_report_reader 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: eb4e7f18e92a981045b328698bf04015f68aee7f22c9bc1a19b8ceab7c416eca
+  data.tar.gz: c0e734f6c22a5ad1ce6af4b2ee094ca140540500ebe75ba1594c6f1d6eb528bd
+SHA512:
+  metadata.gz: c453df46b297daef2ed774d838289e9ea41588d48f07db24964ca6fcd6cd918c8a99f789c7a89cb14bc96b8769091a1e2cb36183b90615a30cb814d7f16d586a
+  data.tar.gz: b9b2b0690d6f0d1a65f0f36e2d8a0a707860653332066f8591b5cdce573a5d5429f3971e57b7d889665398640282abfc6b543b335394a0d0d8e3faff7c740a19

data/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## [0.1.0] - 2024-01-01
+
+### Added
+- Initial release
+- Basic PDF parsing functionality
+- Data readers for various sections:
+  - Basic school information
+  - Teacher details
+  - Room information
+  - Location data
+  - Building information
+  - Sanitation facilities
+  - Digital facilities
+  - RTE compliance
+  - EWS details
+  - Enrollment data
+- HTML and YAML writers for data export
+- CSV export functionality 

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 UDISE Plus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. 

data/README.md

@@ -0,0 +1,45 @@
+# UDISE School Report Reader
+
+A Ruby gem for parsing and extracting data from UDISE (Unified District Information System for Education) school reports.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'udise_school_report_reader'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install udise_school_report_reader
+
+## Usage
+
+```ruby
+require 'udise_school_report_reader'
+
+# Initialize the parser with a PDF file
+parser = UdiseSchoolReportReader::SchoolReportParser.new('path/to/report.pdf')
+
+## Features
+
+- Parse UDISE school reports in PDF format
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/UDISE-Plus/udise-school-report-reader.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](LICENSE.txt). 

data/lib/udise_school_report_reader/activities_data_reader.rb

@@ -0,0 +1,58 @@
+class ActivitiesDataReader
+  def self.read(lines)
+    data = {
+      'activities' => {
+        'inspections' => {},
+        'instructional_days' => {},
+        'student_hours' => {},
+        'teacher_hours' => {}
+      }
+    }
+
+    lines.each_with_index do |line, i|
+      next_line = lines[i + 1]&.strip
+
+      case line
+      when "Acad. Inspections"
+        data['activities']['inspections']['academic'] = next_line.to_i if next_line =~ /^\d+$/
+      when "CRC Coordinator"
+        data['activities']['inspections']['crc_coordinator'] = next_line.to_i if next_line =~ /^\d+$/
+      when "Block Level Officers"
+        data['activities']['inspections']['block_level'] = next_line.to_i if next_line =~ /^\d+$/
+      when "State/District Officers"
+        data['activities']['inspections']['state_district'] = next_line.to_i if next_line =~ /^\d+$/
+      when "Instructional days"
+        if lines[i + 1] =~ /^\d+$/
+          data['activities']['instructional_days']['primary'] = lines[i + 1].to_i
+          data['activities']['instructional_days']['upper_primary'] = lines[i + 2].to_i if lines[i + 2] =~ /^\d+$/
+          data['activities']['instructional_days']['secondary'] = lines[i + 3].to_i if lines[i + 3] =~ /^\d+$/
+          data['activities']['instructional_days']['higher_secondary'] = lines[i + 4].to_i if lines[i + 4] =~ /^\d+$/
+        end
+      when "Avg.School hrs.Std."
+        if lines[i + 1] =~ /^\d+\.?\d*$/
+          data['activities']['student_hours']['primary'] = lines[i + 1].to_f
+          data['activities']['student_hours']['upper_primary'] = lines[i + 2].to_f if lines[i + 2] =~ /^\d+\.?\d*$/
+          data['activities']['student_hours']['secondary'] = lines[i + 3].to_f if lines[i + 3] =~ /^\d+\.?\d*$/
+          data['activities']['student_hours']['higher_secondary'] = lines[i + 4].to_f if lines[i + 4] =~ /^\d+\.?\d*$/
+        end
+      when "Avg.School hrs.Tch."
+        if lines[i + 1] =~ /^\d+\.?\d*$/
+          data['activities']['teacher_hours']['primary'] = lines[i + 1].to_f
+          data['activities']['teacher_hours']['upper_primary'] = lines[i + 2].to_f if lines[i + 2] =~ /^\d+\.?\d*$/
+          data['activities']['teacher_hours']['secondary'] = lines[i + 3].to_f if lines[i + 3] =~ /^\d+\.?\d*$/
+          data['activities']['teacher_hours']['higher_secondary'] = lines[i + 4].to_f if lines[i + 4] =~ /^\d+\.?\d*$/
+        end
+      end
+    end
+
+    # Clean up empty sections
+    data['activities']['inspections'].reject! { |_, v| v.nil? }
+    data['activities']['instructional_days'].reject! { |_, v| v.nil? }
+    data['activities']['student_hours'].reject! { |_, v| v.nil? }
+    data['activities']['teacher_hours'].reject! { |_, v| v.nil? }
+    data['activities'].reject! { |_, v| v.empty? }
+    data.reject! { |_, v| v.empty? }
+
+    data
+  end
+end 

data/lib/udise_school_report_reader/anganwadi_data_reader.rb

@@ -0,0 +1,22 @@
+require_relative 'data_reader_base'
+
+class AnganwadiDataReader
+  include DataReaderBase
+
+  FIELD_MAPPINGS = {
+    'Anganwadi At Premises' => {
+      key_path: ['anganwadi', 'at_premises']
+    },
+    'Anganwadi Boys' => {
+      key_path: ['anganwadi', 'boys'],
+      value_type: :integer
+    },
+    'Anganwadi Girls' => {
+      key_path: ['anganwadi', 'girls'],
+      value_type: :integer
+    },
+    'Anganwadi Worker' => {
+      key_path: ['anganwadi', 'worker']
+    }
+  }
+end 

data/lib/udise_school_report_reader/basic_info_data_reader.rb

@@ -0,0 +1,29 @@
+class BasicInfoDataReader
+  def self.read(lines)
+    require 'yaml'
+    template = YAML.load_file('template.yml')
+    data = { 'basic_info' => template['basic_info'] }
+
+    lines.each_with_index do |line, i|
+      next_line = lines[i + 1]&.strip
+
+      case line
+      when "UDISE CODE"
+        if next_line && (match = next_line.match(/(\d{2})\s*(\d{2})\s*(\d{2})\s*(\d{2})\s*(\d{3})/))
+          data['basic_info']['udise_code'] = match[1..5].join('')
+        end
+      when "School Name"
+        if next_line && next_line.include?("CARMEL")
+          data['basic_info']['name'] = next_line
+        end
+      when /Academic Year.*:\s*(\d{4}-\d{2})/
+        data['basic_info']['academic_year'] = $1
+      end
+    end
+
+    # Clean up empty sections
+    data['basic_info'].reject! { |_, v| v.nil? || (v.is_a?(Hash) && v.empty?) }
+
+    data
+  end
+end 

data/lib/udise_school_report_reader/block_rectangle_combiner.rb

@@ -0,0 +1,115 @@
+class BlockRectangleCombiner
+  def self.combine(blocks, rects)
+    # Filter out blocks with missing coordinates
+    invalid_blocks = blocks.reject { |block| block[:x] && block[:y] || block[:text].to_s.empty? }
+    if invalid_blocks.any?
+      warn "Warning: Found #{invalid_blocks.size} non-empty blocks with missing coordinates"
+      invalid_blocks.each do |block|
+        warn "  - Page #{block[:page]}: '#{block[:text]}'"
+      end
+    end
+    valid_blocks = blocks.select { |block| block[:x] && block[:y] }
+    
+    # Filter out rectangles with missing or invalid coordinates
+    invalid_rects = rects.reject { |rect| rect[:x] && rect[:y] && rect[:width] && rect[:height] && rect[:width] > 0 && rect[:height] > 0 }
+    if invalid_rects.any?
+      warn "Warning: Found #{invalid_rects.size} rectangles with invalid coordinates"
+      invalid_rects.each do |rect|
+        warn "  - Page #{rect[:page]}: x=#{rect[:x]}, y=#{rect[:y]}, w=#{rect[:width]}, h=#{rect[:height]}"
+      end
+    end
+    valid_rects = rects.select { |rect| rect[:x] && rect[:y] && rect[:width] && rect[:height] && rect[:width] > 0 && rect[:height] > 0 }
+    
+    # For each text block, find its smallest containing rectangle
+    combined_data = valid_blocks.map do |block|
+      # Find rectangles on the same page that contain this text block
+      containing_rects = valid_rects.select do |rect|
+        rect[:page] == block[:page] &&
+        block[:x] >= rect[:x] &&
+        block[:x] <= (rect[:x] + rect[:width]) &&
+        block[:y] >= rect[:y] &&
+        block[:y] <= (rect[:y] + rect[:height])
+      end
+
+      # Find the smallest containing rectangle by area
+      smallest_rect = containing_rects.min_by { |r| r[:width] * r[:height] }
+
+      # Create entry with block data and rectangle data (if found)
+      if smallest_rect
+        {
+          page: block[:page],
+          text: block[:text],
+          text_x: block[:x],
+          text_y: block[:y],
+          font: block[:font],
+          font_size: block[:font_size],
+          rect_x: smallest_rect[:x],
+          rect_y: smallest_rect[:y],
+          rect_width: smallest_rect[:width],
+          rect_height: smallest_rect[:height],
+          stroke_color: smallest_rect[:stroke_color],
+          fill_color: smallest_rect[:fill_color],
+          line_width: smallest_rect[:line_width]
+        }
+      else
+        {
+          page: block[:page],
+          text: block[:text],
+          text_x: block[:x],
+          text_y: block[:y],
+          font: block[:font],
+          font_size: block[:font_size],
+          rect_x: nil,
+          rect_y: nil,
+          rect_width: nil,
+          rect_height: nil,
+          stroke_color: nil,
+          fill_color: nil,
+          line_width: nil
+        }
+      end
+    end
+
+    # Add empty rectangles that don't contain any text
+    valid_rects.each do |rect|
+      # Check if this rectangle contains any text blocks
+      has_text = valid_blocks.any? do |block|
+        block[:page] == rect[:page] &&
+        block[:x] >= rect[:x] &&
+        block[:x] <= (rect[:x] + rect[:width]) &&
+        block[:y] >= rect[:y] &&
+        block[:y] <= (rect[:y] + rect[:height])
+      end
+
+      # If it doesn't contain text, add it as an empty entry
+      unless has_text
+        combined_data << {
+          page: rect[:page],
+          text: "",
+          text_x: nil,
+          text_y: nil,
+          font: nil,
+          font_size: nil,
+          rect_x: rect[:x],
+          rect_y: rect[:y],
+          rect_width: rect[:width],
+          rect_height: rect[:height],
+          stroke_color: rect[:stroke_color],
+          fill_color: rect[:fill_color],
+          line_width: rect[:line_width]
+        }
+      end
+    end
+
+    # Sort by page, then y (top to bottom), then x (left to right)
+    combined_data.sort_by! do |data|
+      [
+        data[:page],
+        -(data[:rect_y] || data[:text_y] || 0),  # Negative for top-to-bottom
+        data[:rect_x] || data[:text_x] || 0
+      ]
+    end
+
+    combined_data
+  end
+end 

data/lib/udise_school_report_reader/building_data_reader.rb

@@ -0,0 +1,36 @@
+require_relative 'data_reader_base'
+
+class BuildingDataReader
+  include DataReaderBase
+
+  FIELD_MAPPINGS = {
+    'Building Status' => {
+      key_path: ['building', 'status'],
+      end_pattern: /Boundary/
+    },
+    'Boundary wall' => {
+      key_path: ['building', 'boundary_wall'],
+      end_pattern: /No\.of/
+    },
+    'No.of Building Blocks' => {
+      key_path: ['building', 'blocks'],
+      value_type: :integer
+    },
+    'Pucca Building Blocks' => {
+      key_path: ['building', 'pucca_blocks'],
+      value_type: :integer
+    },
+    'Is this a Shift School?' => {
+      key_path: ['building', 'has_shifts'],
+      end_pattern: /Building/
+    },
+    'Availability of Ramps' => {
+      key_path: ['building', 'accessibility', 'ramps'],
+      end_pattern: /Availability of Hand/
+    },
+    'Availability of Handrails' => {
+      key_path: ['building', 'accessibility', 'handrails'],
+      end_pattern: /Anganwadi/
+    }
+  }
+end 

data/lib/udise_school_report_reader/characteristics_reader.rb

@@ -0,0 +1,28 @@
+require_relative 'data_reader_base'
+
+class CharacteristicsReader
+  include DataReaderBase
+
+  FIELD_MAPPINGS = {
+    'School Category' => {
+      key_path: ['characteristics', 'category'],
+      end_pattern: /Management/
+    },
+    'School Type' => {
+      key_path: ['characteristics', 'type'],
+      end_pattern: /Lowest/
+    },
+    'Lowest & Highest Class' => {
+      key_path: ['characteristics', 'class_range'],
+      end_pattern: /Pre Primary/
+    },
+    'Pre Primary' => {
+      key_path: ['characteristics', 'pre_primary'],
+      end_pattern: /Medium/
+    },
+    'Is Special School for CWSN?' => {
+      key_path: ['characteristics', 'is_special_school'],
+      end_pattern: /Availability/
+    }
+  }
+end 

data/lib/udise_school_report_reader/csv_writer.rb

@@ -0,0 +1,75 @@
+require 'csv'
+
+class CSVWriter
+  def self.write_blocks(blocks, csv_path)
+    CSV.open(csv_path, 'wb') do |csv|
+      csv << ['page', 'x', 'y', 'text', 'font', 'font_size']
+      blocks.each do |block|
+        csv << [
+          block[:page],
+          block[:x],
+          block[:y],
+          block[:text],
+          block[:font],
+          block[:font_size]
+        ]
+      end
+    end
+  end
+
+  def self.write_rectangles(rectangles, csv_path)
+    CSV.open(csv_path, 'wb') do |csv|
+      csv << ['page', 'x', 'y', 'width', 'height', 'stroke_color', 'fill_color', 'line_width']
+      rectangles.each do |rect|
+        csv << [
+          rect[:page],
+          rect[:x],
+          rect[:y],
+          rect[:width],
+          rect[:height],
+          rect[:stroke_color],
+          rect[:fill_color],
+          rect[:line_width]
+        ]
+      end
+    end
+  end
+
+  def self.write_combined(combined_data, output_path)
+    CSV.open(output_path, 'wb') do |csv|
+      csv << [
+        'page',
+        'text',
+        'text_x',
+        'text_y',
+        'font',
+        'font_size',
+        'rect_x',
+        'rect_y',
+        'rect_width',
+        'rect_height',
+        'stroke_color',
+        'fill_color',
+        'line_width'
+      ]
+
+      combined_data.each do |data|
+        csv << [
+          data[:page],
+          data[:text],
+          data[:text_x],
+          data[:text_y],
+          data[:font],
+          data[:font_size],
+          data[:rect_x],
+          data[:rect_y],
+          data[:rect_width],
+          data[:rect_height],
+          data[:stroke_color],
+          data[:fill_color],
+          data[:line_width]
+        ]
+      end
+    end
+  end
+end 

data/lib/udise_school_report_reader/data_reader_base.rb

@@ -0,0 +1,86 @@
+module DataReaderBase
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def read(lines)
+      require 'yaml'
+      template = YAML.load_file('template.yml')
+      data = { base_key => template[base_key] }
+
+      lines.each_with_index do |line, i|
+        next_line = lines[i + 1]&.strip
+        
+        if mapping = self::FIELD_MAPPINGS[line]
+          if mapping[:is_table]
+            process_table_data(data, lines[i+1..i+20], mapping)
+          else
+            process_simple_field(data, next_line, mapping)
+          end
+        end
+      end
+
+      cleanup_data(data)
+      data
+    end
+
+    private
+
+    def process_table_data(data, table_lines, mapping)
+      mapping[:table_config][:sections].each do |section|
+        idx = table_lines.index { |l| l.match?(section[:trigger]) }
+        if idx && idx + section[:offset] + section[:fields].length <= table_lines.length
+          section[:fields].each_with_index do |field, field_idx|
+            value = table_lines[idx + section[:offset] + field_idx]
+            if value =~ /^\d+$/
+              set_value_at_path(data, mapping[:key_path] + field[:key], transform_value(value, field[:value_type]))
+            end
+          end
+        end
+      end
+    end
+
+    def process_simple_field(data, next_line, mapping)
+      return unless next_line
+      return if mapping[:end_pattern] && next_line.match?(mapping[:end_pattern])
+
+      value = transform_value(next_line, mapping[:value_type])
+      set_value_at_path(data, mapping[:key_path], value)
+    end
+
+    def transform_value(value, type)
+      case type
+      when :integer
+        value.to_i if value =~ /^\d+$/
+      else
+        value
+      end
+    end
+
+    def set_value_at_path(data, path, value)
+      current = data
+      path[0..-2].each do |key|
+        current[key] ||= {}
+        current = current[key]
+      end
+      current[path.last] = value
+    end
+
+    def cleanup_data(data)
+      base = data[base_key]
+      return unless base.is_a?(Hash)
+
+      base.each do |_, section|
+        if section.is_a?(Hash)
+          section.reject! { |_, v| v.nil? || (v.is_a?(Hash) && v.empty?) }
+        end
+      end
+      base.reject! { |_, v| v.nil? || (v.is_a?(Hash) && v.empty?) }
+    end
+
+    def base_key
+      self.name.gsub('DataReader', '').downcase
+    end
+  end
+end 

data/lib/udise_school_report_reader/digital_facilities_data_reader.rb

@@ -0,0 +1,42 @@
+class DigitalFacilitiesDataReader
+  def self.read(lines)
+    data = { 'digital_facilities' => YAML.load_file('template.yml')['digital_facilities'] }
+
+    lines.each_with_index do |line, i|
+      case line
+      when /Digital/
+        data['digital_facilities']['ict_lab'] = lines[i + 2] if lines[i + 2] =~ /^[12]-/
+        data['digital_facilities']['internet'] = lines[i + 4] if lines[i + 4] =~ /^[12]-/
+        if lines[i + 6] =~ /^\d+$/
+          data['digital_facilities']['computers']['desktop'] = lines[i + 6].to_i
+        end
+        if lines[i + 8] =~ /^\d+$/
+          data['digital_facilities']['computers']['laptop'] = lines[i + 8].to_i
+        end
+        if lines[i + 10] =~ /^\d+$/
+          data['digital_facilities']['computers']['tablet'] = lines[i + 10].to_i
+        end
+        if lines[i + 12] =~ /^\d+$/
+          data['digital_facilities']['peripherals']['printer'] = lines[i + 12].to_i
+        end
+        if lines[i + 14] =~ /^\d+$/
+          data['digital_facilities']['peripherals']['projector'] = lines[i + 14].to_i
+        end
+        data['digital_facilities']['smart_classroom']['dth'] = lines[i + 16] if lines[i + 16] =~ /^[12]-/
+        if lines[i + 18] =~ /^\d+$/
+          data['digital_facilities']['smart_classroom']['digiboard'] = lines[i + 18].to_i
+        end
+      end
+    end
+
+    # Clean up empty sections
+    data['digital_facilities'].each do |key, section|
+      if section.is_a?(Hash)
+        section.reject! { |_, v| v.nil? || (v.is_a?(Hash) && v.empty?) }
+      end
+    end
+    data['digital_facilities'].reject! { |_, v| v.nil? || (v.is_a?(Hash) && v.empty?) }
+
+    data unless data['digital_facilities'].empty?
+  end
+end