rulethu_stock_exchange 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0dc271fa578eb41d6b0dd67f8cdc82eddc52a9168e0b8f366e66793ed4304af
-  data.tar.gz: '09806e50ef484e84efc7f05044844dc260c82e69368a24f8689e838ec909e884'
+  metadata.gz: 997032ba38a153377cf0576dce101b69ec777e48f58c42e5ec06b7b860222d5b
+  data.tar.gz: b1dc52bc0af36edb912a4af09892023c652c47461de0163d3c9c22544ae6fb59
 SHA512:
-  metadata.gz: 66394403e7cb1219331e9231c7060f0b9733a1c16f76d4f11c19d58c44f1e95570227836dd9ed168089a66892633c4e2afccb008d43f0c5ab616a32313c9689e
-  data.tar.gz: d8c01ffb3e7ebebeb91e31dd36dfd21e93ae29d7fe875f6fb5734a2a9674417558c0861f99fbbc57744969904194e84518be0baa01395937d9df54ff7a427fc0
+  metadata.gz: 7b91fed4ad022fbfb94b08992a4a62b9a80ba492b22fcee7345d87ca977c4f6def7c124c1799c06f0b956c101ed96591cc0b14e13ef7bcbb63315c77fa5c5360
+  data.tar.gz: 8ccb363b6bdec459d6125f6c3c8be366c5ee5ad0a233e5ddad9599b00291fdf615498697382eb3f7b7e1c3e03442f4cb6aac8f7378b3aa4324b0eeb32bfb5572

data/README.md

@@ -1,10 +1,10 @@
 # RulethuStockExchange
 
 This gem allows you to scrape stock exchange data from the following exchanges:
-* Zimbabwe Stock Exchange (ZSE)
-* Johannesburg Stock Exchange (JSE)
-* London Stock Exchange (LSE)
 
+- Zimbabwe Stock Exchange (ZSE)
+- Johannesburg Stock Exchange (JSE)
+- London Stock Exchange (LSE)
 
 ## Installation
 
@@ -18,8 +18,12 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-### Example 1: Scraping Data From London Stock Exchange 
+** Make sure you have firefox browser installed **
+
+### Example 1: Scraping Data From London Stock Exchange
+
 Require the gem
+
 ```ruby
 require 'rulethu_stock_exchange'
 ```
@@ -29,32 +33,39 @@ Get the stock exchange. (Currently available stock exchanges are ZSE, JSE and LS
 ```ruby
 exchange = RulethuStockExchange::Exchanges::LSE
 ```
+
 Create a Scraper passing in the url for the exchangew
+
 ```ruby
 scraper = RulethuStockExchange::Scraper.new exchange[:url]
 ```
 
 Scrape the data passing in a selector
+
 ```ruby
 html = scraper.scrape exchange[:selector]
 ```
 
 Create a parser to parse the data
+
 ```ruby
-parser = RulethuStockExchange::Parser.new html 
+parser = RulethuStockExchange::Parser.new html
 ```
 
 Parse the data
+
 ```ruby
-data = parser.parse() 
+data = parser.parse()
 ```
 
 Write the data to JSON file
+
 ```ruby
 filename = RulethuStockExchange::IO.write_to_json_file data, 'lse'
 ```
 
 Convert the JSON file to CSV
+
 ```ruby
 RulethuStockExchange::IO.json_to_csv filename
 ```

data/lib/rulethu_stock_exchange/io.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
-require 'json'
-require 'csv'
-require 'time'
+require "json"
+require "csv"
+require "time"
 
 module RulethuStockExchange
   class IO
@@ -10,8 +10,8 @@ module RulethuStockExchange
       date = Time.now.strftime("%d-%m-%Y")
       data_dir = Pathname.new("data") / Pathname.new(prefix)
       data_dir.mkpath
-      filename = "#{data_dir}/#{date}-data.json"
-      File.open(filename, 'w') do |f|
+      filename = "#{data_dir}/#{prefix}-#{date}-data.json"
+      File.open(filename, "w") do |f|
         f.write(JSON.dump(data))
       end
       filename
@@ -20,13 +20,13 @@ module RulethuStockExchange
     def self.json_to_csv(json_file)
       puts "\nConverting #{json_file} to CSV"
       data = Object.new
-      File.open(json_file, 'r') do |f|
+      File.open(json_file, "r") do |f|
         data = JSON.parse(f.read)
       end
 
-      filename = json_file.sub(/\.json$/, '.csv')
+      filename = json_file.sub(/\.json$/, ".csv")
 
-      CSV.open(filename, 'w') do |csv|
+      CSV.open(filename, "w") do |csv|
         csv << data.first.keys
         data.each do |row|
           csv << row.values

data/lib/rulethu_stock_exchange/parser.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require 'selenium-webdriver'
+require "selenium-webdriver"
 
 module RulethuStockExchange
   class Parser
@@ -10,39 +10,43 @@ module RulethuStockExchange
     end
 
     def parse_zse
-      tbody = table.find_element(:tag_name, 'tbody')
-      rows = tbody.find_elements(:tag_name, 'tr')
-      body_data = []
-      puts "\nWriting body data"
+      tbody = table.find_element(:tag_name, "tbody")
+      rows = tbody.find_elements(:tag_name, "tr")
+      data_rows = []
       rows.each do |row|
-        cells = row.find_elements(:tag_name, 'td')
-        cells.each do |cell|
-          body_data << cell.text.strip
+        text = row.find_elements(:tag_name, "td").first.text.strip
+        unless text&.upcase == "EQUITIES"
+          data_rows << row unless text.empty?
         end
       end
-      data = body_data.reject(&:empty?)
-      header_data = data[1..4].map { |d| d.gsub("\n", " ") }
-      remaining_data = data[5..]
-      grouped_data = remaining_data.each_slice(4).to_a
-      data = grouped_data.reject { |row| row.first.match?(/^[-+]?\d+(\.\d+)?$/) }
-      data = data[0...-1]
-      output = []
-      data.each do |row|
-        result = {}
-        row.each_with_index do |value, index|
-          result[header_data[index]] = value
+
+      header_data = []
+      data_rows.first.find_elements(:tag_name, "td").each do |td|
+        text = td.text.strip.gsub("\n", " ") || "#"
+        text = "#" if text.empty?
+        header_data << text
+      end
+
+      data_rows = data_rows.slice(1, data_rows.length)
+
+      data = []
+      data_rows.each do |row|
+        row_data = {}
+        cells = row.find_elements :tag_name, "td"
+        cells.each_with_index do |cell, index|
+          key = header_data[index] || "#"
+          row_data[key] = cell.text.strip
         end
-        output << result
-        print "."
+        data << row_data
       end
-      output
+      data
     end
 
     def parse
       header_data = []
       begin
         thead = table.find_element(:tag_name, "thead")
-        cells = thead.find_element(:tag_name, "tr").find_elements(:tag_name, 'th')
+        cells = thead.find_element(:tag_name, "tr").find_elements(:tag_name, "th")
         puts "Writing header data"
         cells.each do |cell|
           header_data << cell.text
@@ -52,12 +56,12 @@ module RulethuStockExchange
         # Handle missing header gracefully (e.g., log the error)
       end
       tbody = table.find_element(:tag_name, "tbody")
-      rows = tbody.find_elements(:tag_name, 'tr')
+      rows = tbody.find_elements(:tag_name, "tr")
       body_data = []
       puts "\nWriting body data"
       rows.each do |row|
         data = {}
-        cells = row.find_elements(:tag_name, 'td')
+        cells = row.find_elements(:tag_name, "td")
         if header_data.empty?
           header_data = cells.map(&:text)
         end

data/lib/rulethu_stock_exchange/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RulethuStockExchange
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rulethu_stock_exchange
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Blessed Sibanda
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-11 00:00:00.000000000 Z
+date: 2024-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
@@ -52,7 +52,7 @@ metadata:
   homepage_uri: https://github.com/rulethu/rulethu_stock_exchange.git
   source_code_uri: https://github.com/rulethu/rulethu_stock_exchange.git
   changelog_uri: https://github.com/rulethu/rulethu_stock_exchange/blob/main/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -67,8 +67,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.7
-signing_key: 
+rubygems_version: 3.5.20
+signing_key:
 specification_version: 4
 summary: Get Stock Exchange Data
 test_files: []