rulethu_stock_exchange 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.idea/.gitignore +8 -0
- data/.idea/misc.xml +4 -0
- data/.idea/modules.xml +8 -0
- data/.idea/rulethu_stock_exchange.iml +88 -0
- data/.idea/vcs.xml +6 -0
- data/.standard.yml +3 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/LSE-Data-21-03-2024.png +0 -0
- data/README.md +82 -0
- data/Rakefile +10 -0
- data/data/lse/21-03-2024-data.csv +8 -0
- data/data/lse/21-03-2024-data.json +1 -0
- data/lib/rulethu_stock_exchange/exchanges.rb +21 -0
- data/lib/rulethu_stock_exchange/io.rb +37 -0
- data/lib/rulethu_stock_exchange/parser.rb +73 -0
- data/lib/rulethu_stock_exchange/scraper.rb +25 -0
- data/lib/rulethu_stock_exchange/version.rb +5 -0
- data/lib/rulethu_stock_exchange.rb +13 -0
- data/rulethu_stock_exchange.gemspec +40 -0
- data/sig/rulethu_stock_exchange.rbs +4 -0
- metadata +82 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ae7e64bb1eeb11ed443caf1a820587474e778fd0c0dc8ebcd857df97a067753c
|
4
|
+
data.tar.gz: b7fd66512ca5d407d74bbef60457b78215aed185382e6a28bfcc0e2bbae2b0be
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 144a6ab8c2386fac547de206a7937837ce9d57d2084f7880e01258ff5ffa0c877e2599188bb2230dc72a51652924f38c06dc998d4b1106b6662063dcd0fe04e3
|
7
|
+
data.tar.gz: bfe874cf4e3cefbd45eecf5e9d6453a95d0d0f36caa9e320067e32de8ef29e6f98073b0c891c061ea9c0ddce5ce8c1773f4d5bec8ba6302ab8edfce92242f0e1
|
data/.idea/.gitignore
ADDED
data/.idea/misc.xml
ADDED
data/.idea/modules.xml
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<project version="4">
|
3
|
+
<component name="ProjectModuleManager">
|
4
|
+
<modules>
|
5
|
+
<module fileurl="file://$PROJECT_DIR$/.idea/rulethu_stock_exchange.iml" filepath="$PROJECT_DIR$/.idea/rulethu_stock_exchange.iml" />
|
6
|
+
</modules>
|
7
|
+
</component>
|
8
|
+
</project>
|
@@ -0,0 +1,88 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<module type="RUBY_MODULE" version="4">
|
3
|
+
<component name="ModuleRunConfigurationManager">
|
4
|
+
<shared />
|
5
|
+
</component>
|
6
|
+
<component name="NewModuleRootManager">
|
7
|
+
<content url="file://$MODULE_DIR$">
|
8
|
+
<sourceFolder url="file://$MODULE_DIR$/features" isTestSource="true" />
|
9
|
+
<sourceFolder url="file://$MODULE_DIR$/spec" isTestSource="true" />
|
10
|
+
<sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
|
11
|
+
</content>
|
12
|
+
<orderEntry type="inheritedJdk" />
|
13
|
+
<orderEntry type="sourceFolder" forTests="false" />
|
14
|
+
<orderEntry type="library" scope="PROVIDED" name="ast (v2.4.2, asdf: 3.2.2) [gem]" level="application" />
|
15
|
+
<orderEntry type="library" scope="PROVIDED" name="base64 (v0.2.0, asdf: 3.2.2) [gem]" level="application" />
|
16
|
+
<orderEntry type="library" scope="PROVIDED" name="bundler (v2.5.6, asdf: 3.2.2) [gem]" level="application" />
|
17
|
+
<orderEntry type="library" scope="PROVIDED" name="json (v2.7.1, asdf: 3.2.2) [gem]" level="application" />
|
18
|
+
<orderEntry type="library" scope="PROVIDED" name="language_server-protocol (v3.17.0.3, asdf: 3.2.2) [gem]" level="application" />
|
19
|
+
<orderEntry type="library" scope="PROVIDED" name="lint_roller (v1.1.0, asdf: 3.2.2) [gem]" level="application" />
|
20
|
+
<orderEntry type="library" scope="PROVIDED" name="minitest (v5.22.3, asdf: 3.2.2) [gem]" level="application" />
|
21
|
+
<orderEntry type="library" scope="PROVIDED" name="parallel (v1.24.0, asdf: 3.2.2) [gem]" level="application" />
|
22
|
+
<orderEntry type="library" scope="PROVIDED" name="parser (v3.3.0.5, asdf: 3.2.2) [gem]" level="application" />
|
23
|
+
<orderEntry type="library" scope="PROVIDED" name="racc (v1.7.3, asdf: 3.2.2) [gem]" level="application" />
|
24
|
+
<orderEntry type="library" scope="PROVIDED" name="rainbow (v3.1.1, asdf: 3.2.2) [gem]" level="application" />
|
25
|
+
<orderEntry type="library" scope="PROVIDED" name="rake (v13.1.0, asdf: 3.2.2) [gem]" level="application" />
|
26
|
+
<orderEntry type="library" scope="PROVIDED" name="regexp_parser (v2.9.0, asdf: 3.2.2) [gem]" level="application" />
|
27
|
+
<orderEntry type="library" scope="PROVIDED" name="rexml (v3.2.6, asdf: 3.2.2) [gem]" level="application" />
|
28
|
+
<orderEntry type="library" scope="PROVIDED" name="rubocop (v1.62.1, asdf: 3.2.2) [gem]" level="application" />
|
29
|
+
<orderEntry type="library" scope="PROVIDED" name="rubocop-ast (v1.31.2, asdf: 3.2.2) [gem]" level="application" />
|
30
|
+
<orderEntry type="library" scope="PROVIDED" name="rubocop-performance (v1.20.2, asdf: 3.2.2) [gem]" level="application" />
|
31
|
+
<orderEntry type="library" scope="PROVIDED" name="ruby-progressbar (v1.13.0, asdf: 3.2.2) [gem]" level="application" />
|
32
|
+
<orderEntry type="library" scope="PROVIDED" name="rubyzip (v2.3.2, asdf: 3.2.2) [gem]" level="application" />
|
33
|
+
<orderEntry type="library" scope="PROVIDED" name="selenium-webdriver (v4.18.1, asdf: 3.2.2) [gem]" level="application" />
|
34
|
+
<orderEntry type="library" scope="PROVIDED" name="standard (v1.35.1, asdf: 3.2.2) [gem]" level="application" />
|
35
|
+
<orderEntry type="library" scope="PROVIDED" name="standard-custom (v1.0.2, asdf: 3.2.2) [gem]" level="application" />
|
36
|
+
<orderEntry type="library" scope="PROVIDED" name="standard-performance (v1.3.1, asdf: 3.2.2) [gem]" level="application" />
|
37
|
+
<orderEntry type="library" scope="PROVIDED" name="unicode-display_width (v2.5.0, asdf: 3.2.2) [gem]" level="application" />
|
38
|
+
<orderEntry type="library" scope="PROVIDED" name="websocket (v1.2.10, asdf: 3.2.2) [gem]" level="application" />
|
39
|
+
</component>
|
40
|
+
<component name="RakeTasksCache-v2">
|
41
|
+
<option name="myRootTask">
|
42
|
+
<RakeTaskImpl id="rake">
|
43
|
+
<subtasks>
|
44
|
+
<RakeTaskImpl description="Build rulethu_stock_exchange-0.1.0.gem into the pkg directory" fullCommand="build" id="build" />
|
45
|
+
<RakeTaskImpl id="build">
|
46
|
+
<subtasks>
|
47
|
+
<RakeTaskImpl description="Generate SHA512 checksum if rulethu_stock_exchange-0.1.0.gem into the checksums directory" fullCommand="build:checksum" id="checksum" />
|
48
|
+
</subtasks>
|
49
|
+
</RakeTaskImpl>
|
50
|
+
<RakeTaskImpl description="Remove any temporary products" fullCommand="clean" id="clean" />
|
51
|
+
<RakeTaskImpl description="Remove any generated files" fullCommand="clobber" id="clobber" />
|
52
|
+
<RakeTaskImpl description="Build and install rulethu_stock_exchange-0.1.0.gem into system gems" fullCommand="install" id="install" />
|
53
|
+
<RakeTaskImpl id="install">
|
54
|
+
<subtasks>
|
55
|
+
<RakeTaskImpl description="Build and install rulethu_stock_exchange-0.1.0.gem into system gems without network access" fullCommand="install:local" id="local" />
|
56
|
+
</subtasks>
|
57
|
+
</RakeTaskImpl>
|
58
|
+
<RakeTaskImpl description="Create tag v0.1.0 and build and push rulethu_stock_exchange-0.1.0.gem to https://github.com/rulethu/rulethu_stock_exchange.git" fullCommand="release[remote]" id="release[remote]" />
|
59
|
+
<RakeTaskImpl description="Lint with the Standard Ruby style guide" fullCommand="standard" id="standard" />
|
60
|
+
<RakeTaskImpl id="standard">
|
61
|
+
<subtasks>
|
62
|
+
<RakeTaskImpl description="Lint and automatically make safe fixes with the Standard Ruby style guide" fullCommand="standard:fix" id="fix" />
|
63
|
+
<RakeTaskImpl description="Lint and automatically make fixes (even unsafe ones" fullCommand="standard:fix_unsafely" id="fix_unsafely" />
|
64
|
+
</subtasks>
|
65
|
+
</RakeTaskImpl>
|
66
|
+
<RakeTaskImpl description="Run the test suite" fullCommand="test" id="test" />
|
67
|
+
<RakeTaskImpl id="test">
|
68
|
+
<subtasks>
|
69
|
+
<RakeTaskImpl description="Print out the test command" fullCommand="test:cmd" id="cmd" />
|
70
|
+
<RakeTaskImpl description="Show which test files fail when run in isolation" fullCommand="test:isolated" id="isolated" />
|
71
|
+
<RakeTaskImpl description="Show bottom 25 tests wrt time" fullCommand="test:slow" id="slow" />
|
72
|
+
<RakeTaskImpl description="" fullCommand="test:deps" id="deps" />
|
73
|
+
</subtasks>
|
74
|
+
</RakeTaskImpl>
|
75
|
+
<RakeTaskImpl description="" fullCommand="default" id="default" />
|
76
|
+
<RakeTaskImpl description="" fullCommand="release" id="release" />
|
77
|
+
<RakeTaskImpl id="release">
|
78
|
+
<subtasks>
|
79
|
+
<RakeTaskImpl description="" fullCommand="release:guard_clean" id="guard_clean" />
|
80
|
+
<RakeTaskImpl description="" fullCommand="release:rubygem_push" id="rubygem_push" />
|
81
|
+
<RakeTaskImpl description="" fullCommand="release:source_control_push" id="source_control_push" />
|
82
|
+
</subtasks>
|
83
|
+
</RakeTaskImpl>
|
84
|
+
</subtasks>
|
85
|
+
</RakeTaskImpl>
|
86
|
+
</option>
|
87
|
+
</component>
|
88
|
+
</module>
|
data/.idea/vcs.xml
ADDED
data/.standard.yml
ADDED
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2024 Blessed Sibanda
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
Binary file
|
data/README.md
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
# RulethuStockExchange
|
2
|
+
|
3
|
+
This gem allows you to scrape stock exchange data from the following exchanges:
|
4
|
+
* Zimbabwe Stock Exchange
|
5
|
+
* Johannesburg Stock Exchange
|
6
|
+
* London Stock Exchange
|
7
|
+
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Install the gem and add to the application's Gemfile by executing:
|
12
|
+
|
13
|
+
$ bundle add rulethu_stock_exchange
|
14
|
+
|
15
|
+
If bundler is not being used to manage dependencies, install the gem by executing:
|
16
|
+
|
17
|
+
$ gem install rulethu_stock_exchange
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
### Example 1: Scraping Data From London Stock Exchange
|
22
|
+
|
23
|
+
Get Exchange
|
24
|
+
|
25
|
+
```ruby
|
26
|
+
exchange = RulethuStockExchange::Exchanges::LSE
|
27
|
+
```
|
28
|
+
Create a Scraper passing in the url for the exchangew
|
29
|
+
```ruby
|
30
|
+
scraper = RulethuStockExchange::Scraper.new exchange[:url]
|
31
|
+
```
|
32
|
+
|
33
|
+
Scrape the data passing in a selector
|
34
|
+
```ruby
|
35
|
+
html = scraper.scrape exchange[:selector]
|
36
|
+
```
|
37
|
+
|
38
|
+
Create a parser to parse the data
|
39
|
+
```ruby
|
40
|
+
parser = RulethuStockExchange::Parser.new html
|
41
|
+
```
|
42
|
+
|
43
|
+
Parse the data
|
44
|
+
```ruby
|
45
|
+
data = parser.parse()
|
46
|
+
```
|
47
|
+
|
48
|
+
Write the data to JSON file
|
49
|
+
```ruby
|
50
|
+
filename =RulethuStockExchange::IO.write_to_json_file data, 'lse'
|
51
|
+
```
|
52
|
+
|
53
|
+
Convert the JSON file to CSV
|
54
|
+
```ruby
|
55
|
+
RulethuStockExchange::IO.json_to_csv filename
|
56
|
+
```
|
57
|
+
|
58
|
+
The CSV file is generated as follows
|
59
|
+
|
60
|
+
!["CSV Data London Stock Exchange 21/03/2023"](./LSE-Data-21-03-2024.png)
|
61
|
+
|
62
|
+
### Example 2: Scraping Data From Zimbabwe Stock Exchange
|
63
|
+
|
64
|
+
For Zimbabwe Stock Exchange, the procedure is the same as in the previous example except that the parsing method is different as shown below.
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
data = parser.parse_zse()
|
68
|
+
```
|
69
|
+
|
70
|
+
## Development
|
71
|
+
|
72
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
73
|
+
|
74
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
75
|
+
|
76
|
+
## Contributing
|
77
|
+
|
78
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/rulethu/rulethu_stock_exchange.
|
79
|
+
|
80
|
+
## License
|
81
|
+
|
82
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
Index,Value,Change,High,Low,Prev close
|
2
|
+
FTSE 100,"7,737.38",-0.92 (-0.01%),"7,749.39","7,711.46","7,738.30"
|
3
|
+
FTSE 250,"19,484.40",51.59 (0.27%),"19,506.67","19,420.99","19,432.81"
|
4
|
+
FTSE 350,"4,268.83",1.15 (0.03%),"4,274.93","4,254.63","4,267.68"
|
5
|
+
FTSE All-Share,"4,225.49",1.13 (0.03%),"4,231.39","4,211.62","4,224.36"
|
6
|
+
FTSE AIM UK 50 Index,"3,869.45",-23.25 (-0.60%),"3,899.37","3,863.99","3,892.70"
|
7
|
+
FTSE AIM 100 Index,"3,550.67",4.54 (0.13%),"3,554.44","3,539.44","3,546.13"
|
8
|
+
FTSE AIM All-Share,735.59,0.03 (-),736.96,734.18,735.56
|
@@ -0,0 +1 @@
|
|
1
|
+
[{"Index":"FTSE 100","Value":"7,737.38","Change":"-0.92 (-0.01%)","High":"7,749.39","Low":"7,711.46","Prev close":"7,738.30"},{"Index":"FTSE 250","Value":"19,484.40","Change":"51.59 (0.27%)","High":"19,506.67","Low":"19,420.99","Prev close":"19,432.81"},{"Index":"FTSE 350","Value":"4,268.83","Change":"1.15 (0.03%)","High":"4,274.93","Low":"4,254.63","Prev close":"4,267.68"},{"Index":"FTSE All-Share","Value":"4,225.49","Change":"1.13 (0.03%)","High":"4,231.39","Low":"4,211.62","Prev close":"4,224.36"},{"Index":"FTSE AIM UK 50 Index","Value":"3,869.45","Change":"-23.25 (-0.60%)","High":"3,899.37","Low":"3,863.99","Prev close":"3,892.70"},{"Index":"FTSE AIM 100 Index","Value":"3,550.67","Change":"4.54 (0.13%)","High":"3,554.44","Low":"3,539.44","Prev close":"3,546.13"},{"Index":"FTSE AIM All-Share","Value":"735.59","Change":"0.03 (-)","High":"736.96","Low":"734.18","Prev close":"735.56"}]
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RulethuStockExchange::Exchanges
|
4
|
+
LSE = {
|
5
|
+
name: 'London Stock Exchange',
|
6
|
+
url: 'https://www.londonstockexchange.com/indices?tab=ftse-indices',
|
7
|
+
selector: 'section#indices-table-IntradayValues > table'
|
8
|
+
}
|
9
|
+
|
10
|
+
JSE = {
|
11
|
+
name: 'Johannesburg Stock Exchange',
|
12
|
+
url: 'https://www.jse.co.za/indices',
|
13
|
+
selector: 'table'
|
14
|
+
}
|
15
|
+
|
16
|
+
ZSE = {
|
17
|
+
name: 'Zimbabwe Stock Exchange',
|
18
|
+
url: 'https://www.zse.co.zw/price-sheet/',
|
19
|
+
selector: 'table'
|
20
|
+
}
|
21
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'json'
|
3
|
+
require 'csv'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
module RulethuStockExchange
|
7
|
+
class IO
|
8
|
+
def self.write_to_json_file(data, prefix)
|
9
|
+
puts "\nWriting to JSON with prefix #{prefix}"
|
10
|
+
date = Time.now.strftime("%d-%m-%Y")
|
11
|
+
data_dir = Pathname.new("data") / Pathname.new(prefix)
|
12
|
+
data_dir.mkpath
|
13
|
+
filename = "#{data_dir}/#{date}-data.json"
|
14
|
+
File.open(filename, 'w') do |f|
|
15
|
+
f.write(JSON.dump(data))
|
16
|
+
end
|
17
|
+
return filename
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.json_to_csv(json_file)
|
21
|
+
puts "\nConverting #{json_file} to CSV"
|
22
|
+
data = Object.new
|
23
|
+
File.open(json_file, 'r') do |f|
|
24
|
+
data = JSON.parse(f.read)
|
25
|
+
end
|
26
|
+
|
27
|
+
filename = json_file.sub(/\.json$/, '.csv')
|
28
|
+
|
29
|
+
CSV.open(filename, 'w') do |csv|
|
30
|
+
csv << data.first.keys
|
31
|
+
data.each do |row|
|
32
|
+
csv << row.values
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'selenium-webdriver'
|
3
|
+
|
4
|
+
module RulethuStockExchange
|
5
|
+
class Parser
|
6
|
+
attr_reader :table
|
7
|
+
|
8
|
+
def initialize(table)
|
9
|
+
@table = table
|
10
|
+
end
|
11
|
+
|
12
|
+
def parse_zse
|
13
|
+
tbody = table.find_element(:tag_name, 'tbody')
|
14
|
+
rows = tbody.find_elements(:tag_name, 'tr')
|
15
|
+
body_data = []
|
16
|
+
puts "\nWriting body data"
|
17
|
+
rows.each do |row|
|
18
|
+
cells = row.find_elements(:tag_name, 'td')
|
19
|
+
cells.each do |cell|
|
20
|
+
body_data << cell.text.strip
|
21
|
+
end
|
22
|
+
end
|
23
|
+
data = body_data.reject(&:empty?)
|
24
|
+
header_data = data[1..4].map { |d| d.gsub("\n", " ") }
|
25
|
+
remaining_data = data[5..]
|
26
|
+
grouped_data = remaining_data.each_slice(4).to_a
|
27
|
+
data = grouped_data.reject { |row| row.first.match?(/^[-+]?\d+(\.\d+)?$/) }
|
28
|
+
data = data[0...-1]
|
29
|
+
output = []
|
30
|
+
data.each do |row|
|
31
|
+
result = {}
|
32
|
+
row.each_with_index do |value, index|
|
33
|
+
result[header_data[index]] = value
|
34
|
+
end
|
35
|
+
output << result
|
36
|
+
print "."
|
37
|
+
end
|
38
|
+
output
|
39
|
+
end
|
40
|
+
|
41
|
+
def parse
|
42
|
+
header_data = []
|
43
|
+
begin
|
44
|
+
thead = table.find_element(:tag_name, "thead")
|
45
|
+
cells = thead.find_element(:tag_name, "tr").find_elements(:tag_name, 'th')
|
46
|
+
puts "Writing header data"
|
47
|
+
cells.each do |cell|
|
48
|
+
header_data << cell.text
|
49
|
+
print "."
|
50
|
+
end
|
51
|
+
rescue Selenium::WebDriver::Error::NoSuchElement => e
|
52
|
+
# Handle missing header gracefully (e.g., log the error)
|
53
|
+
end
|
54
|
+
tbody = table.find_element(:tag_name, "tbody")
|
55
|
+
rows = tbody.find_elements(:tag_name, 'tr')
|
56
|
+
body_data = []
|
57
|
+
puts "\nWriting body data"
|
58
|
+
rows.each do |row|
|
59
|
+
data = {}
|
60
|
+
cells = row.find_elements(:tag_name, 'td')
|
61
|
+
if header_data.empty?
|
62
|
+
header_data = cells.map(&:text)
|
63
|
+
end
|
64
|
+
cells.each_with_index do |cell, index|
|
65
|
+
data[header_data[index]] = cell.text unless header_data[index].strip.empty?
|
66
|
+
end
|
67
|
+
body_data << data
|
68
|
+
print "."
|
69
|
+
end
|
70
|
+
body_data
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'selenium-webdriver'
|
3
|
+
|
4
|
+
module RulethuStockExchange
|
5
|
+
class Scraper
|
6
|
+
def initialize(url)
|
7
|
+
@url = url
|
8
|
+
@driver = setup_driver
|
9
|
+
end
|
10
|
+
|
11
|
+
def scrape(selector)
|
12
|
+
@driver.navigate.to(@url)
|
13
|
+
sleep 5
|
14
|
+
@driver.find_element :css, selector
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def setup_driver
|
20
|
+
options = Selenium::WebDriver::Chrome::Options.new
|
21
|
+
options.add_argument("--headless")
|
22
|
+
Selenium::WebDriver.for :chrome, options: options
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "rulethu_stock_exchange/version"
|
4
|
+
require_relative "rulethu_stock_exchange/parser"
|
5
|
+
require_relative "rulethu_stock_exchange/scraper"
|
6
|
+
require_relative "rulethu_stock_exchange/io"
|
7
|
+
require_relative "rulethu_stock_exchange/exchanges"
|
8
|
+
|
9
|
+
module RulethuStockExchange
|
10
|
+
class Error < StandardError; end
|
11
|
+
# Your code goes here...
|
12
|
+
|
13
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/rulethu_stock_exchange/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "rulethu_stock_exchange"
|
7
|
+
spec.version = RulethuStockExchange::VERSION
|
8
|
+
spec.authors = ["Blessed Sibanda"]
|
9
|
+
spec.email = ["blessed@rulethu.com"]
|
10
|
+
|
11
|
+
spec.summary = "Get Stock Exchange Data"
|
12
|
+
spec.description = "Get Stock Exchange Data Needed by Rulethu from the Web"
|
13
|
+
spec.homepage = "https://github.com/rulethu/rulethu_stock_exchange.git"
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.required_ruby_version = ">= 3.0.0"
|
16
|
+
|
17
|
+
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
18
|
+
|
19
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
20
|
+
spec.metadata["source_code_uri"] = "https://github.com/rulethu/rulethu_stock_exchange.git"
|
21
|
+
spec.metadata["changelog_uri"] = "https://github.com/rulethu/rulethu_stock_exchange/blob/main/CHANGELOG.md"
|
22
|
+
|
23
|
+
# Specify which files should be added to the gem when it is released.
|
24
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
25
|
+
spec.files = Dir.chdir(__dir__) do
|
26
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
27
|
+
(File.expand_path(f) == __FILE__) ||
|
28
|
+
f.start_with?(*%w[bin/ test/ spec/ features/ .git appveyor Gemfile])
|
29
|
+
end
|
30
|
+
end
|
31
|
+
spec.bindir = "exe"
|
32
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
33
|
+
spec.require_paths = ["lib"]
|
34
|
+
|
35
|
+
# Uncomment to register a new dependency of your gem
|
36
|
+
spec.add_dependency "selenium-webdriver", "~> 4.18"
|
37
|
+
|
38
|
+
# For more information and examples about making a new gem, check out our
|
39
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
40
|
+
end
|
metadata
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rulethu_stock_exchange
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Blessed Sibanda
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-03-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: selenium-webdriver
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '4.18'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4.18'
|
27
|
+
description: Get Stock Exchange Data Needed by Rulethu from the Web
|
28
|
+
email:
|
29
|
+
- blessed@rulethu.com
|
30
|
+
executables: []
|
31
|
+
extensions: []
|
32
|
+
extra_rdoc_files: []
|
33
|
+
files:
|
34
|
+
- ".idea/.gitignore"
|
35
|
+
- ".idea/misc.xml"
|
36
|
+
- ".idea/modules.xml"
|
37
|
+
- ".idea/rulethu_stock_exchange.iml"
|
38
|
+
- ".idea/vcs.xml"
|
39
|
+
- ".standard.yml"
|
40
|
+
- CHANGELOG.md
|
41
|
+
- LICENSE.txt
|
42
|
+
- LSE-Data-21-03-2024.png
|
43
|
+
- README.md
|
44
|
+
- Rakefile
|
45
|
+
- data/lse/21-03-2024-data.csv
|
46
|
+
- data/lse/21-03-2024-data.json
|
47
|
+
- lib/rulethu_stock_exchange.rb
|
48
|
+
- lib/rulethu_stock_exchange/exchanges.rb
|
49
|
+
- lib/rulethu_stock_exchange/io.rb
|
50
|
+
- lib/rulethu_stock_exchange/parser.rb
|
51
|
+
- lib/rulethu_stock_exchange/scraper.rb
|
52
|
+
- lib/rulethu_stock_exchange/version.rb
|
53
|
+
- rulethu_stock_exchange.gemspec
|
54
|
+
- sig/rulethu_stock_exchange.rbs
|
55
|
+
homepage: https://github.com/rulethu/rulethu_stock_exchange.git
|
56
|
+
licenses:
|
57
|
+
- MIT
|
58
|
+
metadata:
|
59
|
+
allowed_push_host: https://rubygems.org
|
60
|
+
homepage_uri: https://github.com/rulethu/rulethu_stock_exchange.git
|
61
|
+
source_code_uri: https://github.com/rulethu/rulethu_stock_exchange.git
|
62
|
+
changelog_uri: https://github.com/rulethu/rulethu_stock_exchange/blob/main/CHANGELOG.md
|
63
|
+
post_install_message:
|
64
|
+
rdoc_options: []
|
65
|
+
require_paths:
|
66
|
+
- lib
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: 3.0.0
|
72
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
requirements: []
|
78
|
+
rubygems_version: 3.5.6
|
79
|
+
signing_key:
|
80
|
+
specification_version: 4
|
81
|
+
summary: Get Stock Exchange Data
|
82
|
+
test_files: []
|