uk_planning_scraper 0.4.6 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +87 -0
- data/README.md +2 -0
- data/lib/uk_planning_scraper/authority.rb +1 -1
- data/lib/uk_planning_scraper/authority_scrape_params.rb +17 -0
- data/lib/uk_planning_scraper/idox.rb +1 -0
- data/lib/uk_planning_scraper/northgate.rb +12 -0
- data/lib/uk_planning_scraper/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b54d09a91d835f0bc424cae3deb89bfad0cbb5ce0814d7881ee748beed370f70
|
4
|
+
data.tar.gz: 6146259451950c5ef52f572ccc69fe2cc3fce76c9a7d67b8965c71a2d20f4d45
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d274b2593dbb73e8b16cf2b220d7be945cd3c744aa7a9f19526ce828263f6847dd6931a4a27c38df97cdac227e2c3d2c4d76754dfc88f093d96079f26848760f
|
7
|
+
data.tar.gz: 0c8a4de8c1ceb9f77875e12e6895dfc7d89911d35a712cccfab66a08c6d19e92f070e9bf7ebee038d3681474cb73ce5f2baa6e8b68705f3e615113b797821e70
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
# Changelog
|
2
|
+
All notable changes to this project will be documented in this file.
|
3
|
+
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
|
+
|
7
|
+
## [Unreleased](https://github.com/adrianshort/uk_planning_scraper/compare/v0.5.0...master)
|
8
|
+
|
9
|
+
This functionality is on GitHub but not yet in the gem.
|
10
|
+
|
11
|
+
## [0.5.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.6...v0.5.0) - 2020-04-30
|
12
|
+
|
13
|
+
### Added
|
14
|
+
|
15
|
+
- Case officer search for Northgate
|
16
|
+
- Status search for Idox and Northgate
|
17
|
+
|
18
|
+
## [0.4.6](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.5...v0.4.6) - 2020-04-22
|
19
|
+
|
20
|
+
### Fixed
|
21
|
+
|
22
|
+
- Major change to Northgate output semantics. Scraper previously returned `date_received` from the search results pages. Now it returns `date_validated` which is correct. Previously collected data needs to be moved to this new column or rescraped. [#40](https://github.com/adrianshort/uk_planning_scraper/issues/40)
|
23
|
+
- Upgraded `rake` to fix [CVE-2020-8130](http://cve.circl.lu/cve/CVE-2020-8130).
|
24
|
+
|
25
|
+
## [0.4.5](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.4...v0.4.5) - 2020-02-24
|
26
|
+
|
27
|
+
### Added
|
28
|
+
|
29
|
+
- Gateshead Council.
|
30
|
+
|
31
|
+
## [0.4.4](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.3...v0.4.4) - 2019-10-29
|
32
|
+
|
33
|
+
### Fixed
|
34
|
+
|
35
|
+
- URLs for Camden, Islington and Merton councils.
|
36
|
+
|
37
|
+
## [0.4.3](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.2...v0.4.3) - 2019-01-22
|
38
|
+
|
39
|
+
### Changed
|
40
|
+
|
41
|
+
- `uk_planning_scraper` [published as a gem on RubyGems](https://rubygems.org/gems/uk_planning_scraper/versions/0.4.5) for the first time.
|
42
|
+
|
43
|
+
## [0.4.2](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.1...v0.4.2) - 2019-01-08
|
44
|
+
|
45
|
+
### Changed
|
46
|
+
|
47
|
+
- Kingston becomes Idox.
|
48
|
+
|
49
|
+
### Fixed
|
50
|
+
- Bradford and Leeds tagged `westyorkshire` not `westmidlands`.
|
51
|
+
|
52
|
+
## [0.4.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.0...v0.4.1) - 2018-10-18
|
53
|
+
|
54
|
+
### Added
|
55
|
+
|
56
|
+
- 39 new councils.
|
57
|
+
|
58
|
+
### Changed
|
59
|
+
|
60
|
+
- Use spaces not commas to separate tags in `authorities.csv` file.
|
61
|
+
- Use `pry` not `irb` as default console.
|
62
|
+
|
63
|
+
## [0.4.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.2...v0.4.0) - 2018-10-12
|
64
|
+
|
65
|
+
### Changed
|
66
|
+
|
67
|
+
- Use chained methods rather than an options hash for the scraper parameters. This permits better error checking. [#15](https://github.com/adrianshort/uk_planning_scraper/issues/15)
|
68
|
+
|
69
|
+
## [0.3.2](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.1...v0.3.2) - 2018-10-10
|
70
|
+
|
71
|
+
## [0.3.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.0...v0.3.1) - 2018-10-10
|
72
|
+
|
73
|
+
## [0.3.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.2.0...v0.3.0) - 2019-10-09
|
74
|
+
|
75
|
+
## [0.2.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.1.1...v0.2.0) - 2018-09-19
|
76
|
+
|
77
|
+
## [0.1.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.1.0...v0.1.1) - 2018-09-15
|
78
|
+
|
79
|
+
### Fixed
|
80
|
+
|
81
|
+
- [#3](https://github.com/adrianshort/uk_planning_scraper/issues/3)
|
82
|
+
|
83
|
+
## [0.1.0](https://github.com/adrianshort/uk_planning_scraper/releases/tag/v0.1.0) - 2018-09-14
|
84
|
+
|
85
|
+
### Added
|
86
|
+
|
87
|
+
- Basic Idox scraper.
|
data/README.md
CHANGED
@@ -106,6 +106,8 @@ keywords("hip gable")
|
|
106
106
|
applicant_name("Mr and Mrs Smith") # Currently Idox only
|
107
107
|
application_type("Householder") # Currently Idox only
|
108
108
|
development_type("") # Currently Idox only
|
109
|
+
case_officer_code("100000") # Northgate only
|
110
|
+
status("Pending Consideration") # Check valid status codes for each authority
|
109
111
|
|
110
112
|
scrape # runs the scraper
|
111
113
|
```
|
@@ -62,6 +62,17 @@ module UKPlanningScraper
|
|
62
62
|
self
|
63
63
|
end
|
64
64
|
|
65
|
+
def case_officer_code(s)
|
66
|
+
unless system == 'northgate'
|
67
|
+
raise NoMethodError.new("case_officer_code is only implemented for Northgate. \
|
68
|
+
This authority (#{@name}) is #{system.capitalize}.")
|
69
|
+
end
|
70
|
+
|
71
|
+
check_class(s, String)
|
72
|
+
@scrape_params[:case_officer_code] = s.strip
|
73
|
+
self
|
74
|
+
end
|
75
|
+
|
65
76
|
def application_type(s)
|
66
77
|
unless system == 'idox'
|
67
78
|
raise NoMethodError.new("application_type is only implemented for \
|
@@ -84,6 +95,12 @@ module UKPlanningScraper
|
|
84
95
|
self
|
85
96
|
end
|
86
97
|
|
98
|
+
def status(s)
|
99
|
+
check_class(s, String)
|
100
|
+
@scrape_params[:status] = s.strip
|
101
|
+
self
|
102
|
+
end
|
103
|
+
|
87
104
|
private
|
88
105
|
|
89
106
|
# Handle the simple params with this
|
@@ -42,6 +42,7 @@ module UKPlanningScraper
|
|
42
42
|
form.send(:"date(applicationDecisionEnd)", params[:decided_to].strftime(date_format)) if params[:decided_to]
|
43
43
|
|
44
44
|
form.send(:"searchCriteria\.description", params[:keywords])
|
45
|
+
form.send(:"searchCriteria\.caseStatus", params[:status])
|
45
46
|
|
46
47
|
# Some councils don't have the applicant name on their form, eg Bexley
|
47
48
|
form.send(:"searchCriteria\.applicantName", params[:applicant_name]) if form.has_field? 'searchCriteria.applicantName'
|
@@ -24,6 +24,7 @@ module UKPlanningScraper
|
|
24
24
|
'csbtnSearch' => 'Search' # required
|
25
25
|
}
|
26
26
|
|
27
|
+
# Keywords
|
27
28
|
form_vars['txtProposal'] = params[:keywords]
|
28
29
|
|
29
30
|
# Date received from and to
|
@@ -49,6 +50,17 @@ module UKPlanningScraper
|
|
49
50
|
form_vars['dateStart'] = params[:decided_from].to_s if params[:decided_from] # YYYY-MM-DD
|
50
51
|
form_vars['dateEnd'] = params[:decided_to].to_s if params[:decided_to] # YYYY-MM-DD
|
51
52
|
end
|
53
|
+
|
54
|
+
# Status
|
55
|
+
if params[:status]
|
56
|
+
form_vars['cboStatusCode'] = params[:status]
|
57
|
+
end
|
58
|
+
|
59
|
+
# Case officer code
|
60
|
+
if params[:case_officer_code]
|
61
|
+
form_vars['cboCaseOfficerCode'] = params[:case_officer_code]
|
62
|
+
@url.sub!('GeneralSearch.aspx', 'CaseOfficerWorkloadSearch.aspx')
|
63
|
+
end
|
52
64
|
|
53
65
|
logger.info "Form variables: #{form_vars.to_s}"
|
54
66
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: uk_planning_scraper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adrian Short
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-04-
|
11
|
+
date: 2020-04-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -144,6 +144,7 @@ extra_rdoc_files: []
|
|
144
144
|
files:
|
145
145
|
- ".gitignore"
|
146
146
|
- ".rspec"
|
147
|
+
- CHANGELOG.md
|
147
148
|
- Gemfile
|
148
149
|
- LICENSE
|
149
150
|
- README.md
|