uk_planning_scraper 0.4.6 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 340c6c3789dd7a3f62308e0a603bd577c118560692c0132b348f93160bf9224d
4
- data.tar.gz: be4157ac9594b467f1f8a998ecfbe44c3c948b0bf837bd2bbb98d15726af879b
3
+ metadata.gz: b54d09a91d835f0bc424cae3deb89bfad0cbb5ce0814d7881ee748beed370f70
4
+ data.tar.gz: 6146259451950c5ef52f572ccc69fe2cc3fce76c9a7d67b8965c71a2d20f4d45
5
5
  SHA512:
6
- metadata.gz: add97d946b69f366af36aa499b7f27f15cf04024c6a2eac2561ea3e1fc8cfe5853ebb6ea65e8962ff27010cd46f09c4129a6be4ad453c7cde6fc32717c24c071
7
- data.tar.gz: eb3310870c252a74289478edaffc755e695f715cb25ee24a202a2ba487fba32d583541263fdd2e3a11cb540f2bc1e60fa944941c4f7f63a0a2a0eb83a68f9c82
6
+ metadata.gz: d274b2593dbb73e8b16cf2b220d7be945cd3c744aa7a9f19526ce828263f6847dd6931a4a27c38df97cdac227e2c3d2c4d76754dfc88f093d96079f26848760f
7
+ data.tar.gz: 0c8a4de8c1ceb9f77875e12e6895dfc7d89911d35a712cccfab66a08c6d19e92f070e9bf7ebee038d3681474cb73ce5f2baa6e8b68705f3e615113b797821e70
data/CHANGELOG.md ADDED
@@ -0,0 +1,87 @@
1
+ # Changelog
2
+ All notable changes to this project will be documented in this file.
3
+
4
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [Unreleased](https://github.com/adrianshort/uk_planning_scraper/compare/v0.5.0...master)
8
+
9
+ This functionality is on GitHub but not yet in the gem.
10
+
11
+ ## [0.5.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.6...v0.5.0) - 2020-04-30
12
+
13
+ ### Added
14
+
15
+ - Case officer search for Northgate
16
+ - Status search for Idox and Northgate
17
+
18
+ ## [0.4.6](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.5...v0.4.6) - 2020-04-22
19
+
20
+ ### Fixed
21
+
22
+ - Major change to Northgate output semantics. Scraper previously returned `date_received` from the search results pages. Now it returns `date_validated` which is correct. Previously collected data needs to be moved to this new column or rescraped. [#40](https://github.com/adrianshort/uk_planning_scraper/issues/40)
23
+ - Upgraded `rake` to fix [CVE-2020-8130](http://cve.circl.lu/cve/CVE-2020-8130).
24
+
25
+ ## [0.4.5](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.4...v0.4.5) - 2020-02-24
26
+
27
+ ### Added
28
+
29
+ - Gateshead Council.
30
+
31
+ ## [0.4.4](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.3...v0.4.4) - 2019-10-29
32
+
33
+ ### Fixed
34
+
35
+ - URLs for Camden, Islington and Merton councils.
36
+
37
+ ## [0.4.3](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.2...v0.4.3) - 2019-01-22
38
+
39
+ ### Changed
40
+
41
+ - `uk_planning_scraper` [published as a gem on RubyGems](https://rubygems.org/gems/uk_planning_scraper/versions/0.4.5) for the first time.
42
+
43
+ ## [0.4.2](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.1...v0.4.2) - 2019-01-08
44
+
45
+ ### Changed
46
+
47
+ - Kingston becomes Idox.
48
+
49
+ ### Fixed
50
+ - Bradford and Leeds tagged `westyorkshire` not `westmidlands`.
51
+
52
+ ## [0.4.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.0...v0.4.1) - 2018-10-18
53
+
54
+ ### Added
55
+
56
+ - 39 new councils.
57
+
58
+ ### Changed
59
+
60
+ - Use spaces not commas to separate tags in `authorities.csv` file.
61
+ - Use `pry` not `irb` as default console.
62
+
63
+ ## [0.4.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.2...v0.4.0) - 2018-10-12
64
+
65
+ ### Changed
66
+
67
+ - Use chained methods rather than an options hash for the scraper parameters. This permits better error checking. [#15](https://github.com/adrianshort/uk_planning_scraper/issues/15)
68
+
69
+ ## [0.3.2](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.1...v0.3.2) - 2018-10-10
70
+
71
+ ## [0.3.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.0...v0.3.1) - 2018-10-10
72
+
73
+ ## [0.3.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.2.0...v0.3.0) - 2019-10-09
74
+
75
+ ## [0.2.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.1.1...v0.2.0) - 2018-09-19
76
+
77
+ ## [0.1.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.1.0...v0.1.1) - 2018-09-15
78
+
79
+ ### Fixed
80
+
81
+ - [#3](https://github.com/adrianshort/uk_planning_scraper/issues/3)
82
+
83
+ ## [0.1.0](https://github.com/adrianshort/uk_planning_scraper/releases/tag/v0.1.0) - 2018-09-14
84
+
85
+ ### Added
86
+
87
+ - Basic Idox scraper.
data/README.md CHANGED
@@ -106,6 +106,8 @@ keywords("hip gable")
106
106
  applicant_name("Mr and Mrs Smith") # Currently Idox only
107
107
  application_type("Householder") # Currently Idox only
108
108
  development_type("") # Currently Idox only
109
+ case_officer_code("100000") # Northgate only
110
+ status("Pending Consideration") # Check valid status codes for each authority
109
111
 
110
112
  scrape # runs the scraper
111
113
  ```
@@ -70,7 +70,7 @@ module UKPlanningScraper
70
70
  def system
71
71
  if @url.match(/search\.do\?action=advanced/i)
72
72
  'idox'
73
- elsif @url.match(/generalsearch\.aspx/i)
73
+ elsif @url.match(/\.aspx/i)
74
74
  'northgate'
75
75
  elsif @url.match(/ocellaweb/i)
76
76
  'ocellaweb'
@@ -62,6 +62,17 @@ module UKPlanningScraper
62
62
  self
63
63
  end
64
64
 
65
+ def case_officer_code(s)
66
+ unless system == 'northgate'
67
+ raise NoMethodError.new("case_officer_code is only implemented for Northgate. \
68
+ This authority (#{@name}) is #{system.capitalize}.")
69
+ end
70
+
71
+ check_class(s, String)
72
+ @scrape_params[:case_officer_code] = s.strip
73
+ self
74
+ end
75
+
65
76
  def application_type(s)
66
77
  unless system == 'idox'
67
78
  raise NoMethodError.new("application_type is only implemented for \
@@ -84,6 +95,12 @@ module UKPlanningScraper
84
95
  self
85
96
  end
86
97
 
98
+ def status(s)
99
+ check_class(s, String)
100
+ @scrape_params[:status] = s.strip
101
+ self
102
+ end
103
+
87
104
  private
88
105
 
89
106
  # Handle the simple params with this
@@ -42,6 +42,7 @@ module UKPlanningScraper
42
42
  form.send(:"date(applicationDecisionEnd)", params[:decided_to].strftime(date_format)) if params[:decided_to]
43
43
 
44
44
  form.send(:"searchCriteria\.description", params[:keywords])
45
+ form.send(:"searchCriteria\.caseStatus", params[:status])
45
46
 
46
47
  # Some councils don't have the applicant name on their form, eg Bexley
47
48
  form.send(:"searchCriteria\.applicantName", params[:applicant_name]) if form.has_field? 'searchCriteria.applicantName'
@@ -24,6 +24,7 @@ module UKPlanningScraper
24
24
  'csbtnSearch' => 'Search' # required
25
25
  }
26
26
 
27
+ # Keywords
27
28
  form_vars['txtProposal'] = params[:keywords]
28
29
 
29
30
  # Date received from and to
@@ -49,6 +50,17 @@ module UKPlanningScraper
49
50
  form_vars['dateStart'] = params[:decided_from].to_s if params[:decided_from] # YYYY-MM-DD
50
51
  form_vars['dateEnd'] = params[:decided_to].to_s if params[:decided_to] # YYYY-MM-DD
51
52
  end
53
+
54
+ # Status
55
+ if params[:status]
56
+ form_vars['cboStatusCode'] = params[:status]
57
+ end
58
+
59
+ # Case officer code
60
+ if params[:case_officer_code]
61
+ form_vars['cboCaseOfficerCode'] = params[:case_officer_code]
62
+ @url.sub!('GeneralSearch.aspx', 'CaseOfficerWorkloadSearch.aspx')
63
+ end
52
64
 
53
65
  logger.info "Form variables: #{form_vars.to_s}"
54
66
 
@@ -1,3 +1,3 @@
1
1
  module UKPlanningScraper
2
- VERSION = "0.4.6"
2
+ VERSION = "0.5.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: uk_planning_scraper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.6
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adrian Short
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-04-22 00:00:00.000000000 Z
11
+ date: 2020-04-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -144,6 +144,7 @@ extra_rdoc_files: []
144
144
  files:
145
145
  - ".gitignore"
146
146
  - ".rspec"
147
+ - CHANGELOG.md
147
148
  - Gemfile
148
149
  - LICENSE
149
150
  - README.md