uk_planning_scraper 0.4.6 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 340c6c3789dd7a3f62308e0a603bd577c118560692c0132b348f93160bf9224d
-  data.tar.gz: be4157ac9594b467f1f8a998ecfbe44c3c948b0bf837bd2bbb98d15726af879b
+  metadata.gz: b54d09a91d835f0bc424cae3deb89bfad0cbb5ce0814d7881ee748beed370f70
+  data.tar.gz: 6146259451950c5ef52f572ccc69fe2cc3fce76c9a7d67b8965c71a2d20f4d45
 SHA512:
-  metadata.gz: add97d946b69f366af36aa499b7f27f15cf04024c6a2eac2561ea3e1fc8cfe5853ebb6ea65e8962ff27010cd46f09c4129a6be4ad453c7cde6fc32717c24c071
-  data.tar.gz: eb3310870c252a74289478edaffc755e695f715cb25ee24a202a2ba487fba32d583541263fdd2e3a11cb540f2bc1e60fa944941c4f7f63a0a2a0eb83a68f9c82
+  metadata.gz: d274b2593dbb73e8b16cf2b220d7be945cd3c744aa7a9f19526ce828263f6847dd6931a4a27c38df97cdac227e2c3d2c4d76754dfc88f093d96079f26848760f
+  data.tar.gz: 0c8a4de8c1ceb9f77875e12e6895dfc7d89911d35a712cccfab66a08c6d19e92f070e9bf7ebee038d3681474cb73ce5f2baa6e8b68705f3e615113b797821e70

data/CHANGELOG.md

@@ -0,0 +1,87 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased](https://github.com/adrianshort/uk_planning_scraper/compare/v0.5.0...master)
+
+This functionality is on GitHub but not yet in the gem.
+
+## [0.5.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.6...v0.5.0) - 2020-04-30
+
+### Added
+
+- Case officer search for Northgate
+- Status search for Idox and Northgate
+
+## [0.4.6](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.5...v0.4.6) - 2020-04-22
+
+### Fixed
+
+- Major change to Northgate output semantics. Scraper previously returned `date_received` from the search results pages. Now it returns `date_validated` which is correct. Previously collected data needs to be moved to this new column or rescraped. [#40](https://github.com/adrianshort/uk_planning_scraper/issues/40)
+- Upgraded `rake` to fix [CVE-2020-8130](http://cve.circl.lu/cve/CVE-2020-8130).
+
+## [0.4.5](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.4...v0.4.5) - 2020-02-24
+
+### Added
+
+- Gateshead Council.
+
+## [0.4.4](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.3...v0.4.4) - 2019-10-29
+
+### Fixed
+
+- URLs for Camden, Islington and Merton councils.
+
+## [0.4.3](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.2...v0.4.3) - 2019-01-22
+
+### Changed
+
+- `uk_planning_scraper` [published as a gem on RubyGems](https://rubygems.org/gems/uk_planning_scraper/versions/0.4.5) for the first time.
+
+## [0.4.2](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.1...v0.4.2) - 2019-01-08
+
+### Changed
+
+- Kingston becomes Idox.
+
+### Fixed
+- Bradford and Leeds tagged `westyorkshire` not `westmidlands`.
+
+## [0.4.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.4.0...v0.4.1) - 2018-10-18
+
+### Added
+
+- 39 new councils.
+
+### Changed
+
+- Use spaces not commas to separate tags in `authorities.csv` file.
+- Use `pry` not `irb` as default console.
+
+## [0.4.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.2...v0.4.0) - 2018-10-12
+
+### Changed
+
+- Use chained methods rather than an options hash for the scraper parameters. This permits better error checking. [#15](https://github.com/adrianshort/uk_planning_scraper/issues/15)
+
+## [0.3.2](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.1...v0.3.2) - 2018-10-10
+
+## [0.3.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.3.0...v0.3.1) - 2018-10-10
+
+## [0.3.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.2.0...v0.3.0) - 2019-10-09
+
+## [0.2.0](https://github.com/adrianshort/uk_planning_scraper/compare/v0.1.1...v0.2.0) - 2018-09-19 
+
+## [0.1.1](https://github.com/adrianshort/uk_planning_scraper/compare/v0.1.0...v0.1.1) - 2018-09-15
+
+### Fixed
+
+-  [#3](https://github.com/adrianshort/uk_planning_scraper/issues/3)
+
+## [0.1.0](https://github.com/adrianshort/uk_planning_scraper/releases/tag/v0.1.0) - 2018-09-14
+
+### Added
+
+- Basic Idox scraper.

data/README.md

@@ -106,6 +106,8 @@ keywords("hip gable")
 applicant_name("Mr and Mrs Smith") # Currently Idox only
 application_type("Householder") # Currently Idox only
 development_type("") # Currently Idox only
+case_officer_code("100000") # Northgate only
+status("Pending Consideration") # Check valid status codes for each authority
 
 scrape # runs the scraper
 ```

data/lib/uk_planning_scraper/authority.rb

@@ -70,7 +70,7 @@ module UKPlanningScraper
     def system
       if @url.match(/search\.do\?action=advanced/i)
         'idox'
-      elsif @url.match(/generalsearch\.aspx/i)
+      elsif @url.match(/\.aspx/i)
         'northgate'
       elsif @url.match(/ocellaweb/i)
         'ocellaweb'

data/lib/uk_planning_scraper/authority_scrape_params.rb

@@ -62,6 +62,17 @@ module UKPlanningScraper
       self
     end
 
+    def case_officer_code(s)
+      unless system == 'northgate'
+        raise NoMethodError.new("case_officer_code is only implemented for Northgate. \
+          This authority (#{@name}) is #{system.capitalize}.")
+      end
+      
+      check_class(s, String)
+      @scrape_params[:case_officer_code] = s.strip
+      self
+    end
+
     def application_type(s)
       unless system == 'idox'
         raise NoMethodError.new("application_type is only implemented for \
@@ -84,6 +95,12 @@ module UKPlanningScraper
       self
     end
 
+    def status(s)
+      check_class(s, String)
+      @scrape_params[:status] = s.strip
+      self
+    end
+
     private
     
     # Handle the simple params with this

data/lib/uk_planning_scraper/idox.rb

@@ -42,6 +42,7 @@ module UKPlanningScraper
       form.send(:"date(applicationDecisionEnd)", params[:decided_to].strftime(date_format)) if params[:decided_to]
 
       form.send(:"searchCriteria\.description", params[:keywords])
+      form.send(:"searchCriteria\.caseStatus", params[:status])
       
       # Some councils don't have the applicant name on their form, eg Bexley
       form.send(:"searchCriteria\.applicantName", params[:applicant_name]) if form.has_field? 'searchCriteria.applicantName'

data/lib/uk_planning_scraper/northgate.rb

@@ -24,6 +24,7 @@ module UKPlanningScraper
         'csbtnSearch' => 'Search' # required
       }
 
+      # Keywords
       form_vars['txtProposal'] = params[:keywords]
 
       # Date received from and to
@@ -49,6 +50,17 @@ module UKPlanningScraper
         form_vars['dateStart'] = params[:decided_from].to_s if params[:decided_from] # YYYY-MM-DD
         form_vars['dateEnd'] = params[:decided_to].to_s if params[:decided_to] # YYYY-MM-DD
       end
+      
+      # Status
+      if params[:status]
+        form_vars['cboStatusCode'] = params[:status]
+      end
+
+      # Case officer code
+      if params[:case_officer_code]
+        form_vars['cboCaseOfficerCode'] = params[:case_officer_code]
+        @url.sub!('GeneralSearch.aspx', 'CaseOfficerWorkloadSearch.aspx')
+      end
 
       logger.info "Form variables: #{form_vars.to_s}"
 

data/lib/uk_planning_scraper/version.rb

@@ -1,3 +1,3 @@
 module UKPlanningScraper
-  VERSION = "0.4.6"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: uk_planning_scraper
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.5.0
 platform: ruby
 authors:
 - Adrian Short
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2020-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -144,6 +144,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md