commons-integrity 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b9e522abef1caaeb6fd5b7fa1839bb8428c404d5
+  data.tar.gz: 3f90f3f405be99d6469f1833fccc2e4adaa1117d
+SHA512:
+  metadata.gz: 1c7cd33fe3d7844c6e90e5a28085cfcdb5cff2526a62a4be27e16a15b076e21dc8a355eb77e363b3fd4f4179be95012b31e10ea51822567754d01f1a061365a4
+  data.tar.gz: d154d5216ab6ce881ee07b42b27d6baaa0c00a3fb00d7d145433bc1a9e752b24a0474e485f6b73461925f501005349510c0408d38aa29fbee520db360abc34e5

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  Exclude:
+    - 'Vagrantfile'
+    - 'vendor/**/*'
+  TargetRubyVersion: 2.4
+
+inherit_from:
+  - https://raw.githubusercontent.com/everypolitician/everypolitician-data/master/.rubocop_base.yml

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+cache: bundler
+rvm: 2.4
+script:
+  - bundle exec rake
+  - bash <(curl -fsSL https://github.com/everypolitician/ensure-regression-tests/raw/master/ensure-regression-tests)

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+ruby '~> 2.4.0'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in commons-builder.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,97 @@
+PATH
+  remote: .
+  specs:
+    commons-integrity (0.1)
+      activesupport
+      json
+      require_all
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    axiom-types (0.1.1)
+      descendants_tracker (~> 0.0.4)
+      ice_nine (~> 0.11.0)
+      thread_safe (~> 0.3, >= 0.3.1)
+    codeclimate-engine-rb (0.4.1)
+      virtus (~> 1.0)
+    coderay (1.1.2)
+    coercible (1.0.0)
+      descendants_tracker (~> 0.0.1)
+    concurrent-ruby (1.0.5)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    equalizer (0.0.11)
+    hashdiff (0.3.7)
+    i18n (1.0.1)
+      concurrent-ruby (~> 1.0)
+    ice_nine (0.11.2)
+    json (2.1.0)
+    method_source (0.9.0)
+    minitest (5.11.3)
+    parallel (1.12.1)
+    parser (2.5.1.0)
+      ast (~> 2.4.0)
+    powerpack (0.1.1)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    public_suffix (3.0.2)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    reek (4.8.0)
+      codeclimate-engine-rb (~> 0.4.0)
+      parser (>= 2.5.0.0, < 2.6)
+      rainbow (~> 3.0)
+    require_all (2.0.0)
+    rubocop (0.55.0)
+      parallel (~> 1.10)
+      parser (>= 2.5)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
+    safe_yaml (1.0.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.3.2)
+    virtus (1.0.5)
+      axiom-types (~> 0.1)
+      coercible (~> 1.0)
+      descendants_tracker (~> 0.0, >= 0.0.3)
+      equalizer (~> 0.0, >= 0.0.9)
+    webmock (2.3.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  commons-integrity!
+  minitest (~> 5.0)
+  pry
+  rake (~> 10.0)
+  reek
+  rubocop
+  webmock (~> 2.0)
+
+RUBY VERSION
+   ruby 2.4.1p111
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -0,0 +1,56 @@
+# commons-integrity
+
+[![Build Status](https://travis-ci.org/everypolitician/commons-integrity.svg?branch=master)](https://travis-ci.org/everypolitician/commons-integrity)
+
+## Usage
+
+### Installation
+
+Add to the `Gemfile` dependencies for a proto-commons- repository:
+
+    gem 'commons-integrity', :github => 'everypolitician/commons-integrity'
+
+### Configuration
+
+Each check you wish to run within your project should be listed in a
+configuration file (by default `.integrity.yml` in the project's root
+directory, though this can also be specified separately.)
+
+This will usually specify which file(s) to run against, and any options
+specific to that individual check.
+
+For example:
+
+```yaml
+  WikidataIdentifiers:
+    AppliesTo: 'boundaries/**/*.csv'
+    column_name: 'WIKIDATA'
+    column_case: 'fixed'
+```
+
+## Orchestration
+
+Currently you need to create your own script to run this, and choose how
+to display the errors. We plan to make both of these much simpler.
+
+An example `bin/check` could do something like:
+
+```ruby
+  require 'commons/integrity'
+
+  root = Pathname.new(ARGV.first || '.')
+  files = Pathname.glob(root + '**/*')
+  errors = files.map { |file| Commons::Integrity::Report.new(file: file).errors }
+  puts errors
+```
+
+## Creating new Checks
+
+Add new checks in `lib/commons/integrity/check/` such that they look
+like existing checks.
+
+Each should inherit from `Commons::Integrity::Check::Base` and supply
+its own `errors` method.
+
+Documentation for the Check should be added in [YARD](https://yardoc.org/)
+format.

data/Rakefile

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'rake/testtask'
+require 'reek/rake/task'
+require 'rubocop/rake_task'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.warning = true
+  t.verbose = true
+  t.test_files = FileList['test/**/*.rb']
+end
+
+RuboCop::RakeTask.new
+Reek::Rake::Task.new
+
+task default: %w[test rubocop reek]

data/commons-integrity.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name          = 'commons-integrity'
+  spec.version       = '0.1'
+  spec.authors       = ['Tony Bowden', 'Alex Dutton']
+  spec.email         = ['parliaments@mysociety.org']
+
+  spec.summary       = 'Check the integrity of Democratic Commons data'
+  spec.homepage      = 'https://github.com/everypolitician/commons-integrity'
+  spec.license       = 'MIT'
+
+  spec.required_ruby_version = '~> 2.4.0'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir = 'bin'
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'json'
+  spec.add_dependency 'require_all'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'reek'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'webmock', '~> 2.0'
+end

data/lib/commons/integrity.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'require_all'
+
+require_rel 'integrity/check'
+require_rel 'integrity/report'
+
+module Commons
+  # This module contains all the library code for checking the
+  # integrity of data.
+  module Integrity
+  end
+end

data/lib/commons/integrity/check/area_positions_known.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'csv'
+require 'json'
+
+module Commons
+  module Integrity
+    class Check
+      # Ensure we have rich position data for positions associated with boundaries
+      #
+      # This check should be applied to a
+      # boundaries/build/position-data.json file. It'll check
+      # that that file has data for every position mentioned in
+      # the index.json file in the same directory.
+      #
+      # == Configuration Options
+      class AreaPositionsKnown < Base
+        # @return [Array<Error>]
+        # Errors will be in the category `:position_id_message`
+        def errors
+          positions_missing_role_data.map do |position|
+            error(
+              position_id_message: "#{position} was found in #{boundaries_index_pathname} but not in #{pathname}"
+            )
+          end
+        end
+
+        private
+
+        def positions_missing_role_data
+          (position_items_from_index - positions_with_role_data).sort
+        end
+
+        def boundaries_index_pathname
+          pathname.dirname.join('index.json')
+        end
+
+        def boundaries_index_data
+          @boundaries_index_data ||= JSON.parse(
+            boundaries_index_pathname.read,
+            symbolize_names: true
+          )
+        end
+
+        def position_items_from_index
+          Set.new(
+            boundaries_index_data.flat_map do |entry|
+              BoundaryIndexEntry.new(entry).associated_positions
+            end
+          )
+        end
+
+        def positions_with_role_data
+          Set.new(
+            JSON.parse(pathname.read, symbolize_names: true).map do |role|
+              role[:role_id]
+            end
+          )
+        end
+      end
+    end
+
+    # This encapsulates a top-level entry in the JSON file which
+    # acts as an index of the boundary data directories
+    class BoundaryIndexEntry
+      def initialize(entry_data)
+        @entry_data = entry_data
+      end
+
+      def associated_positions
+        entry_data[:associations].map { |association| association[:position_item_id] }
+      end
+
+      private
+
+      attr_reader :entry_data
+    end
+  end
+end

data/lib/commons/integrity/check/base.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Commons
+  module Integrity
+    class Check
+      # All other checks should inherit from here, and override `errors`
+      class Base
+        def initialize(filename, config: nil)
+          @filename = filename
+          @given_config = config
+        end
+
+        def errors
+          []
+        end
+
+        def self.moniker
+          name.sub('Commons::Integrity::Check::', '')
+        end
+
+        private
+
+        attr_reader :filename, :given_config
+
+        # Simple struct to represent any errors raised
+        Error = Struct.new(:category, :message, :filename)
+
+        def error(pair)
+          Error.new(*pair.first, filename)
+        end
+
+        def config
+          @config ||= @given_config || Config.new
+        end
+
+        def pathname
+          @pathname ||= Pathname.new(filename)
+        end
+
+        def my_config
+          config.for(self.class.moniker)
+        end
+      end
+    end
+  end
+end

data/lib/commons/integrity/check/wikidata_identifiers.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'csv'
+
+module Commons
+  module Integrity
+    class Check
+      # Check that any values in a "wikidata" column look like valid IDs
+      #
+      # Given a CSV file with a column of Wikidata identifiers, this
+      # Check will ensure that all the values in that column look like
+      # valid identifiers (i.e. are Q-numbers). It does *not* check that
+      # they are _actually_ valid IDs: only that they are of the correct
+      # form.
+      #
+      # == Configuration Options
+      # * column_name: the column containing the IDs (default: "wikidata")
+      # * column_case: "fixed" if the column_name must be exactly as specified (default: "any")
+      class WikidataIdentifiers < Base
+        # @return [Array<Error>]
+        # Errors will be in the category `:wikidata_id_format`
+        def errors
+          problematic_values.map { |val| error(wikidata_id_format: "Invalid wikidata ID: #{val}") }
+        end
+
+        private
+
+        DEFAULT_COLUMN_NAME = 'wikidata'
+        DEFAULT_COLUMN_CASE = 'any'
+
+        def csv
+          @csv ||= CSV.read(pathname, headers: true)
+        end
+
+        def headers
+          csv.headers
+        end
+
+        def column_name
+          my_config.to_h['column_name'] || DEFAULT_COLUMN_NAME
+        end
+
+        def comparison_type
+          my_config.to_h['column_case'] || DEFAULT_COLUMN_CASE
+        end
+
+        def comparison_conversion
+          return :to_s if comparison_type == 'fixed'
+          :downcase
+        end
+
+        def wikidata_column
+          headers.find { |header| header.send(comparison_conversion) == column_name.send(comparison_conversion) }
+        end
+
+        def wikidata_column?
+          wikidata_column
+        end
+
+        def wikidata_values
+          return [] unless wikidata_column?
+          csv.map { |row| row[wikidata_column] }
+        end
+
+        def problematic_values
+          wikidata_values.reject { |value| value =~ /^Q[1-9][0-9]*$/ }
+        end
+      end
+    end
+  end
+end

data/lib/commons/integrity/config.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Commons
+  module Integrity
+    # This represents the configuration. For now we only have a single
+    # config file, but I expect this will become more complex later, and
+    # will likely need split into an abstract `Config` combining
+    # multiple `Config::File` objects.
+    class Config
+      def initialize(supplied_location = nil)
+        @supplied_location = supplied_location
+      end
+
+      def for(check)
+        yaml[check]
+      end
+
+      private
+
+      attr_reader :supplied_location
+
+      DEFAULT_LOCATION = Pathname.pwd + '.integrity.yml'
+
+      def yaml
+        return {} unless config_exists?
+        @yaml ||= YAML.load_file(pathname)
+      end
+
+      def possible_file_locations
+        [supplied_location, DEFAULT_LOCATION]
+      end
+
+      def pathname
+        possible_file_locations.compact.find { |file| Pathname.new(file).exist? }
+      end
+
+      def config_exists?
+        pathname
+      end
+    end
+  end
+end

data/lib/commons/integrity/report.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/class/subclasses'
+require 'require_all'
+
+require_relative 'config'
+require_rel 'check'
+
+module Commons
+  module Integrity
+    # Collate the errors from all reports applying to a file
+    class Report
+      def initialize(file:, config: nil)
+        @file = Pathname.new(file)
+        @config = config
+      end
+
+      def errors
+        relevant_checks.flat_map { |check| check.new(file).errors }
+      end
+
+      private
+
+      attr_reader :file, :config
+
+      ALL_CHECKS = Commons::Integrity::Check::Base.descendants
+
+      def relevant_checks
+        return [] unless config
+        ALL_CHECKS.select do |check|
+          check_config = config.for(check.moniker)
+          if check_config
+            pattern = check_config.dig('AppliesTo')
+            file.fnmatch pattern if pattern
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,198 @@
+--- !ruby/object:Gem::Specification
+name: commons-integrity
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Tony Bowden
+- Alex Dutton
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-06-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: require_all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: reek
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: 
+email:
+- parliaments@mysociety.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- commons-integrity.gemspec
+- lib/commons/integrity.rb
+- lib/commons/integrity/check/area_positions_known.rb
+- lib/commons/integrity/check/base.rb
+- lib/commons/integrity/check/wikidata_identifiers.rb
+- lib/commons/integrity/config.rb
+- lib/commons/integrity/report.rb
+homepage: https://github.com/everypolitician/commons-integrity
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: Check the integrity of Democratic Commons data
+test_files: []