rails_cve 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7710d41cfcbd020a20e7e80c182d9ca3dfc1833d
+  data.tar.gz: c2fd962891aa4517b740ffb40976385d28e5c485
+SHA512:
+  metadata.gz: c9a4fe45b865081da707e6c6fc582ab87fec7540e53e76b8947c18af9bd45780881a4322b9e3a23a8bb4d0fcc360c7aa2fd372f9b89e319cbbe9dd0ad89592a5
+  data.tar.gz: 557546b6e29aeaf568f33d434df27708b2fd4cf9b621bc61592af3dbbd6317b48dbd90c96cfdb9df90d2496761a22ceddcbffecb958b0c2b7c94b1699ff3a378

data/.gitignore

@@ -0,0 +1,11 @@
+.bundle/
+Gemfile.lock
+log/*.log
+pkg/
+test/dummy/data/
+test/dummy/db/*.sqlite3
+test/dummy/db/*.sqlite3-journal
+test/dummy/log/*.log
+test/dummy/tmp/
+test/dummy/.sass-cache
+*~

data/.travis.yml

@@ -0,0 +1,7 @@
+ruby:
+  - 2.0.0
+  - 2.1.1
+
+before_script:
+  - psql -U postgres -c "CREATE ROLR railscve WITH LOGIN ENCRYPTED PASSWORD 'railscve';"
+  - psql -U postgres -c "CREATE DATABASE railscve_test WITH OWNER railscve;"

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright 2014 Dominik Menke and contributors. For a full list of contributors
+see https://github.com/havainto/rails-cve/graphs/contributors.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,116 @@
+# Rails Common Vulnerability Engine
+
+If you ever need a local copy of the [CVE list][1] in your Rails application,
+you could consider using this engine.
+
+RailsCVE gives you a `RailsCVE::Entry` model, on which you can perform various
+tasks, notably searching the CVE descriptions:
+
+```ruby
+# in `rails console`
+
+list = RailsCVE::Entry.search 'ruby'
+# SELECT "rails_cve_entries".* FROM "rails_cve_entries"
+#  WHERE (to_tsvector('english', description) @@ to_tsquery('ruby'))
+#  ORDER BY ts_rank(to_tsvector(description), plainto_tsquery('ruby')) DESC
+
+list.count
+#=> 165
+
+list.last
+# #<RailsCVE::Entry
+#   id: 29523,
+#   name: "CVE-2007-6183",
+#   descri ption: "Format string vulnerability in the mdiag_initialize...",
+#   references: [
+#     "BUGTRAQ:20071127 Ruby/Gnome2 0.16.0 Format String Vulnerability",
+#     "URL:http://www.securityfocus.com/archive/1/archive/1/484240/100/0/threaded",
+#     "MISC:http://em386.blogspot.com/2007/11/your-favorite-better-than-c-scripting.html",
+#     "MISC:https://bugzilla.redhat.com/show_bug.cgi?id=402871",
+#     …,
+#     "XF:rubygnome2-mdiaginitialize-format-string(38757)",
+#     "URL:http://xforce.iss.net/xforce/xfdb/38757"
+# ]>
+```
+
+## Full text search capabilities
+
+As you can see in the example above, we take advantage of PostgreSQL's text
+search features. Hence, you will need to migrate to PostgreSQL unless you
+haven't already.
+
+I should mention, that it is **strongly recommended** to change your schema
+format from Ruby to SQL, like so:
+
+```ruby
+# in config/application.rb
+config.active_record.schema_format = :sql
+```
+
+While `RailsCVE::Entry.search` uses a predefined search query, you can, of
+course, build your own using Postgres' [text search functions and operators][2]
+
+
+## Installation
+
+Add `rails-cve` to your Gemfile and run `bundle install`
+
+```ruby
+gem 'rails-cve'
+```
+
+Then install and run the migrations via
+
+```
+rake db:migrate
+```
+
+and seed/import all currently knwon CVE entries (~70.000 at the time of
+writing):
+
+```
+rake rails_cve:seed
+```
+
+## Maintenance
+
+You should (at least daily) import the updates. RailsCVE currently uses the
+[Cassandra][3] web service. To do so, you need to call
+
+```ruby
+RailsCVE::Entry.update_entries
+```
+
+This does not only import new entries, but also updates existing entries.
+These updates usually include updates in the references list and/or
+description text, but can also mark a CVE entry as candidate (CAN) or reject
+the entry. Currently, the updating method does not provide any hooks, but
+patches are welcome!
+
+
+## Tests
+
+I'm working on it. Don't use this release unless you know what you're doing!
+
+
+## Contributing
+
+Either
+
+1. open an issue and ask for directions
+
+or
+
+1. Fork this repository.
+2. Commit your changes (tests are very welcome).
+3. Open a pull request.
+
+
+## License
+
+MIT. See LICENSE file.
+
+
+[1]: http://cve.mitre.org/
+[2]: http://www.postgresql.org/docs/9.1/static/functions-textsearch.html
+[3]: https://cassandra.cerias.purdue.edu/CVE_changes/

data/Rakefile

@@ -0,0 +1,34 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'RailsCVE'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/app/models/concerns/rails_cve/entry/presentation.rb

@@ -0,0 +1,12 @@
+module RailsCVE::Entry::Presentation
+  extend ActiveSupport::Concern
+
+  def to_s
+    name
+  end
+
+  def year
+    @year ||= name.gsub(/^CVE-(\d{4})/, '\1').to_i
+  end
+
+end

data/app/models/concerns/rails_cve/entry/references.rb

@@ -0,0 +1,22 @@
+module RailsCVE::Entry::References
+  extend ActiveSupport::Concern
+
+  def reference_uris
+    return [] unless references?
+    references.grep(/^URL:/).map{|r| r.gsub(/^URL:/, '') }
+  end
+
+  def references=(refs)
+    case refs
+      when Array
+        references_will_change!
+        super refs.flatten
+      when String
+        references_will_change!
+        super refs.split('   |   ')
+      else
+        raise ArgumentError, 'invalid type'
+    end
+  end
+
+end

data/app/models/concerns/rails_cve/entry/search.rb

@@ -0,0 +1,14 @@
+module RailsCVE::Entry::Search
+  extend ActiveSupport::Concern
+
+  included do
+    scope :search, ->(query){
+      q = sanitize(query)
+
+      rank = "ts_rank(to_tsvector(description), plainto_tsquery(%{q}))" % { q: q }
+      cond = "to_tsvector('english', description) @@ to_tsquery(%{q})"  % { q: q }
+
+      where(cond).order("#{rank} desc")
+    }
+  end
+end

data/app/models/concerns/rails_cve/entry/updater.rb

@@ -0,0 +1,43 @@
+module RailsCVE::Entry::Updater
+  extend ActiveSupport::Concern
+
+  DATA_PATH = if Rails.env.production?
+    Rails.root.join('data/cve-list')
+  else
+    Rails.root.join("data/cve-list-#{Rails.env}")
+  end
+
+  module ClassMethods
+
+    def rebuild_entries!
+      transaction do
+        RailsCVE::Utils.fetch_all_entries do |name, desc, ref|
+          entry = create_or_update_with_data! name, desc, ref
+          yield entry if block_given?
+        end.size
+      end
+    end
+
+    def update_entries
+      now = Time.current
+
+      transaction do
+        RailsCVE::Utils.fetch_updates(now.year, now.month) do |name, desc, ref|
+          create_or_update_with_data! name, desc, ref
+        end.size
+      end
+    end
+
+    def create_or_update_with_data!(name, description, references)
+      entry = where(name: name).first_or_initialize
+      entry.description = description
+      entry.references  = references
+      entry.save!
+      entry
+    rescue ActiveRecord::RecordNotUnique
+      retry
+    end
+
+  end
+
+end

data/app/models/rails_cve/entry.rb

@@ -0,0 +1,10 @@
+module RailsCVE
+  class Entry < ActiveRecord::Base
+    include RailsCVE::Entry::Updater
+    include RailsCVE::Entry::Presentation
+    include RailsCVE::Entry::References
+    include RailsCVE::Entry::Search
+
+    validates :name, presence: true, uniqueness: true
+  end
+end

data/bin/rails

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails 4 gems installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('../..', __FILE__)
+ENGINE_PATH = File.expand_path('../../lib/rails_cve/engine', __FILE__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+
+require 'rails/all'
+require 'rails/engine/commands'

data/db/migrate/20140402144115_create_rails_cve_entries.rb

@@ -0,0 +1,31 @@
+class CreateRailsCveEntries < ActiveRecord::Migration
+  def up
+    create_table :rails_cve_entries do |t|
+      t.string  :name,        null: false
+      t.text    :description
+      t.string  :references,  array: true, limit: 1024
+    end
+
+    add_index :rails_cve_entries, :name, unique: true
+
+    # It is recommended to set
+    #
+    #   config.active_record.schema_format = :sql
+    #
+    # in your config/application.rb. If you don't and you just run
+    # `rake db:schema:load` later on, you'll end up having REALLY SLOW search
+    # queries.
+    execute <<-SQL.squish
+      CREATE INDEX
+        cve_entry_text_search ON rails_cve_entries
+      USING
+        gin(to_tsvector('english', description))
+    SQL
+  end
+
+  def down
+    execute "DROP INDEX cve_entry_text_search"
+    remove_index :rails_cve_entries, :name
+    drop_table :rails_cve_entries
+  end
+end

data/lib/rails_cve/engine.rb

@@ -0,0 +1,13 @@
+module RailsCVE
+  class Engine < ::Rails::Engine
+    isolate_namespace RailsCVE
+
+    initializer :append_migrations do |app|
+      unless app.root.to_s.match root.to_s
+        config.paths["db/migrate"].expanded.each do |expanded_path|
+          app.config.paths["db/migrate"] << expanded_path
+        end
+      end
+    end
+  end
+end

data/lib/rails_cve/utils.rb

@@ -0,0 +1,100 @@
+require 'csv'
+require 'nokogiri'
+require 'httparty'
+
+module RailsCVE
+  Error         = Class.new(StandardError)
+  UnknownTarget = Class.new(ArgumentError)
+
+  module Utils
+    extend self
+
+    # Source of full CVE list
+    def allitems_source(basename)
+      "http://cve.mitre.org/data/downloads/#{basename}.gz"
+    end
+
+    # Source for updates
+    def updates_source(basename)
+      "https://cassandra.cerias.purdue.edu/CVE_changes/#{basename}"
+    end
+
+    # Path for downloads, needs to be writeable
+    def data_path(target)
+      RailsCVE::Entry::DATA_PATH.join(target).tap {|path| path.dirname.mkpath }
+    end
+
+    # Downloads latest allitems.cvs(.gz) from cve.mitre.org and yields name,
+    # description and references list of every CVE entry
+    def fetch_all_entries(&block)
+      file = data_path('allitems.csv')
+      download allitems_source(file.basename), to: file
+
+      latch_locked = true
+      result = []
+
+      CSV.parse(file.read) do |row|
+        # there are some comments prefacing the entry list
+        latch_locked = false if row[0].nil?
+        next if latch_locked || row[0].nil?
+
+        name, _, description, references, _, _, _ = row
+
+        if block_given?
+          result << yield(name, description, references)
+        else
+          result << [name, description, references]
+        end
+      end
+
+      result
+    end
+
+    # Fetches the updates for a given year/month from Cassandra and
+    # yields name, desc. and refs. for each CVE change
+    def fetch_updates(year, month, &block)
+      list = data_path('CVE.%4d.%02d.html' % [year.to_i, month.to_i])
+      download updates_source(list.basename), to: list
+
+      # Sadly, there is only little structure in the downloaded list.
+      Nokogiri::HTML(list.read).css('a').map do |a|
+        url = URI.parse(a[:href])
+        next unless url.host == 'cve.mitre.org'
+        next unless url.query =~ /name=(\d{4})-(\d{4,})/
+
+        y,id  = $1, $2
+        name  = "CVE-#{y}-#{id}"
+
+        # max. 2 digits per dirname → max. 100 subdirs per directory
+        parts = id.to_s.scan(/\d\d/).join('/')
+
+        # for now, we fetch *each* embedded link, which could take a while…
+        file = data_path("entries/#{y}/#{parts}/#{name}.html")
+        download a[:href], to: file
+
+        table       = Nokogiri::HTML(file.read).xpath('//div[@id="GeneratedTable"]/table')
+        description = table.xpath('.//tr[4]/td').text
+        references  = table.xpath('.//tr[7]//li').map(&:text)
+
+        yield name, description, references
+      end
+    end
+
+    def download(url, to: nil)
+      unless to.present? && Pathname === to
+        raise UnknownTarget, '"to" parameter needs to be Pathname instance'
+      end
+
+      response = HTTParty.get(url, verify: false)
+      return false if response.code != 200
+
+      data = response.body.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+      to.open('w') {|f| f.write data }
+
+      true
+    rescue HTTParty::Error, Net::HTTPError
+      return false
+    end
+  end
+
+end

data/lib/rails_cve/version.rb

@@ -0,0 +1,3 @@
+module RailsCVE
+  VERSION = "0.0.1"
+end

data/lib/rails_cve.rb

@@ -0,0 +1,5 @@
+require "rails_cve/engine"
+require "rails_cve/utils"
+
+module RailsCVE
+end

data/lib/tasks/rails_cve_tasks.rake

@@ -0,0 +1,12 @@
+namespace :rails_cve do
+
+  desc "Seed the database"
+  task seed: :environment do
+    puts '(Re-) Importing CVE list. This could take a while...'
+    n = RailsCVE::Entry.rebuild_entries! do |entry|
+      puts "\t#{entry.name} imported"
+    end
+    puts "Done! Imported #{n} CVE entries."
+  end
+
+end

data/rails_cve.gemspec

@@ -0,0 +1,31 @@
+$:.push File.expand_path('../lib', __FILE__)
+
+# Maintain your gem's version:
+require 'rails_cve/version'
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = 'rails_cve'
+  s.version     = RailsCVE::VERSION
+  s.authors     = 'Dominik Menke'
+  s.email       = 'dominik.menke@gmail.com'
+  s.homepage    = 'https://havain.to'
+  s.summary     = 'Rails Common Vulnerability Engine'
+
+
+  s.files       = `git ls-files -z`.split("\x0")
+  s.test_files  = s.files.grep(%r{^(test|spec|features)/})
+
+  #s.files       = Dir['{app,config,db,lib}/**/*', 'MIT-LICENSE', 'Rakefile', 'README.rdoc']
+  #s.test_files  = Dir['test/**/*']
+
+  s.add_dependency 'rails', '~> 4.0.4'
+  s.add_dependency 'nokogiri'
+  s.add_dependency 'httparty'
+  s.add_dependency 'pg'
+
+  s.add_development_dependency 'pry-byebug'
+  s.add_development_dependency 'pry-rails'
+  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'webmock'
+end

data/test/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Dummy::Application.load_tasks

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,13 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
+ * compiled file, but it's generally better to create a new file per style scope.
+ *
+ *= require_self
+ *= require_tree .
+ */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    "application", media: "all" %>
+  <%= javascript_include_tag "application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/test/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

data/test/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/test/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/test/dummy/config/application.rb

@@ -0,0 +1,28 @@
+require File.expand_path('../boot', __FILE__)
+
+# Pick the frameworks you want:
+require "active_record/railtie"
+#require "action_controller/railtie"
+#require "action_mailer/railtie"
+# require "sprockets/railtie"
+require "rails/test_unit/railtie"
+
+Bundler.require(*Rails.groups)
+require "rails_cve"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+  end
+end
+

data/test/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/test/dummy/config/database.yml

@@ -0,0 +1,16 @@
+development: &local
+  adapter:        postgresql
+  encoding:       unicode
+  database:       railscve_dev
+  pool:           5
+  username:       railscve
+  password:       railscve
+
+  #host:           localhost
+  #port:           5432
+  min_messages:   warning
+
+test:
+  <<:             *local
+  database:       railscve_test
+  min_messages:   notice