middleman-crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 304f53a614e91d793893025e8c7e9e1bd97de1a6
+  data.tar.gz: 991befa646235f146eb996f0608786130b62095a
+SHA512:
+  metadata.gz: 785c85c945369332430d67a6bd139103eeadcfb6d166baae3aead4b21576247ad1764a8115516d9d4f2dc35df6c78b8d44921ee14f3879f97f3461f73b18ef97
+  data.tar.gz: c7e800ab263c448a8526245e5c0e8c3a7c8302c4d584735c77748a99af37f147294fa7e85684307e585e671eeea4dd377c24dbf0897250442f49dfb6dfbb7d7a

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,14 @@
+# If you do not have OpenSSL installed, update
+# the following line to use "http://" instead
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'rake'
+  gem 'rdoc'
+  gem 'yard'
+end
+
+group :test do
+end

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Filippo Gangi Dino
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# Middleman::Crawler
+
+Add a crawler CLI to crawl your middleman site.
+
+[![Gem Version](https://badge.fury.io/rb/middleman-crawler.png)](http://badge.fury.io/rb/middleman-crawler)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'middleman-crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install middleman-crawler
+
+## Usage
+
+First you need to activate the module in your `config.rb`.
+
+```
+activate :crawler
+```
+
+After you must declare in your `environments/*.rb` this config var:
+
+```
+config[:base_url] = "http://someurl.here"
+```
+
+This extension use [Rawler Gem](https://github.com/oscardelben/rawler) and implements the same interface with one more option `environment`.
+
+In CLI you can write `middleman crawler http://example.com [options]`
+
+      where [options] are:
+        --environment, -e <s>:   The Middleman Enviroment
+        --username, -u <s>:   HTTP Basic Username
+        --password, -p <s>:   HTTP Basic Password
+            --wait, -w <f>:   Seconds to wait between requests, may be fractional e.g. '1.5' (default: 3.0)
+                 --log, -l:   Log results to file rawler_log.txt
+         --logfile, -o <s>:   Specify logfile, implies --log (default: rawler_log.txt)
+                 --css, -c:   Check CSS links
+            --skip, -s <s>:   Skip URLs that match a regexp
+           --iskip, -i <s>:   Skip URLs that match a case insensitive regexp
+             --include <s>:   Only include URLs that match a regexp
+            --iinclude <s>:   Only include URLs that match a case insensitive regexp
+               --local <s>:   Restrict to the given URL and below. Equivalent to '--include ^http://mysite.com/*'.
+        --ignore_fragments:   Strips any fragment from parsed links
+
+
+## Contributing
+
+1. Fork it ( https://github.com/welaika/middleman-crawler )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+require 'bundler'
+Bundler.setup
+Bundler::GemHelper.install_tasks

data/features/support/env.rb

@@ -0,0 +1,4 @@
+PROJECT_ROOT_PATH = File.dirname(File.dirname(File.dirname(__FILE__)))
+require 'middleman-core'
+require 'middleman-core/step_definitions'
+require File.join(PROJECT_ROOT_PATH, 'lib', 'middleman-crawler')

data/lib/middleman-crawler.rb

@@ -0,0 +1,8 @@
+require 'middleman-core'
+
+require 'middleman-crawler/commands'
+
+Middleman::Extensions.register(:crawler) do
+  require 'middleman-crawler/extension'
+  Crawler
+end

data/lib/middleman-crawler/commands.rb

@@ -0,0 +1,169 @@
+require 'middleman-core/cli'
+require 'middleman-core/rack' if Middleman::VERSION.to_i > 3
+require 'middleman-crawler/extension'
+
+require 'rawler'
+
+module Middleman
+  module Cli
+    # This class provides a "crawler" command for the middleman CLI.
+    class CrawlerIndex < Thor::Group
+      include Thor::Actions
+
+      check_unknown_options!
+
+      namespace :crawler
+
+      class_option :environment,
+                   aliases: '-e',
+                   type: :string,
+                   default: ENV['MM_ENV'] || ENV['RACK_ENV'] || 'development',
+                   desc: 'The environment Middleman will run under'
+
+      class_option :username,
+                   aliases: '-u',
+                   type: :string,
+                   desc: 'HTTP Basic Username'
+
+      class_option :password,
+                   aliases: '-p',
+                   type: :string,
+                   desc: 'HTTP Basic Password'
+
+      class_option :skip,
+                   aliases: '-s',
+                   type: :string,
+                   default: "\#$",
+                   desc: 'Skip URLs that match a regexp'
+
+      class_option :wait,
+                   aliases: '-w',
+                   type: :numeric,
+                   default: 0.1,
+                   desc: "Seconds to wait between requests, may be fractional e.g. '1.5' (default: 3.0)"
+
+      class_option :css,
+                   aliases: '-c',
+                   type: :boolean,
+                   default: true,
+                   desc: "Check CSS links"
+
+      class_option :local,
+                   type: :boolean,
+                   default: true,
+                   desc: "Restrict to the given URL and below. Equivalent to '--include ^http://mysite.com/*'."
+
+      class_option :log,
+                   aliases: '-l',
+                   type: :boolean,
+                   desc: "Log results to file rawler_log.txt"
+
+      class_option :logfile,
+                   aliases: '-o',
+                   type: :string,
+                   desc: "Specify logfile, implies --log (default: rawler_log.txt)"
+
+      class_option :iskip,
+                   aliases: '-i',
+                   type: :string,
+                   desc: "Skip URLs that match a case insensitive regexp"
+
+      class_option :include,
+                   type: :string,
+                   desc: "Only include URLs that match a regexp"
+
+      class_option :iinclude,
+                   type: :string,
+                   desc: "Only include URLs that match a case insensitive regexp"
+
+      class_option :ignore_fragments,
+                   type: :boolean,
+                   desc: "Strips any fragment from parsed links"
+
+
+      # Tell Thor to exit with a nonzero exit code on failure
+      def self.exit_on_failure?
+        true
+      end
+
+      def crawler
+        configs = get_configs
+
+        env_config = File.join(Middleman::Application.root, 'environments', "#{configs[:environment]}.rb")
+        return unless File.exist? env_config
+
+        configurations = Configure.new
+
+        configurations.instance_eval File.read(env_config), env_config, 1
+
+        response = {}
+        raw = Rawler::Base.new(configurations.config[:base_url], $stdout, rawler_options)
+        raw.validate
+        response['errors'] = raw.responses.reject{ |link, response|
+          (100..399).include?(response[:status].to_i)
+        }
+        say("There are some errors: #{errors_message_for(response['errors'])}", :red) if response['errors']
+        puts "All crawling actions ok" unless response['errors']
+      end
+
+      private
+
+      def get_configs
+        configs ||= {
+          environment: (options['environment'].presence || "development").to_sym,
+          username: options['username'],
+          password: options['password'],
+          skip: options['skip'],
+          wait: options['wait'],
+          css: options['css'],
+          local: options['local'],
+          log: options['log'],
+          logfile: options['logfile'],
+          iskip: options['iskip'],
+          include: options['include'],
+          iinclude: options['iinclude'],
+          ignore_fragments: options['ignore_fragments']
+        }
+      end
+
+      def rawler_options
+        opts = {}
+        opts[:username] = get_configs[:username].to_s if get_configs[:username].present?
+        opts[:password] = get_configs[:password].to_s if get_configs[:password].present?
+        opts[:skip] = get_configs[:skip].present? ? get_configs[:skip].to_s : "\#$"
+        opts[:wait] = get_configs[:wait].present? ? get_configs[:wait].to_f : 0.1
+        opts[:css] = get_configs[:css] != nil ? get_configs[:css] : true
+        opts[:local] = get_configs[:local] != nil ? get_configs[:local] : true
+        opts[:log] = get_configs[:log] if get_configs[:log] != nil
+        opts[:logfile] = get_configs[:logfile].to_s if get_configs[:logfile].present?
+        opts[:iskip] = get_configs[:iskip].to_s if get_configs[:iskip].present?
+        opts[:include] = get_configs[:include].to_s if get_configs[:include].present?
+        opts[:iinclude] = get_configs[:iinclude].to_s if get_configs[:iskip].present?
+        opts[:ignore_fragments] = get_configs[:ignore_fragments] if get_configs[:ignore_fragments] != nil
+        opts
+      end
+
+      def errors_message_for(responses)
+        collection = responses.each_with_object([])  do |(url, response), a|
+          a << "url: #{url}, error_code: #{response[:status]}"
+        end
+
+        collection.join("; ")
+
+      end
+
+      class Configure
+
+        attr_reader :config
+
+        def initialize
+          @config = {}
+        end
+      end
+
+    end
+
+    # Add to CLI
+    Base.register(Middleman::Cli::CrawlerIndex, 'crawler', 'crawler [options]', "Starts a crawler")
+  end
+end

data/lib/middleman-crawler/extension.rb

@@ -0,0 +1,31 @@
+# Require core library
+require 'middleman-core'
+
+# Extension namespace
+class Crawler < ::Middleman::Extension
+
+  expose_to_template :crawler
+
+  def initialize(app, options_hash={}, &block)
+    # Call super to build options from the options_hash
+    super
+
+    # Require libraries only when activated
+    # require 'necessary/library'
+
+    # set up your extension
+  end
+
+  def after_configuration
+    # Do something
+  end
+
+  # A Sitemap Manipulator
+  # def manipulate_resource_list(resources)
+  # end
+
+  # helpers do
+  #   def a_helper
+  #   end
+  # end
+end

data/lib/middleman-extension.rb

@@ -0,0 +1 @@
+require 'middleman-crawler'

data/middleman-crawler.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "middleman-crawler"
+  s.version     = "0.0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["weLaika", "Filippo Gangi Dino", "Fabrizio Monti"]
+  s.email       = ["info@welaika.com"]
+  s.homepage    = "https://github.com/welaika/middleman-crawler"
+  s.summary     = %q{crawler for Middleman site}
+  s.description = %q{it starts a crawler for Middleman sites}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.license       = 'MIT'
+
+  # The version of middleman-core your extension depends on
+  s.add_runtime_dependency "middleman-core", [">= 4.1.9"]
+
+  # Additional dependencies
+  s.add_runtime_dependency "rawler", "~> 0.1.8"
+
+  s.add_dependency 'bundler', '~> 1.3'
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: middleman-crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- weLaika
+- Filippo Gangi Dino
+- Fabrizio Monti
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: middleman-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.9
+- !ruby/object:Gem::Dependency
+  name: rawler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: it starts a crawler for Middleman sites
+email:
+- info@welaika.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- features/support/env.rb
+- lib/middleman-crawler.rb
+- lib/middleman-crawler/commands.rb
+- lib/middleman-crawler/extension.rb
+- lib/middleman-extension.rb
+- middleman-crawler.gemspec
+homepage: https://github.com/welaika/middleman-crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: crawler for Middleman site
+test_files:
+- features/support/env.rb
+has_rdoc: