pedophile 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a88a4801a4b0f5e66827ee308edee32de2f9882b
+  data.tar.gz: 31a68e039b567c0085d810000e959071322ec6ad
+SHA512:
+  metadata.gz: cdcc88116835bc2678a3f8f4c28cac9767c2037bf2cf9ba7e4e83eda05c128b90f08920d90a7f2de1f53185aa8a0fa081b0b8ecf881f6d985347c66bed377c73
+  data.tar.gz: eb284fdaedd82df541ca2248280eb030778467f5339f068f41d942c6f381e3ce73bff2b2e59a77c1e1ed18dd26edb5adcc9c2e39865930615f5d9908e79dfd20

data/Gemfile

@@ -0,0 +1,11 @@
+source "http://rubygems.org"
+
+gem 'activesupport'
+gem 'colorize'
+
+group :development do
+  gem "rspec"
+  gem "bundler"
+  gem "jeweler"
+  gem "simplecov"
+end

data/Gemfile.lock

@@ -0,0 +1,90 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.3.7)
+    builder (3.2.2)
+    colorize (0.7.5)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.9.1)
+    github_api (0.12.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.3)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.3)
+      oauth2
+    hashie (3.4.0)
+    highline (1.7.1)
+    i18n (0.7.0)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.2)
+    jwt (1.3.0)
+    mini_portile (0.6.2)
+    minitest (5.5.1)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.6.0)
+    rake (10.4.2)
+    rdoc (4.2.0)
+      json (~> 1.4)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.1)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    simplecov (0.9.2)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.9.0)
+    simplecov-html (0.9.0)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  bundler
+  colorize
+  jeweler
+  rspec
+  simplecov

data/LICENSE.txt

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,25 @@
+Pedophile
+=========
+
+Download static web pages.
+
+Sample usage
+------------
+
+<pre><code>
+p = Pedophile::Downloader.new
+
+p.url = "http://www.classnamer.com/"
+
+# clear tmp directory
+p.wget.clear!
+
+# sign in using devise like form
+#p.login.devise_login("http://www.classnamer.com/login", "email@email.com", "password")
+
+# download, process
+p.make_it_so
+
+# zip into single file in tmp/site/site.zip
+p.zip("site.zip")
+</code></pre>

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "pedophile"
+  gem.homepage = "http://github.com/akwiatkowski/pedophile"
+  gem.license = "LGPLv3"
+  gem.summary = %Q{download static pages for offline usage}
+  gem.description = %Q{download static pages for offline usage.}
+  gem.email = "bobikx@poczta.fm"
+  gem.authors = ["Aleksander Kwiatkowski"]
+  # dependencies defined in Gemfile
+
+  gem.files = FileList[
+    "[A-Z]*", "{bin,generators,lib,test}/**/*"
+  ]
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+
+desc "Run RSpec with code coverage"
+task :coverage do
+  `rake spec COVERAGE=true`
+  #`open coverage/index.html`
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/lib/pedophile.rb

@@ -0,0 +1,11 @@
+require 'colorize'
+
+require 'pedophile/login'
+require 'pedophile/wget'
+require 'pedophile/offline_tree'
+require 'pedophile/big_files'
+
+require 'pedophile/downloader'
+
+module Pedophile
+end

data/lib/pedophile/big_files.rb

@@ -0,0 +1,107 @@
+require 'fileutils'
+require 'yaml'
+
+module Pedophile
+  class BigFiles
+    USE_MIME = false
+    TMP_STRUCTURE_PATH = File.absolute_path(File.join(Wget::TMP_PATH, "big_files.yaml"))
+
+    def initialize(downloader)
+      @downloader = downloader
+      @files = Array.new
+    end
+
+    attr_reader :downloader, :full_path, :files, :files_path
+
+    def offline_path
+      self.downloader.wget.offline_path
+    end
+
+    def copy_folder(path)
+      puts "copying big files path #{path.to_s.cyan}"
+      FileUtils.cp_r(path, offline_path)
+      puts "done copying path #{path.to_s.cyan}"
+      big_files_path = path
+    end
+
+    def big_files_path=(path)
+      @files_path = path
+      @full_path = File.join(offline_path, path)
+    end
+
+    def analyze
+      glob_path = "#{full_path}/**/**"
+      puts "big files path #{full_path.to_s.cyan}"
+
+      Dir.glob(glob_path) do |item|
+        next if item == '.' or item == '..' or File.directory?(item)
+
+        puts "analyze file #{item.to_s.yellow}"
+
+        h = Hash.new
+        h[:path] = item
+
+        if USE_MIME
+          mime = `file --mime #{item}`
+          if mime =~ /(\w+\/\w+);/
+            mime = $1
+          else
+            mime = nil
+          end
+          h[:mime] = mime
+        end
+
+        @files << h
+      end
+
+      save_analyzed
+    end
+
+    def save_analyzed
+      f = File.new(TMP_STRUCTURE_PATH, "w")
+      f.puts @files.to_yaml
+      f.close
+    end
+
+    def load_analyzed
+      @files = YAML.load_file(TMP_STRUCTURE_PATH)
+    end
+
+    def gsub_links
+      files.each do |f|
+        file_path = f[:path].clone
+        smaller_path = file_path.gsub(full_path, "")
+        smaller_path.gsub!(/^\//, '')
+
+        gsub_big_file(smaller_path)
+        f[:done] = true
+      end
+    end
+
+    def gsub_big_file(smaller_path)
+      puts "process big file #{smaller_path.to_s.green}"
+
+      self.downloader.offline_tree.files.each do |f|
+        if f[:inside]
+          to_rename = f[:inside].select do |fi|
+            fi[:path].index(smaller_path)
+          end
+
+          # TODO gsub path issue with html files inside
+          to_rename.each do |fi|
+            original_string = fi[:path]
+            new_string = File.join(files_path, smaller_path)
+
+            puts "rename big file #{original_string.to_s.blue} to #{new_string.to_s.green}"
+
+            self.downloader.offline_tree.process_massive_gsub(original_string, new_string, true)
+          end
+
+
+        end
+      end
+
+    end
+
+  end
+end

data/lib/pedophile/downloader.rb

@@ -0,0 +1,25 @@
+require 'active_support/all'
+
+module Pedophile
+  class Downloader
+    def initialize
+      @login = Login.new(self)
+      @wget = Wget.new(self)
+      @offline_tree = OfflineTree.new(self)
+      @big_files = BigFiles.new(self)
+    end
+
+    attr_reader :login, :wget, :offline_tree, :big_files
+    attr_accessor :url
+
+    def make_it_so
+      wget.mirror
+      offline_tree.make_it_so
+    end
+
+    def zip(name = "site.zip")
+      offline_tree.zip(name)
+    end
+
+  end
+end

data/lib/pedophile/login.rb

@@ -0,0 +1,54 @@
+require 'uri'
+require 'colorize'
+
+module Pedophile
+  class Login
+
+    attr_reader :downloader
+
+    def initialize(downloader)
+      @downloader = downloader
+    end
+
+    def devise_login(url, email, password)
+      uri = URI.parse(url)
+
+      string = @downloader.wget.download(url)
+
+      token = nil
+      if string =~ /<input name=\"authenticity_token\" type=\"hidden\" value=\"([^"]+)\" \/>/
+        token = $1
+        puts "got devise token #{token.to_s.blue}"
+      end
+
+      action_url = nil
+      if string =~ /action=\"([^"]+)\"/
+        action_url = $1
+        puts "got action url #{action_url.to_s.blue}"
+      end
+
+      sign_url = "http://#{uri.host}#{action_url}"
+      puts "sign action url #{sign_url.to_s.blue}"
+
+      post_params = {
+        "authenticity_token" => token,
+        "user" => {
+          "email" => email,
+          "password" => password,
+          "remember_me" => 1
+        }
+      }
+      post_params = {
+        "utf8"=>"✓",
+        "authenticity_token" => token,
+        "user[email]" => email,
+        "user[password]" => password,
+        "user[remember_me]" => 1
+      }
+
+      string = @downloader.wget.post(url, post_params)
+      string
+    end
+
+  end
+end

data/lib/pedophile/offline_tree.rb

@@ -0,0 +1,307 @@
+require 'yaml'
+require 'pathname'
+
+module Pedophile
+  class OfflineTree
+    TMP_STRUCTURE_PATH = File.absolute_path(File.join(Wget::TMP_PATH, "files.yaml"))
+    TMP_CHANGES_PATH = File.absolute_path(File.join(Wget::TMP_PATH, "changes.yaml"))
+    FIX_RELATIVE_PATH = false
+
+    def initialize(downloader)
+      @downloader = downloader
+      @files = Array.new
+      @changes = Array.new
+    end
+
+    attr_reader :downloader, :files
+
+    def make_it_so
+      analyze
+      load_analyzed
+
+      process_bad_suffix1
+      process_bad_suffix2
+      process_bad_filenames
+      save_analyzed
+      save_changes
+    end
+
+    def zip(output_file = 'site.zip')
+      command = "cd #{Wget::TMP_OFFLINE_PATH}; zip -r #{output_file} #{self.downloader.wget.site_last_path}"
+      puts command
+      `#{command}`
+    end
+
+    # Desctructive part
+    def after_process
+      load_processed
+      remove_bad_suffix
+      rename_files
+    end
+
+    def path
+      @path ||= self.downloader.wget.offline_path
+      @path
+    end
+
+    def analyze
+      # because I don't want to read all wget options...
+      glob_path = "#{path}/**/**"
+      puts "offline path #{path.to_s.cyan}"
+
+      Dir.glob(glob_path) do |item|
+        next if item == '.' or item == '..' or File.directory?(item)
+
+        puts "analyze file #{item.to_s.yellow}"
+
+        h = Hash.new
+        h[:path] = item
+
+        mime = `file --mime #{item}`
+        if mime =~ /(\w+\/\w+);/
+          mime = $1
+        else
+          mime = nil
+        end
+
+        h[:mime] = mime
+
+        if mime == 'text/html' or mime == 'text/plain'
+          h[:inside] = analyze_file(item)
+        end
+
+        @files << h
+      end
+
+      save_analyzed
+    end
+
+    def save_analyzed
+      f = File.new(TMP_STRUCTURE_PATH, "w")
+      f.puts @files.to_yaml
+      f.close
+    end
+
+    def save_changes
+      f = File.new(TMP_CHANGES_PATH, "w")
+      f.puts @changes.to_yaml
+      f.close
+    end
+
+    def load_analyzed
+      @files = YAML.load_file(TMP_STRUCTURE_PATH)
+    end
+
+    def analyze_file(file)
+      s = File.read(file)
+
+      possible_paths_regexp = /"([^"]+)"/
+      possible_paths = s.scan(possible_paths_regexp).flatten.uniq
+
+      possible_paths_regexp = /'([^']+)'/
+      possible_paths += s.scan(possible_paths_regexp).flatten.uniq
+
+      relative_file_path = File.dirname(file)
+
+      paths = Array.new
+      possible_paths.each do |pp|
+        if is_path_ok?(pp)
+          h = Hash.new
+          f = File.join(relative_file_path, pp)
+          h[:exists] = File.exists?(f)
+          h[:is_file] = File.file?(f)
+          h[:path] = pp
+
+          paths << h if should_add_path?(h)
+        end
+      end
+
+      paths
+    end
+
+    # TODO  - check if this string is correct unix path
+    def is_path_ok?(pp)
+      # pp =~ /\A(?:[0-9a-zA-Z\_\-]+\/?)+\z/
+      pp.size < 200
+    end
+
+    # TODO
+    def should_add_path?(h)
+      return true
+      #return h[:is_file]
+    end
+
+    def base_path
+      @base_path ||= self.downloader.wget.offline_path
+      @base_path
+    end
+
+    # PROCESSING
+    def process_bad_suffix2
+      @files.each do |f|
+        old_file = f[:path]
+        new_file = old_file.gsub(/\?body=1/, '')
+
+        if not new_file == old_file
+          process_rename_file(old_file, new_file)
+        end
+      end
+
+      process_massive_gsub("%3Fbody=1", "", false)
+    end
+
+    def process_bad_suffix1
+      @files.each do |f|
+        old_file = f[:path]
+        new_file = old_file.gsub(/\?\d+/, '').gsub(/\%3F\d+/, '')
+
+        if not new_file == old_file
+          process_rename_file(old_file, new_file)
+        end
+
+        if f[:inside]
+          f[:inside].each do |fi|
+            old_file = fi[:path]
+            if File.exists?(old_file)
+              new_file = old_file.gsub(/\?\d+/, '').gsub(/\%3F\d+/, '')
+
+              if not new_file == old_file
+                process_rename_file(old_file, new_file)
+              end
+
+            end
+          end
+        end
+      end
+
+      process_massive_gsub(/\%3F\d+/, "", false)
+    end
+
+    def process_bad_filenames
+      @files.each do |f|
+        old_file = f[:path]
+        new_file = old_file.gsub(/[^0-9A-Za-z.\-\/:]/, '_')
+
+        if not new_file == old_file
+          process_rename_file(old_file, new_file)
+        end
+
+        if f[:inside]
+          f[:inside].each do |fi|
+            old_file = fi[:path]
+            if File.exists?(old_file)
+              new_file = old_file.gsub(/[^0-9A-Za-z.\-\/:]/, '_')
+
+              if not new_file == old_file
+                process_rename_file(old_file, new_file)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    #def process_bad_filenames_links
+    #  process_massive_gsub(/\%3F/, "_", false)
+    #end
+
+    def process_rename_file(old_file_path, new_file_path)
+      puts "rename from #{old_file_path.to_s.blue} to #{new_file_path.to_s.green}"
+
+      # clone to not allow modify of @files
+      old_file = old_file_path.clone
+      new_file = new_file_path.clone
+      # this will be with full path
+      old_file_with_path = old_file_path.clone
+
+      old_file.gsub!(base_path, '')
+      new_file.gsub!(base_path, '')
+
+      # ignore slashes
+      old_file.gsub!(/^\//, '')
+      new_file.gsub!(/^\//, '')
+
+      # 1. rename 1 file
+      new_file_path = old_file_with_path.gsub(old_file, new_file)
+      File.rename(old_file_with_path, new_file_path)
+
+      # internal log-like
+      @changes << { rename: { old: old_file_with_path, new: new_file_path } }
+
+      # 2. rename in @files
+      @files.each do |f|
+        if f[:path] == old_file_with_path
+          f[:path] = new_file_path
+        end
+
+        if f[:inside]
+          f[:inside].each do |fi|
+            if fi[:path] == old_file_with_path
+              fi[:path] = new_file_path
+            end
+          end
+        end
+      end
+
+      # 3. gsub all files
+      # gsub files after renaming
+      process_massive_gsub(old_file, new_file, true)
+      process_massive_gsub(old_file.gsub("?", "%3F"), new_file, true)
+
+      puts "RENAMED #{old_file.to_s.blue} to #{new_file.to_s.green}"
+    end
+
+    def process_massive_gsub(from, to, check_paths = false)
+      puts "massive gsub #{from.to_s.blue} to #{to.to_s.green}"
+
+      @files.each do |f|
+        # must be proper mime before, so not needed to check
+        if f[:inside]
+          file_path = f[:path].clone
+
+          puts " open #{file_path.to_s.red}"
+
+          old_from = from.to_s
+          old_to = to.to_s
+
+          # relative path fix
+          if check_paths and FIX_RELATIVE_PATH
+            absolute_path = File.absolute_path(File.dirname(file_path))
+            first = Pathname.new(absolute_path)
+
+            to_path = File.join(path, to)
+            second = Pathname.new(File.absolute_path(to_path))
+            to = second.relative_path_from(first).to_s
+          end
+
+          exists = File.exists?(file_path)
+          if exists
+            j = File.open(file_path)
+            s = j.read
+            j.close
+
+            # logs
+            if s.index(from)
+              @changes << { gsub: { old: from, new: to, file: file_path, old_from: old_from, old_to: old_to } }
+            end
+
+            s.gsub!(from, to)
+
+            j = File.open(file_path, "w")
+            j.puts(s)
+            j.close
+
+            f[:inside].each do |fi|
+              fi[:path].gsub!(from, to)
+            end
+
+            puts " done #{file_path.to_s.red}"
+          else
+            raise "file #{file_path} not found"
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/pedophile/wget.rb

@@ -0,0 +1,59 @@
+require 'active_support/all'
+require 'fileutils'
+
+module Pedophile
+  class Wget
+    TMP_PATH = "tmp"
+    TMP_ABSOLUTE_PATH = File.absolute_path(TMP_PATH)
+    TMP_FILE_PATH = File.absolute_path(File.join(TMP_PATH, "tmp.tmp"))
+    COOKIES_FILE_PATH = File.absolute_path(File.join(TMP_PATH, "cookies.txt"))
+    TMP_OFFLINE_PATH = File.join(TMP_PATH, "site")
+
+    WGET_PARAMS = "-v --random-wait --user-agent=Mozilla/5.0 --keep-session-cookies --load-cookies #{COOKIES_FILE_PATH} --save-cookies #{COOKIES_FILE_PATH}"
+    # http://www.gnu.org/software/wget/manual/html_node/Download-Options.html
+    #WGET_RESTRICT_FILE_NAMES = "windows" # windows, ascii, unix
+    WGET_RESTRICT_FILE_NAMES = "unix"
+    WGET_MIRROR_PARAMS = "--adjust-extension --mirror --page-requisites --convert-links --restrict-file-names=#{WGET_RESTRICT_FILE_NAMES}"
+
+    def initialize(downloader)
+      @downloader = downloader
+      prepare_tmp_path
+    end
+
+    attr_reader :downloader
+
+    def prepare_tmp_path
+      Dir.mkdir(TMP_PATH) unless File.exists?(TMP_PATH)
+      Dir.mkdir(TMP_OFFLINE_PATH) unless File.exists?(TMP_OFFLINE_PATH)
+    end
+
+    def download(url)
+      `wget #{WGET_PARAMS} #{url} -O#{TMP_FILE_PATH}`
+      File.open(TMP_FILE_PATH).read
+    end
+
+    def post(url, params)
+      post_data = params.to_query
+      `wget #{WGET_PARAMS} #{url} --post-data '#{post_data}' -O#{TMP_FILE_PATH}`
+      File.open(TMP_FILE_PATH).read
+    end
+
+    def mirror
+      `cd #{TMP_OFFLINE_PATH}; wget #{WGET_PARAMS} #{WGET_MIRROR_PARAMS} #{self.downloader.url}`
+    end
+
+    def clear!
+      FileUtils.rm_rf(TMP_ABSOLUTE_PATH)
+      prepare_tmp_path
+    end
+
+    def site_last_path
+      (Dir.entries(Wget::TMP_OFFLINE_PATH) - ["..", "."]).first
+    end
+
+    def offline_path
+      File.join(TMP_OFFLINE_PATH, site_last_path)
+    end
+
+  end
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: pedophile
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Aleksander Kwiatkowski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: download static pages for offline usage.
+email: bobikx@poczta.fm
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- lib/pedophile.rb
+- lib/pedophile/big_files.rb
+- lib/pedophile/downloader.rb
+- lib/pedophile/login.rb
+- lib/pedophile/offline_tree.rb
+- lib/pedophile/wget.rb
+homepage: http://github.com/akwiatkowski/pedophile
+licenses:
+- LGPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: download static pages for offline usage
+test_files: []