sitemap_maker 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.rspec +1 -0
- data/Gemfile +16 -0
- data/Gemfile.lock +77 -0
- data/LICENSE.txt +20 -0
- data/README.rdoc +19 -0
- data/Rakefile +52 -0
- data/bin/sitemap_maker +33 -0
- data/lib/SitemapMaker/Nokogiri/XML/node.rb +27 -0
- data/lib/SitemapMaker/Nokogiri/html.rb +32 -0
- data/lib/SitemapMaker/core_ext/object/blank.rb +13 -0
- data/lib/SitemapMaker/core_ext/object/try.rb +9 -0
- data/lib/SitemapMaker/core_ext/uri/derectory.rb +9 -0
- data/lib/SitemapMaker/html.rb +9 -0
- data/lib/SitemapMaker/site_tree.rb +109 -0
- data/lib/SitemapMaker/utils.rb +67 -0
- data/lib/SitemapMaker/version.rb +3 -0
- data/lib/sitemap_maker.rb +27 -0
- data/sitemap_maker.gemspec +52 -0
- data/spec/html_spec.rb +22 -0
- data/spec/site_tree_spec.rb +32 -0
- data/spec/sitemap_maker_spec.rb +5 -0
- data/spec/spec_helper.rb +20 -0
- metadata +250 -0
data/.document
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
binding_of_caller (0.6.8)
|
5
|
+
diff-lcs (1.1.3)
|
6
|
+
domain_name (0.5.4)
|
7
|
+
unf (~> 0.0.3)
|
8
|
+
fuubar (1.1.0)
|
9
|
+
rspec (~> 2.0)
|
10
|
+
rspec-instafail (~> 0.2.0)
|
11
|
+
ruby-progressbar (~> 1.0.0)
|
12
|
+
git (1.2.5)
|
13
|
+
jeweler (1.8.4)
|
14
|
+
bundler (~> 1.0)
|
15
|
+
git (>= 1.2.5)
|
16
|
+
rake
|
17
|
+
rdoc
|
18
|
+
json (1.7.5)
|
19
|
+
mechanize (2.5.1)
|
20
|
+
domain_name (~> 0.5, >= 0.5.1)
|
21
|
+
mime-types (~> 1.17, >= 1.17.2)
|
22
|
+
net-http-digest_auth (~> 1.1, >= 1.1.1)
|
23
|
+
net-http-persistent (~> 2.5, >= 2.5.2)
|
24
|
+
nokogiri (~> 1.4)
|
25
|
+
ntlm-http (~> 0.1, >= 0.1.1)
|
26
|
+
webrobots (~> 0.0, >= 0.0.9)
|
27
|
+
mime-types (1.19)
|
28
|
+
multi_json (1.3.6)
|
29
|
+
net-http-digest_auth (1.2.1)
|
30
|
+
net-http-persistent (2.8)
|
31
|
+
nokogiri (1.5.5)
|
32
|
+
ntlm-http (0.1.1)
|
33
|
+
plymouth (0.3.3)
|
34
|
+
pry-exception_explorer (~> 0.1.7)
|
35
|
+
pry-exception_explorer (0.1.9)
|
36
|
+
pry-stack_explorer (>= 0.3.9)
|
37
|
+
pry-stack_explorer (0.4.7)
|
38
|
+
binding_of_caller (~> 0.6.8)
|
39
|
+
rake (0.9.2.2)
|
40
|
+
rdoc (3.12)
|
41
|
+
json (~> 1.4)
|
42
|
+
rspec (2.11.0)
|
43
|
+
rspec-core (~> 2.11.0)
|
44
|
+
rspec-expectations (~> 2.11.0)
|
45
|
+
rspec-mocks (~> 2.11.0)
|
46
|
+
rspec-core (2.11.1)
|
47
|
+
rspec-expectations (2.11.3)
|
48
|
+
diff-lcs (~> 1.1.3)
|
49
|
+
rspec-instafail (0.2.4)
|
50
|
+
rspec-mocks (2.11.3)
|
51
|
+
ruby-progressbar (1.0.2)
|
52
|
+
simplecov (0.7.1)
|
53
|
+
multi_json (~> 1.0)
|
54
|
+
simplecov-html (~> 0.7.1)
|
55
|
+
simplecov-html (0.7.1)
|
56
|
+
spork (0.9.2)
|
57
|
+
to_regexp (0.1.1)
|
58
|
+
unf (0.0.5)
|
59
|
+
unf_ext
|
60
|
+
unf_ext (0.0.5)
|
61
|
+
webrobots (0.0.13)
|
62
|
+
|
63
|
+
PLATFORMS
|
64
|
+
ruby
|
65
|
+
|
66
|
+
DEPENDENCIES
|
67
|
+
bundler
|
68
|
+
fuubar
|
69
|
+
jeweler
|
70
|
+
mechanize
|
71
|
+
nokogiri
|
72
|
+
plymouth
|
73
|
+
rdoc
|
74
|
+
rspec
|
75
|
+
simplecov
|
76
|
+
spork
|
77
|
+
to_regexp
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 Hiroyuki Ishii
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
= sitemap_maker
|
2
|
+
|
3
|
+
Description goes here.
|
4
|
+
|
5
|
+
== Contributing to sitemap_maker
|
6
|
+
|
7
|
+
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
8
|
+
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
9
|
+
* Fork the project.
|
10
|
+
* Start a feature/bugfix branch.
|
11
|
+
* Commit and push until you are happy with your contribution.
|
12
|
+
* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
13
|
+
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
14
|
+
|
15
|
+
== Copyright
|
16
|
+
|
17
|
+
Copyright (c) 2012 Hiroyuki Ishii. See LICENSE.txt for
|
18
|
+
further details.
|
19
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
require 'rake'
|
6
|
+
require 'jeweler'
|
7
|
+
require './lib/sitemap_maker'
|
8
|
+
|
9
|
+
begin
|
10
|
+
Bundler.setup(:default, :development)
|
11
|
+
rescue Bundler::BundlerError => e
|
12
|
+
$stderr.puts e.message
|
13
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
14
|
+
exit e.status_code
|
15
|
+
end
|
16
|
+
|
17
|
+
Jeweler::Tasks.new do |gem|
|
18
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
19
|
+
gem.version = SitemapMaker::VERSION
|
20
|
+
gem.name = "sitemap_maker"
|
21
|
+
gem.homepage = "http://github.com/taichouchou2/sitemap_maker"
|
22
|
+
gem.license = "MIT"
|
23
|
+
gem.summary = %Q{one-line summary of your gem}
|
24
|
+
gem.description = %Q{longer description of your gem}
|
25
|
+
gem.email = "alprhcp666@gmail.com"
|
26
|
+
gem.authors = ["Hiroyuki Ishii"]
|
27
|
+
# dependencies defined in Gemfile
|
28
|
+
end
|
29
|
+
Jeweler::RubygemsDotOrgTasks.new
|
30
|
+
|
31
|
+
require 'rspec/core'
|
32
|
+
require 'rspec/core/rake_task'
|
33
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
34
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
35
|
+
end
|
36
|
+
|
37
|
+
RSpec::Core::RakeTask.new(:rcov) do |spec|
|
38
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
39
|
+
spec.rcov = true
|
40
|
+
end
|
41
|
+
|
42
|
+
task :default => :spec
|
43
|
+
|
44
|
+
require 'rdoc/task'
|
45
|
+
Rake::RDocTask.new do |rdoc|
|
46
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
47
|
+
|
48
|
+
rdoc.rdoc_dir = 'rdoc'
|
49
|
+
rdoc.title = "sitemap_maker #{version}"
|
50
|
+
rdoc.rdoc_files.include('README*')
|
51
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
52
|
+
end
|
data/bin/sitemap_maker
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# coding: utf-8
|
3
|
+
|
4
|
+
require 'sitemap_maker'
|
5
|
+
require 'csv'
|
6
|
+
require 'nkf'
|
7
|
+
|
8
|
+
opt = SitemapMaker::Utils.option_parser ARGV
|
9
|
+
|
10
|
+
url = opt[:target]
|
11
|
+
deep_level = opt[:level] # 何階層下までクロールするか
|
12
|
+
result = SitemapMaker::SiteTree.tree_all( url, deep_level )
|
13
|
+
|
14
|
+
CSV.open(opt[:path], "w") do |csv|
|
15
|
+
csv << [NKF::nkf("-s", "ターゲットURL" ), url ]
|
16
|
+
csv << ["URL", "title", "description", "keywords"]
|
17
|
+
|
18
|
+
result.each do |key, hash|
|
19
|
+
row = []
|
20
|
+
row << key
|
21
|
+
[:title, :description, :keywords].each do |column|
|
22
|
+
if hash[column]
|
23
|
+
row << NKF::nkf("-s", hash[column])
|
24
|
+
else
|
25
|
+
row << NKF::nkf("-s", hash[:error]) if hash[:error]
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
csv << row
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# nookgiriを拡張
|
2
|
+
# hrefの取得ができるように
|
3
|
+
module Nokogiri::XML
|
4
|
+
class Node
|
5
|
+
def method_missing(method_name, args = nil, &block)
|
6
|
+
case method_name
|
7
|
+
when /_hash$/
|
8
|
+
get_attribute(method_name.to_s.gsub(/_hash$/, ''))
|
9
|
+
else
|
10
|
+
super(method_name, args, &block)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
def get_attribute(key)
|
16
|
+
return nil unless respond_to?(:attributes)
|
17
|
+
|
18
|
+
if attributes.respond_to?(:has_key?)
|
19
|
+
return attributes[key]
|
20
|
+
else
|
21
|
+
return nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# SitemapMaker::HTML(open("http://..."))で返されるオブジェクトです
|
2
|
+
# method_missingの拡張
|
3
|
+
class Nokogiri::HTML::Document
|
4
|
+
|
5
|
+
# meta情報取得メソッドを定義
|
6
|
+
['keywords', 'description', 'author', 'robots', 'generator', ].each do |name|
|
7
|
+
define_method(name) do
|
8
|
+
# meta情報の取得
|
9
|
+
# metaが存在しない場合、一文字めだけ大文字にする #=> Description
|
10
|
+
meta = xpath( "/html/head/meta[@name=\"#{name}\"]" ).first
|
11
|
+
meta ||= xpath( "/html/head/meta[@name=\"#{name.capitalize}\"]" ).first
|
12
|
+
|
13
|
+
return nil unless meta.respond_to?(:attributes)
|
14
|
+
meta.attributes["content"].value #keywords => "スマートフォンアプリ,Facebookアプリ開発,大阪,"
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
def method_missing(method_name, args = nil, &block)
|
20
|
+
method_name = method_name.to_s
|
21
|
+
|
22
|
+
# page.meta_all などのメソッドは、xpathで要素を返す
|
23
|
+
case method_name
|
24
|
+
when /_all$/
|
25
|
+
@meta_all = self.xpath("//#{method_name.sub(/_all$/, '')}")
|
26
|
+
else
|
27
|
+
super
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
@@ -0,0 +1,9 @@
|
|
1
|
+
module SitemapMaker
|
2
|
+
|
3
|
+
# learn more [gems... lib/nokogiri.rb & lib/nokogiri/html.rb etc...]
|
4
|
+
def HTML thing, url = nil, encoding = nil, options = Nokogiri::XML::ParseOptions::DEFAULT_HTML, &block
|
5
|
+
Nokogiri::HTML::Document.parse(thing, url, encoding, options, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
module_function :HTML
|
9
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
module SitemapMaker::SiteTree
|
4
|
+
|
5
|
+
# return domain
|
6
|
+
def root(url)
|
7
|
+
uri = URI.parse(url)
|
8
|
+
"http://#{uri.host}"
|
9
|
+
end
|
10
|
+
|
11
|
+
# return url have all links with own domain
|
12
|
+
def tree_all(url, tree_level = 5) #=> recursive all links ["http://dlab-inc.jp", "http://dlab-inc.jp/company.html"...]
|
13
|
+
Tree.new(url).tree_all(url, {}, tree_level)
|
14
|
+
end
|
15
|
+
|
16
|
+
# return url have links with own domain
|
17
|
+
def own_links(url) #=> own page have own links ["http://dlab-inc.jp", "http://dlab-inc.jp/company.html"]
|
18
|
+
Tree.new(url).own_links
|
19
|
+
end
|
20
|
+
|
21
|
+
class Tree
|
22
|
+
def initialize( url )
|
23
|
+
@page = Nokogiri::HTML open(url)
|
24
|
+
@url = url
|
25
|
+
@uri = URI.parse( url )
|
26
|
+
end
|
27
|
+
|
28
|
+
# return page have own links
|
29
|
+
def own_links
|
30
|
+
agent = Mechanize.new
|
31
|
+
page = agent.get(@url)
|
32
|
+
|
33
|
+
# XXX Regexp修正必要
|
34
|
+
# test more http://www.rubular.com
|
35
|
+
own_link_regexp = Regexp.new(%r!(#{@uri.host})[a-zA-Z0-9.?\/=%&一-龠亜-煕-]+!)
|
36
|
+
|
37
|
+
# pageが持っているリンク
|
38
|
+
have_links = page.links_with( href: own_link_regexp )
|
39
|
+
|
40
|
+
own_links = []
|
41
|
+
have_links.each do |link|
|
42
|
+
url = complete_url(page.uri, link.uri)
|
43
|
+
own_links << url unless own_links.include?( url )
|
44
|
+
end
|
45
|
+
|
46
|
+
own_links
|
47
|
+
end
|
48
|
+
|
49
|
+
# return recursive own links
|
50
|
+
def tree_all(url = @url, target_links = {}, deep_level = nil )
|
51
|
+
deep_level ||= SitemapMaker::Utils::DEFAULT_LEVEL
|
52
|
+
deep_level = deep_level.to_i if deep_level.is_a?(String)
|
53
|
+
domain_links = target_links.dup
|
54
|
+
|
55
|
+
Tree.new(url).own_links.each do |link|
|
56
|
+
link_key = link.to_s
|
57
|
+
# 新規リンクの場合は追記
|
58
|
+
if domain_links[link_key].blank?
|
59
|
+
begin
|
60
|
+
# deep_levelに達したら終了
|
61
|
+
if deep_level > 0
|
62
|
+
# meta情報の取得
|
63
|
+
domain_links[link_key] = get_meta(link_key)
|
64
|
+
domain_links.merge! tree_all(link_key, domain_links, deep_level - 1)
|
65
|
+
end
|
66
|
+
rescue => e
|
67
|
+
# TODO 404なら、リンク切れと明記
|
68
|
+
domain_links[link_key] = { error: "リンク切れ" }
|
69
|
+
puts e
|
70
|
+
ensure
|
71
|
+
puts link_key
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
domain_links
|
76
|
+
end
|
77
|
+
|
78
|
+
domain_links
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
# return meta detail
|
83
|
+
def get_meta(url)
|
84
|
+
page = SitemapMaker::HTML open(url)
|
85
|
+
|
86
|
+
return {
|
87
|
+
description: page.description,
|
88
|
+
title: page.title,
|
89
|
+
keywords: page.keywords,
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
# fix up url with base_uri
|
94
|
+
def complete_url(base_uri, path = nil)
|
95
|
+
return path if path.to_s.match(@uri.host)
|
96
|
+
|
97
|
+
if path.to_s.match(/^\//)
|
98
|
+
# 絶対path
|
99
|
+
"http://#{base_uri.host}#{path.to_s}"
|
100
|
+
else
|
101
|
+
# 相対path
|
102
|
+
"#{base_uri.directory_path}#{path}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
|
108
|
+
module_function :root, :tree_all, :own_links
|
109
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module SitemapMaker::Utils
|
2
|
+
DEFAULT_LEVEL = 3
|
3
|
+
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def option_parser args
|
7
|
+
args << '-h' if args.empty?
|
8
|
+
|
9
|
+
options = {}
|
10
|
+
optparse = OptionParser.new do |opts|
|
11
|
+
opts.banner = get_banner.gsub(/^\s*/, '')
|
12
|
+
|
13
|
+
opts.on('-h','--help','HELP') do
|
14
|
+
puts opts
|
15
|
+
exit
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on('-l', '--deep_level VAL', 'deep_level') do |l|
|
19
|
+
options[:level] = l
|
20
|
+
options[:level] ||= SitemapMaker::Utils::DEFAULT_LEVEL
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on('-f VAL', '--filepath VAL', 'output path') do |l|
|
24
|
+
options[:path] = l
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on('-t VAL', '--target VAL', 'target url') do |l|
|
28
|
+
options[:target] = l
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on('-v', '--version', 'Version') do
|
32
|
+
puts SitemapMaker::VERSION
|
33
|
+
exit
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
optparse.parse! args
|
39
|
+
options[:files] = args.to_a
|
40
|
+
|
41
|
+
{
|
42
|
+
level: options[:level],
|
43
|
+
path: options[:path],
|
44
|
+
target: options[:target],
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
def get_banner
|
49
|
+
return <<-BANNAR
|
50
|
+
Usage:
|
51
|
+
alpacaComplete [OPTIONS]
|
52
|
+
|
53
|
+
etc..
|
54
|
+
sitemap_maker -l 3 -f data.csv -t http://dlab-inc.jp/
|
55
|
+
---------------------------------------------------------------------
|
56
|
+
BANNAR
|
57
|
+
end
|
58
|
+
|
59
|
+
def run cmd, path, uncomplete=""
|
60
|
+
case cmd
|
61
|
+
when "locale"
|
62
|
+
puts SitemapMaker::LocaleComplete.complete path, uncomplete
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
# require gem
|
4
|
+
require 'nokogiri'
|
5
|
+
require 'open-uri'
|
6
|
+
require 'mechanize'
|
7
|
+
require 'pry'
|
8
|
+
require 'to_regexp'
|
9
|
+
|
10
|
+
# extend gem
|
11
|
+
Dir["./**/*.rb"].each { |f| require f }
|
12
|
+
|
13
|
+
# require 'SitemapMaker/Nokogiri/XML/node'
|
14
|
+
# require 'SitemapMaker/Nokogiri/html'
|
15
|
+
#
|
16
|
+
# # extend module and class
|
17
|
+
# require 'SitemapMaker/core_ext/object/blank'
|
18
|
+
# require 'SitemapMaker/core_ext/object/try'
|
19
|
+
|
20
|
+
# require SitemapMaker
|
21
|
+
# require 'SitemapMaker/html'
|
22
|
+
|
23
|
+
module SitemapMaker
|
24
|
+
autoload :VERSION, 'SitemapMaker/version'
|
25
|
+
autoload :SiteTree, 'SitemapMaker/site_tree'
|
26
|
+
end
|
27
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
require './lib/SitemapMaker.rb'
|
7
|
+
|
8
|
+
Gem::Specification.new do |s|
|
9
|
+
s.name = "sitemap_maker"
|
10
|
+
s.version = SitemapMaker::VERSION
|
11
|
+
|
12
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
13
|
+
s.authors = ["Hiroyuki Ishii"]
|
14
|
+
s.date = "2012-10-27"
|
15
|
+
s.description = "show meta detail as csv"
|
16
|
+
s.email = "alprhcp666@gmail.com"
|
17
|
+
s.extra_rdoc_files = [
|
18
|
+
"LICENSE.txt",
|
19
|
+
"README.rdoc"
|
20
|
+
]
|
21
|
+
s.files = `git ls-files`.split("\n")
|
22
|
+
s.homepage = "http://github.com/taichouchou2/sitemap_maker"
|
23
|
+
s.licenses = ["MIT"]
|
24
|
+
s.require_paths = ["lib"]
|
25
|
+
s.rubygems_version = "1.8.23"
|
26
|
+
s.summary = "show site detail for seo manager"
|
27
|
+
|
28
|
+
if s.respond_to? :specification_version then
|
29
|
+
s.specification_version = 3
|
30
|
+
|
31
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
32
|
+
s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
|
33
|
+
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
34
|
+
s.add_development_dependency(%q<bundler>, [">= 0"])
|
35
|
+
s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
|
36
|
+
s.add_development_dependency(%q<simplecov>, [">= 0"])
|
37
|
+
else
|
38
|
+
s.add_dependency(%q<rspec>, ["~> 2.8.0"])
|
39
|
+
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
40
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
41
|
+
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
42
|
+
s.add_dependency(%q<simplecov>, [">= 0"])
|
43
|
+
end
|
44
|
+
else
|
45
|
+
s.add_dependency(%q<rspec>, ["~> 2.8.0"])
|
46
|
+
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
47
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
48
|
+
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
49
|
+
s.add_dependency(%q<simplecov>, [">= 0"])
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
data/spec/html_spec.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
4
|
+
|
5
|
+
describe SitemapMaker do
|
6
|
+
before do
|
7
|
+
@url = "http://dlab-inc.jp"
|
8
|
+
@page = SitemapMaker::HTML open( @url )
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should get title" do
|
12
|
+
@page.title.should eql("株式会社DreamLab|スマートフォンアプリ開発のWeb制作会社")
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should get description" do
|
16
|
+
@page.description.should eql("DreamLabのホームページです。スマートフォンアプリケーションの企画、制作、ダウンロード促進企画を一括で行えます。成果を出すためのノウハウを凝縮し、幅広いWEBプロモーションを行っていくことを前提としたホームページを設計します。")
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should get keywords" do
|
20
|
+
@page.keywords.should eql("スマートフォンアプリ,Facebookアプリ開発,大阪,")
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
4
|
+
|
5
|
+
describe SitemapMaker do
|
6
|
+
before do
|
7
|
+
@url = "http://omocoro.jp"
|
8
|
+
end
|
9
|
+
|
10
|
+
context :root do
|
11
|
+
it "should return root url" do
|
12
|
+
url = "http://dlab-inc.jp/top/mod/"
|
13
|
+
root_url = SitemapMaker::SiteTree.root( url )
|
14
|
+
root_url.should eql("http://dlab-inc.jp")
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
context :own_links do
|
19
|
+
it "should return own links" do
|
20
|
+
own_links = SitemapMaker::SiteTree.own_links( @url )
|
21
|
+
own_links.class.should eql(Array)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
context :tree_all do
|
26
|
+
it "should return own links" do
|
27
|
+
tree_all = SitemapMaker::SiteTree.tree_all( @url )
|
28
|
+
binding.pry
|
29
|
+
tree_all.class.should eql(Array)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'spork'
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
5
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
6
|
+
require 'rspec'
|
7
|
+
require 'sitemap_maker'
|
8
|
+
|
9
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
10
|
+
|
11
|
+
RSpec.configure do |config|
|
12
|
+
config.order = "random"
|
13
|
+
end
|
14
|
+
|
15
|
+
Spork.prefork do
|
16
|
+
end
|
17
|
+
|
18
|
+
Spork.each_run do
|
19
|
+
end
|
20
|
+
|
metadata
ADDED
@@ -0,0 +1,250 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sitemap_maker
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Hiroyuki Ishii
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-10-30 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rspec
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rdoc
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: bundler
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: jeweler
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: fuubar
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: simplecov
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: nokogiri
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: spork
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: plymouth
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
- !ruby/object:Gem::Dependency
|
159
|
+
name: mechanize
|
160
|
+
requirement: !ruby/object:Gem::Requirement
|
161
|
+
none: false
|
162
|
+
requirements:
|
163
|
+
- - ! '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
type: :development
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
none: false
|
170
|
+
requirements:
|
171
|
+
- - ! '>='
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
- !ruby/object:Gem::Dependency
|
175
|
+
name: to_regexp
|
176
|
+
requirement: !ruby/object:Gem::Requirement
|
177
|
+
none: false
|
178
|
+
requirements:
|
179
|
+
- - ! '>='
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0'
|
182
|
+
type: :development
|
183
|
+
prerelease: false
|
184
|
+
version_requirements: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
description: longer description of your gem
|
191
|
+
email: alprhcp666@gmail.com
|
192
|
+
executables:
|
193
|
+
- sitemap_maker
|
194
|
+
extensions: []
|
195
|
+
extra_rdoc_files:
|
196
|
+
- LICENSE.txt
|
197
|
+
- README.rdoc
|
198
|
+
files:
|
199
|
+
- .document
|
200
|
+
- .rspec
|
201
|
+
- Gemfile
|
202
|
+
- Gemfile.lock
|
203
|
+
- LICENSE.txt
|
204
|
+
- README.rdoc
|
205
|
+
- Rakefile
|
206
|
+
- bin/sitemap_maker
|
207
|
+
- lib/SitemapMaker/Nokogiri/XML/node.rb
|
208
|
+
- lib/SitemapMaker/Nokogiri/html.rb
|
209
|
+
- lib/SitemapMaker/core_ext/object/blank.rb
|
210
|
+
- lib/SitemapMaker/core_ext/object/try.rb
|
211
|
+
- lib/SitemapMaker/core_ext/uri/derectory.rb
|
212
|
+
- lib/SitemapMaker/html.rb
|
213
|
+
- lib/SitemapMaker/site_tree.rb
|
214
|
+
- lib/SitemapMaker/utils.rb
|
215
|
+
- lib/SitemapMaker/version.rb
|
216
|
+
- lib/sitemap_maker.rb
|
217
|
+
- sitemap_maker.gemspec
|
218
|
+
- spec/html_spec.rb
|
219
|
+
- spec/site_tree_spec.rb
|
220
|
+
- spec/sitemap_maker_spec.rb
|
221
|
+
- spec/spec_helper.rb
|
222
|
+
homepage: http://github.com/taichouchou2/sitemap_maker
|
223
|
+
licenses:
|
224
|
+
- MIT
|
225
|
+
post_install_message:
|
226
|
+
rdoc_options: []
|
227
|
+
require_paths:
|
228
|
+
- lib
|
229
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
230
|
+
none: false
|
231
|
+
requirements:
|
232
|
+
- - ! '>='
|
233
|
+
- !ruby/object:Gem::Version
|
234
|
+
version: '0'
|
235
|
+
segments:
|
236
|
+
- 0
|
237
|
+
hash: -1644386599553737571
|
238
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
239
|
+
none: false
|
240
|
+
requirements:
|
241
|
+
- - ! '>='
|
242
|
+
- !ruby/object:Gem::Version
|
243
|
+
version: '0'
|
244
|
+
requirements: []
|
245
|
+
rubyforge_project:
|
246
|
+
rubygems_version: 1.8.23
|
247
|
+
signing_key:
|
248
|
+
specification_version: 3
|
249
|
+
summary: one-line summary of your gem
|
250
|
+
test_files: []
|