nicoquery 0.0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 4818aead4f0b2660fd00f602297d2e755adbf0d7
4
+ data.tar.gz: 54c26b1e9febb492a364b9791cb789eb95f42e08
5
+ SHA512:
6
+ metadata.gz: 79afd659ca62825f85d6a901a31b8646b7f2d6231a5fe8f817ba8ef9fc34272653eece0e4503a088dc98c9b2bc18c7e4e221b4090e5da78ec0dd5738b290e54b
7
+ data.tar.gz: e01cfaf6ee04f94654fd6262310d321ec39445100560613fbe780016a692a851a2db5f6cc62cb0222cea77cfb44f1d6bf49031cf920fd304eb3a7d1aae0bd628
data/.gitignore ADDED
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ coverage
6
+ InstalledFiles
7
+ lib/bundler/man
8
+ pkg
9
+ rdoc
10
+ spec/reports
11
+ test/tmp
12
+ test/version_tmp
13
+ tmp
14
+
15
+ # YARD artifacts
16
+ .yardoc
17
+ _yardoc
18
+ doc/
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ 2.0.0-p247
data/.travis.yml ADDED
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.0.0
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,67 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ nicoquery (0.0.1.1)
5
+ activesupport (~> 4.0.0)
6
+ i18n
7
+ nicoapi
8
+ nokogiri
9
+ nori
10
+ rest-client
11
+
12
+ GEM
13
+ remote: https://rubygems.org/
14
+ specs:
15
+ activesupport (4.0.0)
16
+ i18n (~> 0.6, >= 0.6.4)
17
+ minitest (~> 4.2)
18
+ multi_json (~> 1.3)
19
+ thread_safe (~> 0.1)
20
+ tzinfo (~> 0.3.37)
21
+ atomic (1.1.12)
22
+ coderay (1.0.9)
23
+ diff-lcs (1.2.4)
24
+ i18n (0.6.4)
25
+ method_source (0.8.2)
26
+ mime-types (1.23)
27
+ mini_portile (0.5.1)
28
+ minitest (4.7.5)
29
+ multi_json (1.7.8)
30
+ nicoapi (0.0.4)
31
+ activesupport (~> 4.0.0)
32
+ i18n
33
+ rest-client
34
+ nokogiri (1.6.0)
35
+ mini_portile (~> 0.5.0)
36
+ nori (2.3.0)
37
+ pry (0.9.12.2)
38
+ coderay (~> 1.0.5)
39
+ method_source (~> 0.8)
40
+ slop (~> 3.4)
41
+ rake (10.1.0)
42
+ rest-client (1.6.7)
43
+ mime-types (>= 1.16)
44
+ rspec (2.14.1)
45
+ rspec-core (~> 2.14.0)
46
+ rspec-expectations (~> 2.14.0)
47
+ rspec-mocks (~> 2.14.0)
48
+ rspec-core (2.14.4)
49
+ rspec-expectations (2.14.0)
50
+ diff-lcs (>= 1.1.3, < 2.0)
51
+ rspec-mocks (2.14.2)
52
+ slop (3.4.6)
53
+ thread_safe (0.1.2)
54
+ atomic
55
+ tzinfo (0.3.37)
56
+ watchr (0.7)
57
+
58
+ PLATFORMS
59
+ ruby
60
+
61
+ DEPENDENCIES
62
+ bundler (~> 1.3)
63
+ nicoquery!
64
+ pry
65
+ rake
66
+ rspec
67
+ watchr
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2013 Masami Yonehara
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,37 @@
1
+ [![Build Status](https://secure.travis-ci.org/hdemon/NicoQuery.png)](http://travis-ci.org/hdemon/NicoQuery)
2
+
3
+ # NicoQuery::Core
4
+
5
+ TODO: Write a gem description
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ gem 'NicoQuery'
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install NicoQuery
20
+
21
+ ## Usage
22
+
23
+ TODO: Write usage instructions here
24
+
25
+ ## Contributing
26
+
27
+ 1. Fork it
28
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
29
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
30
+ 4. Push to the branch (`git push origin my-new-feature`)
31
+ 5. Create new Pull Request
32
+ =======
33
+ nicoquery
34
+ =========
35
+
36
+ the manipulator of niconico douga's resources
37
+
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task :default => [:spec]
4
+ begin
5
+ require 'rspec/core/rake_task'
6
+ RSpec::Core::RakeTask.new(:spec) do |spec|
7
+ spec.pattern = 'spec/**/*_spec.rb'
8
+ spec.rspec_opts = ['-cfs']
9
+ end
10
+ rescue LoadError => e
11
+ end
data/autorun.rb ADDED
@@ -0,0 +1,5 @@
1
+ watch( 'app/(.*)\.rb' ) {|md| system("bundle exec rspec spec/#{md[1]}_spec.rb") }
2
+ watch( 'spec/(.*)\.rb' ) {|md| system("bundle exec rspec spec/#{md[1]}.rb") }
3
+
4
+
5
+
@@ -0,0 +1,44 @@
1
+ require "nicoapi"
2
+ require "nicoquery/parser"
3
+ require "nori"
4
+
5
+
6
+ module NicoQuery
7
+ module Crawler
8
+ module TagSearch
9
+ def execute(tag: tag, sort: sort, order: order, &block)
10
+ parser = NicoQuery::Parser::TagSearch.new
11
+ page = 0
12
+
13
+ loop do
14
+ command = nil
15
+ page += 1
16
+
17
+ result = NicoAPI.tag_search(tag: tag, sort: sort, order: order, page: page)
18
+ parser.parse result
19
+
20
+ self.each_movie(parser.items) do |movie|
21
+ command = block.call movie
22
+ break if command == :break || command != :continue
23
+ end
24
+
25
+ break if command == :break || command != :continue
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ def continue; :continue end
32
+ def stop; :stop end
33
+
34
+ def each_movie(parsed_movies, &block)
35
+ while parsed_movies.length > 0 do
36
+ block.call parsed_movies.shift
37
+ end
38
+ end
39
+
40
+ module_function :execute
41
+ module_function :each_movie
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,3 @@
1
+ module Nicocrawler
2
+ VERSION = "0.0.2.1"
3
+ end
@@ -0,0 +1,12 @@
1
+ require "nicoquery/crawler/tag_search"
2
+
3
+
4
+ module NicoQuery
5
+ module Crawler
6
+ def tag_search(tag: tag, sort: sort, order: order, &block)
7
+ NicoQuery::Crawler::TagSearch.execute(tag: tag, sort: sort, order: order, &block)
8
+ end
9
+
10
+ module_function :tag_search
11
+ end
12
+ end
@@ -0,0 +1,106 @@
1
+ require "nori"
2
+ require "nokogiri"
3
+
4
+
5
+ module NicoQuery
6
+ module Parser
7
+ class TagSearch
8
+ def initialize
9
+ @parser = Nori.new
10
+ end
11
+
12
+ def parse(xml)
13
+ @object = (@parser.parse xml)["rss"]["channel"]
14
+
15
+ @items = @object["item"].map do |item_object|
16
+ Item.new item_object
17
+ end
18
+ end
19
+
20
+ def items
21
+ @items
22
+ end
23
+
24
+ def tag
25
+ @object["title"].scan(/(?<=タグ\s).+(?=\‐ニコニコ動画)/)[0].split(' ')
26
+ end
27
+
28
+ def publish_date
29
+ Time.parse @object["pubDate"]
30
+ end
31
+
32
+ def last_build_date
33
+ Time.parse @object["lastBuildDate"]
34
+ end
35
+
36
+ class Item
37
+ def initialize(object)
38
+ @object = object
39
+ end
40
+
41
+ def title
42
+ @object["title"]
43
+ end
44
+
45
+ def video_id
46
+ url.scan(/(sm|nm)\d{1,}\Z/)
47
+ $&
48
+ end
49
+
50
+ def url
51
+ @object["link"]
52
+ end
53
+
54
+ def publish_date
55
+ Time.parse @object["pubDate"]
56
+ end
57
+
58
+ def thumbnail_url
59
+ description.raw_text.scan(/(?<=src\=\").{1,}(?=\"\swidth)/)
60
+ $&
61
+ end
62
+
63
+ def description
64
+ @_description ||= Description.new @object["description"]
65
+ end
66
+
67
+ def length
68
+ description.raw_text.scan(/(?<=class\=\"nico\-info\-length\"\>)\d{1,}\:\d{1,2}(?=\<\/strong\>)/)
69
+ length_string = $&.to_s.split(':')
70
+ length_string[0].to_i * 60 + length_string[1].to_i
71
+ end
72
+
73
+ class Description
74
+ attr_reader :raw_text
75
+
76
+ def initialize(raw_text)
77
+ @raw_text = raw_text.to_s
78
+ end
79
+
80
+ def text
81
+ @raw_text.scan /(?<=class\=\"nico\-description\"\>).{1,}(?=\<\/p\>)/
82
+ $&
83
+ end
84
+
85
+ def movie_references
86
+ # is this the high road?
87
+ text.scan(/((sm|nm)\d{1,})/).map {|e| e[0]}
88
+ end
89
+
90
+ def mylist_references
91
+ text.scan /(?<=mylist\/)\d{1,}/
92
+ end
93
+
94
+ def community_references
95
+ text.scan /co\d{1,}/
96
+ end
97
+
98
+ def seiga_references
99
+ text.scan /im\d{1,}/
100
+ end
101
+ end
102
+ end
103
+
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,8 @@
1
+ require "nicoquery/parser/tag_search"
2
+
3
+
4
+ module NicoQuery
5
+ module Parser
6
+
7
+ end
8
+ end
@@ -0,0 +1,3 @@
1
+ module NicoQuery
2
+ VERSION = "0.0.1.1"
3
+ end
data/lib/nicoquery.rb ADDED
@@ -0,0 +1,38 @@
1
+ require "nicoquery/source/version"
2
+
3
+ require "nicoquery/source"
4
+
5
+
6
+ module NicoQuery
7
+ def tag_search(tag: tag, sort: sort, order: order, &block)
8
+ NicoCrawler::TagSearch.execute(tag: tag, sort: sort, order: order, &block)
9
+ end
10
+
11
+ module_function :tag_search
12
+ end
13
+
14
+
15
+
16
+ module NicoQuery
17
+ def getthumbinfo(video_id)
18
+ instance = NicoQuery::GetThumbInfo.new
19
+ instance.set video_id
20
+ instance.get
21
+ end
22
+
23
+ def video_array(video_id_array)
24
+ instance = NicoQuery::VideoArray.new
25
+ instance.set video_id_array
26
+ instance.get
27
+ end
28
+
29
+ def tag_search(tag: tag, sort: sort, order: order, page: page)
30
+ instance = NicoQuery::TagSearch.new
31
+ instance.set tag: tag, sort: sort, order: order, page: page
32
+ instance.get
33
+ end
34
+
35
+ module_function :getthumbinfo
36
+ module_function :video_array
37
+ module_function :tag_search
38
+ end
data/nicoquery.gemspec ADDED
@@ -0,0 +1,33 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'nicoquery/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "nicoquery"
8
+ spec.version = NicoQuery::VERSION
9
+ spec.authors = ["Masami Yonehara"]
10
+ spec.email = ["zeitdiebe@gmail.com"]
11
+ spec.description = %q{niconico manipulator}
12
+ spec.summary = %q{niconico manipulator}
13
+ spec.homepage = ""
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_runtime_dependency "activesupport", "~> 4.0.0"
22
+ spec.add_runtime_dependency 'rest-client'
23
+ spec.add_runtime_dependency 'nori'
24
+ spec.add_runtime_dependency 'nokogiri'
25
+ spec.add_runtime_dependency "i18n"
26
+ spec.add_runtime_dependency "nicoapi"
27
+
28
+ spec.add_development_dependency "bundler", "~> 1.3"
29
+ spec.add_development_dependency "rake"
30
+ spec.add_development_dependency "rspec"
31
+ spec.add_development_dependency "watchr"
32
+ spec.add_development_dependency "pry"
33
+ end
@@ -0,0 +1,17 @@
1
+ # This file was generated by the `rspec --init` command. Conventionally, all
2
+ # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
+ # Require this file using `require "spec_helper"` to ensure that it is only
4
+ # loaded once.
5
+ #
6
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
7
+ RSpec.configure do |config|
8
+ config.treat_symbols_as_metadata_keys_with_true_values = true
9
+ config.run_all_when_everything_filtered = true
10
+ config.filter_run :focus
11
+
12
+ # Run specs in random order to surface order dependencies. If you find an
13
+ # order dependency and want to debug it, you can fix the order by providing
14
+ # the seed, which is printed after each run.
15
+ # --seed 1234
16
+ config.order = 'random'
17
+ end
@@ -0,0 +1,28 @@
1
+ require "nicoquery/crawler"
2
+
3
+
4
+ describe "NicoQuery::Crawler" do
5
+ describe "tag_search" do
6
+ before do
7
+ counter = 0
8
+ @acquired_movies = []
9
+
10
+ NicoQuery::Crawler.tag_search( tag: "ゆっくり実況プレイ",
11
+ sort: :published_at,
12
+ order: :asc
13
+ ) do |result|
14
+ counter += 1
15
+ @acquired_movies.push result
16
+ if counter >= 50 then :break else :continue end
17
+ end
18
+ end
19
+
20
+ it "should sorted by published date" do
21
+ expect(@acquired_movies[0].publish_date).to be < @acquired_movies[1].publish_date
22
+ end
23
+
24
+ it "should scrape movie's info the number of times -1 that returns :continue" do
25
+ expect(@acquired_movies.length).to eq 50
26
+ end
27
+ end
28
+ end