fly_parser 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/fly_parser/version.rb +3 -0
- data/lib/fly_parser.rb +14 -9
- metadata +30 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7e159421fc321f0df2f306091165a72155f57abd
|
|
4
|
+
data.tar.gz: f7348be727d4ec8dc5a6e6bc7527f9b6e2a193b7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ab584fc837d217ede31b77d7f4cf06fae1b04955d2306b276f4d5928ba07eae7ff5a76dad798114ddb83bd85103f70ffe6826953a6bac32e3551e606a88fcd3a
|
|
7
|
+
data.tar.gz: 80cd5806a09f8610bee270bde4257055a9f99eb3f2bd218201d9381347b0fa7245924ab607dfd07feefc662f5f6b14e293e1c6e36c9f8cb89b3e5aa1fd9a0cff
|
data/lib/fly_parser.rb
CHANGED
|
@@ -5,6 +5,8 @@ require 'pry'
|
|
|
5
5
|
require 'open-uri'
|
|
6
6
|
require 'yaml'
|
|
7
7
|
require 'mechanize'
|
|
8
|
+
require 'benchmark'
|
|
9
|
+
|
|
8
10
|
BASE_PATH = File.expand_path("fly_parser/*.rb", File.dirname(__FILE__))
|
|
9
11
|
LOGO_PATH = File.expand_path("fly_parser/logo.txt", File.dirname(__FILE__))
|
|
10
12
|
|
|
@@ -22,16 +24,18 @@ def require_all(path)
|
|
|
22
24
|
end
|
|
23
25
|
end
|
|
24
26
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
require base_file
|
|
28
|
-
end
|
|
29
|
-
Dir.chdir RAILS_ROOT
|
|
30
|
-
require RAILS_BOOT_PATH
|
|
31
|
-
require RAILS_CONFIG_PATH
|
|
32
|
-
require_all 'fly_parser/sources'
|
|
27
|
+
Dir[BASE_PATH].each do |base_file|
|
|
28
|
+
require base_file
|
|
33
29
|
end
|
|
34
30
|
|
|
31
|
+
# Require whole rails app , not only ActiveRecord
|
|
32
|
+
# because there is a lot of AR dependecies ( carrierwave, minimagick, state_machine etc )
|
|
33
|
+
|
|
34
|
+
Dir.chdir RAILS_ROOT
|
|
35
|
+
require RAILS_BOOT_PATH
|
|
36
|
+
require RAILS_CONFIG_PATH
|
|
37
|
+
require_all 'fly_parser/sources'
|
|
38
|
+
|
|
35
39
|
module Parser
|
|
36
40
|
class << self
|
|
37
41
|
include Enable if defined? Enable
|
|
@@ -64,7 +68,8 @@ module Parser
|
|
|
64
68
|
init_parser(source)
|
|
65
69
|
parse_and_save(source["items"])
|
|
66
70
|
end
|
|
67
|
-
|
|
71
|
+
# Threads are more slowly in this case, so don't use it
|
|
72
|
+
# because we need delay(some sources ban you for a lot of requests concurrently)
|
|
68
73
|
def parse_and_save(items)
|
|
69
74
|
items.each do |item|
|
|
70
75
|
ap "Parsing #{item['type']}"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fly_parser
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ruslan Korolev
|
|
@@ -10,20 +10,6 @@ bindir: bin
|
|
|
10
10
|
cert_chain: []
|
|
11
11
|
date: 2014-10-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
|
-
- !ruby/object:Gem::Dependency
|
|
14
|
-
name: rails
|
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
|
16
|
-
requirements:
|
|
17
|
-
- - ~>
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: 4.1.4
|
|
20
|
-
type: :runtime
|
|
21
|
-
prerelease: false
|
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
-
requirements:
|
|
24
|
-
- - ~>
|
|
25
|
-
- !ruby/object:Gem::Version
|
|
26
|
-
version: 4.1.4
|
|
27
13
|
- !ruby/object:Gem::Dependency
|
|
28
14
|
name: nokogiri
|
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -108,6 +94,34 @@ dependencies:
|
|
|
108
94
|
- - '>='
|
|
109
95
|
- !ruby/object:Gem::Version
|
|
110
96
|
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: rake
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - '>='
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - '>='
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: rspec
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - '>='
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '0'
|
|
118
|
+
type: :development
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - '>='
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '0'
|
|
111
125
|
description: Simple fly parser for internal uses
|
|
112
126
|
email:
|
|
113
127
|
- rusik3@gmail.com
|
|
@@ -128,6 +142,7 @@ files:
|
|
|
128
142
|
- lib/fly_parser/sources/news-nl.rb
|
|
129
143
|
- lib/fly_parser/sources/news.rb
|
|
130
144
|
- lib/fly_parser/sources/sport.rb
|
|
145
|
+
- lib/fly_parser/version.rb
|
|
131
146
|
- lib/fly_parser/xml_base.rb
|
|
132
147
|
homepage: http://rubygems.org
|
|
133
148
|
licenses:
|