nistbib 0.1.8 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/lib/nistbib/hit_collection.rb +4 -5
- data/lib/nistbib/nist_bibliography.rb +1 -1
- data/lib/nistbib/scrapper.rb +3 -1
- data/lib/nistbib/version.rb +1 -1
- data/nistbib.gemspec +1 -1
- metadata +3 -4
- data/lib/nistbib/workers_pool.rb +0 -41
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3fd84906d23f136d265529518acb196d88634951
|
4
|
+
data.tar.gz: 3f061f6e0fc98fe427e21191ec04625753ebf6e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c350a8519d63c55d12abd1d368d805dbfce664385056e3c6562e0f2659c0cfb56aec36eed20d5588ea5ced1479521821a69aa6fdbd1e3cc27a413c3b6c554c5f
|
7
|
+
data.tar.gz: 608506b77c1051759e51bfd65c45e9c85723b6baf98eeb04916623023194c55e098e150121baa68ca1ca98dae0980de20f80e9ba4034f6929dc6bc23cdf49f6d
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
nistbib (0.1.
|
5
|
-
relaton-bib (~> 0.1.
|
4
|
+
nistbib (0.1.9)
|
5
|
+
relaton-bib (~> 0.1.6)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
@@ -32,9 +32,9 @@ GEM
|
|
32
32
|
pry-byebug (3.7.0)
|
33
33
|
byebug (~> 11.0)
|
34
34
|
pry (~> 0.10)
|
35
|
-
public_suffix (3.0
|
35
|
+
public_suffix (3.1.0)
|
36
36
|
rake (10.5.0)
|
37
|
-
relaton-bib (0.1.
|
37
|
+
relaton-bib (0.1.6)
|
38
38
|
addressable
|
39
39
|
nokogiri (~> 1.8.4)
|
40
40
|
rspec (3.8.0)
|
@@ -1,15 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require 'nistbib/workers_pool'
|
3
|
+
require "nistbib/hit"
|
5
4
|
require "addressable/uri"
|
6
|
-
require
|
5
|
+
require "open-uri"
|
7
6
|
|
8
7
|
module NistBib
|
9
8
|
# Page of hit collection.
|
10
9
|
class HitCollection < Array
|
11
10
|
|
12
|
-
DOMAIN =
|
11
|
+
DOMAIN = "https://csrc.nist.gov"
|
13
12
|
|
14
13
|
# @return [TrueClass, FalseClass]
|
15
14
|
attr_reader :fetched
|
@@ -61,7 +60,7 @@ module NistBib
|
|
61
60
|
|
62
61
|
# @return [Iecbib::HitCollection]
|
63
62
|
def fetch
|
64
|
-
workers = WorkersPool.new 4
|
63
|
+
workers = RelatonBib::WorkersPool.new 4
|
65
64
|
workers.worker(&:fetch)
|
66
65
|
each do |hit|
|
67
66
|
workers << hit
|
@@ -116,7 +116,7 @@ module NistBib
|
|
116
116
|
end
|
117
117
|
|
118
118
|
def fetch_pages(s, n)
|
119
|
-
workers = WorkersPool.new n
|
119
|
+
workers = RelatonBib::WorkersPool.new n
|
120
120
|
workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
|
121
121
|
s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
|
122
122
|
workers.end
|
data/lib/nistbib/scrapper.rb
CHANGED
@@ -5,10 +5,11 @@ module NistBib
|
|
5
5
|
class << self
|
6
6
|
DOMAIN = "https://csrc.nist.gov".freeze
|
7
7
|
|
8
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
9
|
+
|
8
10
|
# Parse page.
|
9
11
|
# @param hit_data [Hash]
|
10
12
|
# @return [Hash]
|
11
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
12
13
|
def parse_page(hit_data)
|
13
14
|
doc = get_page hit_data[:url]
|
14
15
|
|
@@ -33,6 +34,7 @@ module NistBib
|
|
33
34
|
commentperiod: fetch_commentperiod(doc),
|
34
35
|
)
|
35
36
|
end
|
37
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
36
38
|
|
37
39
|
private
|
38
40
|
|
data/lib/nistbib/version.rb
CHANGED
data/nistbib.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nistbib
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
@@ -170,14 +170,14 @@ dependencies:
|
|
170
170
|
requirements:
|
171
171
|
- - "~>"
|
172
172
|
- !ruby/object:Gem::Version
|
173
|
-
version: 0.1.
|
173
|
+
version: 0.1.6
|
174
174
|
type: :runtime
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
|
-
version: 0.1.
|
180
|
+
version: 0.1.6
|
181
181
|
description: 'NistBib: retrive NIST standards.'
|
182
182
|
email:
|
183
183
|
- open.source@ribose.com
|
@@ -207,7 +207,6 @@ files:
|
|
207
207
|
- lib/nistbib/nist_bibliography.rb
|
208
208
|
- lib/nistbib/scrapper.rb
|
209
209
|
- lib/nistbib/version.rb
|
210
|
-
- lib/nistbib/workers_pool.rb
|
211
210
|
- lib/nistbib/xml_parser.rb
|
212
211
|
- lib/relaton/processor.rb
|
213
212
|
- nistbib.gemspec
|
data/lib/nistbib/workers_pool.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Workers poll.
|
4
|
-
class WorkersPool
|
5
|
-
attr_accessor :nb_hits
|
6
|
-
|
7
|
-
def initialize(num_workers = 2)
|
8
|
-
@num_workers = num_workers < 2 ? 2 : num_workers
|
9
|
-
@queue = SizedQueue.new(num_workers * 2)
|
10
|
-
@result = []
|
11
|
-
@nb_hits = 0
|
12
|
-
end
|
13
|
-
|
14
|
-
def worker(&block)
|
15
|
-
@threads = Array.new @num_workers do
|
16
|
-
Thread.new do
|
17
|
-
until (item = @queue.pop) == :END
|
18
|
-
@result << yield(item) if block
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
def result
|
25
|
-
@threads.each(&:join)
|
26
|
-
@result
|
27
|
-
end
|
28
|
-
|
29
|
-
def <<(item)
|
30
|
-
@queue << item
|
31
|
-
self
|
32
|
-
end
|
33
|
-
|
34
|
-
def end
|
35
|
-
@num_workers.times { @queue << :END }
|
36
|
-
end
|
37
|
-
|
38
|
-
# def size
|
39
|
-
# @result.size
|
40
|
-
# end
|
41
|
-
end
|