cdnget 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGES.md +12 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/bin/cdnget +155 -105
  6. data/cdnget.gemspec +4 -5
  7. data/lib/cdnget.rb +155 -105
  8. data/test/cdnget_test.rb +707 -410
  9. metadata +11 -26
data/lib/cdnget.rb CHANGED
@@ -1,21 +1,24 @@
1
1
  #!/usr/bin/env ruby
2
2
  # -*- coding: utf-8 -*-
3
+ # frozen_string_literal: true
3
4
 
4
5
  ##
5
- ## Download files from CDN (CDNJS, jsDelivr, UNPKG, Google).
6
+ ## Download JS/CSS files from public CDN.
6
7
  ##
7
- ## - CDNJS (https://cdnjs.com/)
8
- ## - jsDelivr (https://www.jsdelivr.com/)
9
- ## - UNPKG (https://unpkg.com/)
10
- ## - Google (https://developers.google.com/speed/libraries/)
8
+ ## * CDNJS (https://cdnjs.com/)
9
+ ## * jsDelivr (https://www.jsdelivr.com/)
10
+ ## * UNPKG (https://unpkg.com/)
11
+ ## * Google (https://developers.google.com/speed/libraries/)
11
12
  ##
12
13
  ## Example:
13
14
  ## $ cdnget # list public CDN
14
- ## $ cdnget [-q] cdnjs # list libraries
15
+ ## $ cdnget [-q] cdnjs # list libraries (except jsdelivr/unpkg)
16
+ ## $ cdnget [-q] cdnjs '*jquery*' # search libraries
15
17
  ## $ cdnget [-q] cdnjs jquery # list versions
16
18
  ## $ cdnget [-q] cdnjs jquery latest # detect latest version
17
19
  ## $ cdnget [-q] cdnjs jquery 3.6.0 # list files
18
- ## $ cdnget [-q] cdnjs jquery 3.6.0 /tmp # download files
20
+ ## $ mkdir -p static/lib # create a directory
21
+ ## $ cdnget [-q] cdnjs jquery 3.6.0 static/lib # download files
19
22
  ##
20
23
 
21
24
  require 'open-uri'
@@ -30,7 +33,7 @@ require 'pp'
30
33
  module CDNGet
31
34
 
32
35
 
33
- RELEASE = '$Release: 1.0.2 $'.split()[1]
36
+ RELEASE = '$Release: 1.1.0 $'.split()[1]
34
37
 
35
38
 
36
39
  class HttpConnection
@@ -57,26 +60,34 @@ module CDNGet
57
60
  end
58
61
 
59
62
  def get(uri)
60
- resp = @http.send_request('GET', uri.path, nil, @headers)
63
+ resp = request('GET', uri.path, uri.query)
64
+ return _get_resp_body(resp)
65
+ end
66
+
67
+ def post(uri, payload)
68
+ resp = request('POST', uri.path, uri.query, payload: payload)
69
+ return _get_resp_body(resp)
70
+ end
71
+
72
+ def request(meth, path, query=nil, payload: nil, headers: nil)
73
+ path += "?" + query if query
74
+ if @headers
75
+ headers ||= {}
76
+ headers.update(@headers)
77
+ end
78
+ resp = @http.send_request(meth, path, payload, headers)
79
+ return resp
80
+ end
81
+
82
+ def _get_resp_body(resp)
61
83
  case resp
62
84
  when Net::HTTPSuccess
63
85
  return resp.body
64
- #when HTTPInformation, Net::HTTPRedirection, HTTPClientError, HTTPServerError
65
86
  else
66
87
  raise HttpError.new(resp.code.to_i, resp.message)
67
88
  end
68
89
  end
69
90
 
70
- def post(uri, payload)
71
- path = uri.path
72
- path += "?"+uri.query if uri.query && !uri.query.empty?
73
- resp = @http.send_request('POST', path, payload, @headers)
74
- case resp
75
- when Net::HTTPSuccess ; return resp.body
76
- else ; raise HttpError.new(resp.code.to_i, resp.message)
77
- end
78
- end
79
-
80
91
  def close()
81
92
  @http.finish()
82
93
  end
@@ -178,6 +189,12 @@ module CDNGet
178
189
  nil
179
190
  end
180
191
 
192
+ def latest_version(library)
193
+ validate(library, nil)
194
+ d = self.find(library)
195
+ return d[:versions].first
196
+ end
197
+
181
198
  protected
182
199
 
183
200
  def http_get(url)
@@ -203,24 +220,34 @@ module CDNGet
203
220
  end
204
221
  end
205
222
 
223
+ LIBRARY_REXP = /\A[-.\w]+\z/
224
+ VERSION_REXP = /\A\d+(\.\d+)+([-.\w]+)?/
225
+
206
226
  def validate(library, version)
207
227
  if library
208
- library =~ /\A[-.\w]+\z/ or
209
- raise ArgumentError.new("#{library.inspect}: Unexpected library name.")
228
+ rexp = self.class.const_get(:LIBRARY_REXP)
229
+ library =~ self.class.const_get(:LIBRARY_REXP) or
230
+ raise CommandError.new("#{library}: Invalid library name.")
210
231
  end
211
232
  if version
212
- version =~ /\A\d+(\.\d+)+([-.\w]+)?/ or
213
- raise ArgumentError.new("#{version.inspect}: Unexpected version number.")
233
+ version =~ self.class.const_get(:VERSION_REXP) or
234
+ raise CommandError.new("#{version}: Invalid version number.")
214
235
  end
215
236
  end
216
237
 
238
+ def npmpkg_url(library, version)
239
+ pkg = library.sub(/^@[-\w]+/, '')
240
+ path = "/#{library.gsub('/', '%2f')}/-/#{pkg}-#{version}.tgz"
241
+ return "https://registry.npmjs.org#{path}"
242
+ end
243
+
217
244
  def format_integer(value)
218
245
  return value.to_s.reverse.scan(/..?.?/).collect {|s| s.reverse }.reverse.join(',')
219
246
  end
220
247
 
221
248
  def _debug_print(x)
222
249
  if @debug_mode
223
- $stderr.puts "\e[0;35m*** #{PP.pp(x,'')}\e[0m"
250
+ $stderr.puts "\e[0;35m*** #{PP.pp(x,String.new)}\e[0m"
224
251
  end
225
252
  end
226
253
 
@@ -229,7 +256,9 @@ module CDNGet
229
256
 
230
257
  class CDNJS < Base
231
258
  CODE = "cdnjs"
232
- SITE_URL = 'https://cdnjs.com/'
259
+ SITE_URL = "https://cdnjs.com/"
260
+ API_URL = "https://api.cdnjs.com/libraries"
261
+ CDN_URL = "https://cdnjs.cloudflare.com/ajax/libs"
233
262
 
234
263
  def fetch(url, library=nil)
235
264
  json_str = super
@@ -248,7 +277,7 @@ module CDNGet
248
277
  protected :fetch
249
278
 
250
279
  def list()
251
- jstr = fetch("https://api.cdnjs.com/libraries?fields=name,description")
280
+ jstr = fetch("#{API_URL}?fields=name,description")
252
281
  jdata = JSON.parse(jstr)
253
282
  _debug_print(jdata)
254
283
  libs = jdata['results'].collect {|d| {name: d['name'], desc: d['description']} }
@@ -257,7 +286,7 @@ module CDNGet
257
286
 
258
287
  def find(library)
259
288
  validate(library, nil)
260
- jstr = fetch("https://api.cdnjs.com/libraries/#{library}", library)
289
+ jstr = fetch("#{API_URL}/#{library}", library)
261
290
  jdata = JSON.parse(jstr)
262
291
  _debug_print(jdata)
263
292
  versions = jdata['assets'].collect {|d| d['version'] }\
@@ -275,12 +304,12 @@ module CDNGet
275
304
 
276
305
  def get(library, version)
277
306
  validate(library, version)
278
- jstr = fetch("https://api.cdnjs.com/libraries/#{library}", library)
307
+ jstr = fetch("#{API_URL}/#{library}", library)
279
308
  jdata = JSON.parse(jstr)
280
309
  _debug_print(jdata)
281
310
  d = jdata['assets'].find {|d| d['version'] == version } or
282
311
  raise CommandError.new("#{library}/#{version}: Library or version not found.")
283
- baseurl = "https://cdnjs.cloudflare.com/ajax/libs/#{library}/#{version}/"
312
+ baseurl = "#{CDN_URL}/#{library}/#{version}/"
284
313
  return {
285
314
  name: library,
286
315
  version: version,
@@ -303,10 +332,12 @@ module CDNGet
303
332
  SITE_URL = "https://www.jsdelivr.com/"
304
333
  #API_URL = "https://api.jsdelivr.com/v1/jsdelivr/libraries"
305
334
  API_URL = "https://data.jsdelivr.com/v1"
335
+ CDN_URL = "https://cdn.jsdelivr.net/npm"
306
336
  HEADERS = {
307
337
  "x-algo""lia-app""lication-id"=>"OFCNC""OG2CU",
308
338
  "x-algo""lia-api""-key"=>"f54e21fa3a2""a0160595bb05""8179bfb1e",
309
339
  }
340
+ LIBRARY_REXP = /\A([-.\w]+|\@[-\w]+\/[-.\w]+)\z/
310
341
 
311
342
  def list()
312
343
  return nil # nil means that this CDN can't list libraries without pattern
@@ -335,7 +366,7 @@ module CDNGet
335
366
 
336
367
  def find(library)
337
368
  validate(library, nil)
338
- url = "https://ofcncog2cu-dsn.algolia.net/1/indexes/npm-search/#{library}"
369
+ url = "https://ofcncog2cu-dsn.algolia.net/1/indexes/npm-search/#{library.sub('/', '%2f')}"
339
370
  uri = URI.parse(url)
340
371
  begin
341
372
  json = HttpConnection.open(uri, HEADERS) {|http| http.get(uri) }
@@ -372,7 +403,7 @@ module CDNGet
372
403
  end
373
404
  jdata = JSON.load(json)
374
405
  files = jdata["files"].collect {|d| d["name"] }
375
- baseurl = "https://cdn.jsdelivr.net/npm/#{library}@#{version}"
406
+ baseurl = "#{CDN_URL}/#{library}@#{version}"
376
407
  _debug_print(jdata)
377
408
  #
378
409
  dict = find(library)
@@ -380,7 +411,7 @@ module CDNGet
380
411
  dict.update({
381
412
  version: version,
382
413
  info: File.join(SITE_URL, "/package/npm/#{library}?version=#{version}"),
383
- npmpkg: "https://registry.npmjs.org/#{library}/-/#{library}-#{version}.tgz",
414
+ npmpkg: npmpkg_url(library, version),
384
415
  urls: files.collect {|x| baseurl + x },
385
416
  files: files,
386
417
  baseurl: baseurl,
@@ -390,6 +421,13 @@ module CDNGet
390
421
  return dict
391
422
  end
392
423
 
424
+ def latest_version(library)
425
+ validate(library, nil)
426
+ json = fetch("#{API_URL}/package/npm/#{library}")
427
+ jdict = JSON.load(json)
428
+ return jdict["tags"]["latest"]
429
+ end
430
+
393
431
  end
394
432
 
395
433
 
@@ -398,6 +436,7 @@ module CDNGet
398
436
  SITE_URL = "https://unpkg.com/"
399
437
  #API_URL = "https://www.npmjs.com"
400
438
  API_URL = "https://api.npms.io/v2"
439
+ LIBRARY_REXP = /\A([-.\w]+|\@[-\w]+\/[-.\w]+)\z/
401
440
 
402
441
  protected
403
442
 
@@ -428,7 +467,7 @@ module CDNGet
428
467
 
429
468
  def find(library)
430
469
  validate(library, nil)
431
- json = fetch("#{API_URL}/package/#{library}", library)
470
+ json = fetch("#{API_URL}/package/#{library.sub('/', '%2f')}", library)
432
471
  jdata = JSON.load(json)
433
472
  _debug_print(jdata)
434
473
  dict = jdata["collected"]["metadata"]
@@ -458,43 +497,69 @@ module CDNGet
458
497
  dict = find(library)
459
498
  dict.delete(:versions)
460
499
  #
461
- url = "https://data.jsdelivr.com/v1/package/npm/#{library}@#{version}/flat"
500
+ url = "#{SITE_URL}#{library}@#{version}/?meta"
462
501
  begin
463
502
  json = fetch(url, library)
464
503
  rescue CommandError
465
- raise CommandError.new("#{library}@#{version}: Library or version not found.")
504
+ raise CommandError.new("#{library}@#{version}: Version not found.")
466
505
  end
467
- jdata = JSON.load(json)
468
- files = jdata["files"].collect {|d| d["name"] }
469
- baseurl = File.join(SITE_URL, "/#{library}@#{version}")
506
+ jdata = JSON.load(json)
470
507
  _debug_print(jdata)
508
+ pr = proc do |jdata, files|
509
+ jdata['files'].each do |d|
510
+ d['type'] == "directory" ? pr.call(d, files) \
511
+ : (files << d['path'])
512
+ end if jdata['files']
513
+ files
514
+ end
515
+ files = pr.call(jdata, [])
516
+ #files = files.sort_by {|s| s.downcase }
517
+ baseurl = File.join(SITE_URL, "/#{library}@#{version}")
471
518
  #
472
519
  dict.update({
473
520
  name: library,
474
521
  version: version,
475
522
  info: File.join(SITE_URL, "/browse/#{library}@#{version}/"),
476
- npmpkg: "https://registry.npmjs.org/#{library}/-/#{library}-#{version}.tgz",
523
+ npmpkg: npmpkg_url(library, version),
477
524
  urls: files.collect {|x| baseurl+x },
478
525
  files: files,
479
526
  baseurl: baseurl,
480
- default: jdata["default"],
527
+ #default: jdata["default"],
481
528
  destdir: "#{library}@#{version}",
482
529
  skipfile: /\.DS_Store\z/, # downloading '.DS_Store' from UNPKG results in 403
483
530
  })
484
531
  return dict
485
532
  end
486
533
 
534
+ def latest_version(library)
535
+ validate(library, nil)
536
+ version = nil
537
+ url = File.join(SITE_URL, "/browse/#{library}/")
538
+ uri = URI.parse(url)
539
+ HttpConnection.open(URI.parse(SITE_URL)) do |http|
540
+ resp = http.request('HEAD', "/browse/#{library}/")
541
+ if resp.code == "302" # 302 Found
542
+ location = resp.header['Location']
543
+ location =~ /@([^@\/]+)\/\z/
544
+ version = $1
545
+ end
546
+ end
547
+ version ||= super(library)
548
+ return version
549
+ end
550
+
487
551
  end
488
552
 
489
553
 
490
554
  class GoogleCDN < Base
491
555
  CODE = "google"
492
- SITE_URL = 'https://developers.google.com/speed/libraries/'
556
+ SITE_URL = "https://developers.google.com/speed/libraries/"
557
+ CDN_URL = "https://ajax\.googleapis\.com/ajax/libs"
493
558
 
494
559
  def list()
495
- html = fetch("https://developers.google.com/speed/libraries/")
560
+ html = fetch(SITE_URL)
496
561
  _debug_print(html)
497
- rexp = %r`"https://ajax\.googleapis\.com/ajax/libs/([^/]+)/([^/]+)/([^"]+)"`
562
+ rexp = %r`"#{Regexp.escape(CDN_URL)}/([^/]+)/([^/]+)/([^"]+)"`
498
563
  libs = []
499
564
  html.scan(rexp) do |lib, ver, file|
500
565
  libs << {name: lib, desc: "latest version: #{ver}" }
@@ -504,9 +569,9 @@ module CDNGet
504
569
 
505
570
  def find(library)
506
571
  validate(library, nil)
507
- html = fetch("https://developers.google.com/speed/libraries/")
572
+ html = fetch(SITE_URL)
508
573
  _debug_print(html)
509
- rexp = %r`"https://ajax\.googleapis\.com/ajax/libs/#{library}/`
574
+ rexp = %r`"#{Regexp.escape(CDN_URL)}/#{library}/`
510
575
  site_url = nil
511
576
  versions = []
512
577
  urls = []
@@ -546,14 +611,14 @@ module CDNGet
546
611
  def get(library, version)
547
612
  validate(library, version)
548
613
  d = find(library)
549
- d[:versions].find(version) or
550
- raise CommandError.new("#{version}: No such version of #{library}.")
614
+ d[:versions].find {|s| s == version } or
615
+ raise CommandError.new("#{library} #{version}: Version not found.")
551
616
  urls = d[:urls]
552
617
  if urls
553
618
  rexp = /(\/libs\/#{library})\/[^\/]+/
554
619
  urls = urls.collect {|x| x.gsub(rexp, "\\1/#{version}") }
555
620
  end
556
- baseurl = "https://ajax.googleapis.com/ajax/libs/#{library}/#{version}"
621
+ baseurl = "#{CDN_URL}/#{library}/#{version}"
557
622
  files = urls ? urls.collect {|x| x[baseurl.length..-1] } : nil
558
623
  return {
559
624
  name: d[:name],
@@ -605,13 +670,13 @@ Options:
605
670
  --debug : (debug mode)
606
671
 
607
672
  Example:
608
- $ #{script} # list public CDN names
609
- $ #{script} [-q] cdnjs # list libraries
610
- $ #{script} [-q] cdnjs 'jquery*' # search libraries
611
- $ #{script} [-q] cdnjs jquery # list versions
612
- $ #{script} [-q] cdnjs jquery latest # show latest version
613
- $ #{script} [-q] cdnjs jquery 2.2.0 # list files
614
- $ mkdir -p static/lib # create a directory
673
+ $ #{script} # list public CDN names
674
+ $ #{script} [-q] cdnjs # list libraries (except jsdelivr/unpkg)
675
+ $ #{script} [-q] cdnjs 'jquery*' # search libraries
676
+ $ #{script} [-q] cdnjs jquery # list versions
677
+ $ #{script} [-q] cdnjs jquery latest # show latest version
678
+ $ #{script} [-q] cdnjs jquery 2.2.0 # list files
679
+ $ mkdir -p static/lib # create a directory
615
680
  $ #{script} [-q] cdnjs jquery 2.2.0 static/lib # download files
616
681
  static/lib/jquery/2.2.0/jquery.js ... Done (258,388 byte)
617
682
  static/lib/jquery/2.2.0/jquery.min.js ... Done (85,589 byte)
@@ -638,9 +703,6 @@ END
638
703
  return RELEASE + "\n" if cmdopts['v'] || cmdopts['version']
639
704
  @quiet = cmdopts['quiet'] || cmdopts['q']
640
705
  @debug_mode = cmdopts['debug']
641
- #
642
- validate(args[1], args[2])
643
- #
644
706
  case args.length
645
707
  when 0
646
708
  return do_list_cdns()
@@ -664,17 +726,6 @@ END
664
726
  end
665
727
  end
666
728
 
667
- def validate(library, version)
668
- if library && ! library.include?('*')
669
- library =~ /\A[-.\w]+\z/ or
670
- raise CommandError.new("#{library}: Unexpected library name.")
671
- end
672
- if version
673
- version =~ /\A[-.\w]+\z/ or
674
- raise CommandError.new("#{version}: Unexpected version number.")
675
- end
676
- end
677
-
678
729
  def parse_cmdopts(cmdargs, short_opts, long_opts)
679
730
  cmdopts = {}
680
731
  while cmdargs[0] && cmdargs[0].start_with?('-')
@@ -733,67 +784,66 @@ END
733
784
  def do_find_library(cdn_code, library)
734
785
  cdn = find_cdn(cdn_code)
735
786
  d = cdn.find(library)
736
- s = ""
787
+ buf = []
737
788
  if @quiet
738
789
  d[:versions].each do |ver|
739
- s << "#{ver}\n"
740
- end if d[:versions]
790
+ buf << "#{ver}\n"
791
+ end unless empty?(d[:versions])
741
792
  else
742
- s << "name: #{d[:name]}\n"
743
- s << "desc: #{d[:desc]}\n" if d[:desc]
744
- s << "tags: #{d[:tags]}\n" if d[:tags]
745
- s << "site: #{d[:site]}\n" if d[:site]
746
- s << "info: #{d[:info]}\n" if d[:info]
747
- s << "license: #{d[:license]}\n" if d[:license]
748
- s << "snippet: |\n" << d[:snippet].gsub(/^/, ' ') if d[:snippet]
749
- s << "versions:\n"
793
+ buf << "name: #{d[:name]}\n"
794
+ buf << "desc: #{d[:desc]}\n" unless empty?(d[:desc])
795
+ buf << "tags: #{d[:tags]}\n" unless empty?(d[:tags])
796
+ buf << "site: #{d[:site]}\n" unless empty?(d[:site])
797
+ buf << "info: #{d[:info]}\n" unless empty?(d[:info])
798
+ buf << "license: #{d[:license]}\n" unless empty?(d[:license])
799
+ buf << "snippet: |\n" << d[:snippet].gsub(/^/, ' ') unless empty?(d[:snippet])
800
+ buf << "versions:\n"
750
801
  d[:versions].each do |ver|
751
- s << " - #{ver}\n"
752
- end if d[:versions]
802
+ buf << " - #{ver}\n"
803
+ end unless empty?(d[:versions])
753
804
  end
754
- return s
805
+ return buf.join()
755
806
  end
756
807
 
757
808
  def do_get_library(cdn_code, library, version)
758
809
  cdn = find_cdn(cdn_code)
759
- version = _latest_version(cdn, library) if version == 'latest'
810
+ version = cdn.latest_version(library) if version == 'latest'
760
811
  d = cdn.get(library, version)
761
- s = ""
812
+ buf = []
762
813
  if @quiet
763
814
  d[:urls].each do |url|
764
- s << "#{url}\n"
815
+ buf << "#{url}\n"
765
816
  end if d[:urls]
766
817
  else
767
- s << "name: #{d[:name]}\n"
768
- s << "version: #{d[:version]}\n"
769
- s << "desc: #{d[:desc]}\n" if d[:desc]
770
- s << "tags: #{d[:tags]}\n" if d[:tags]
771
- s << "site: #{d[:site]}\n" if d[:site]
772
- s << "info: #{d[:info]}\n" if d[:info]
773
- s << "npmpkg: #{d[:npmpkg]}\n" if d[:npmpkg]
774
- s << "default: #{d[:default]}\n" if d[:default]
775
- s << "license: #{d[:license]}\n" if d[:license]
776
- s << "snippet: |\n" << d[:snippet].gsub(/^/, ' ') if d[:snippet]
777
- s << "urls:\n" if d[:urls]
818
+ buf << "name: #{d[:name]}\n"
819
+ buf << "version: #{d[:version]}\n"
820
+ buf << "desc: #{d[:desc]}\n" unless empty?(d[:desc])
821
+ buf << "tags: #{d[:tags]}\n" unless empty?(d[:tags])
822
+ buf << "site: #{d[:site]}\n" unless empty?(d[:site])
823
+ buf << "info: #{d[:info]}\n" unless empty?(d[:info])
824
+ buf << "npmpkg: #{d[:npmpkg]}\n" unless empty?(d[:npmpkg])
825
+ buf << "default: #{d[:default]}\n" unless empty?(d[:default])
826
+ buf << "license: #{d[:license]}\n" unless empty?(d[:license])
827
+ buf << "snippet: |\n" << d[:snippet].gsub(/^/, ' ') unless empty?(d[:snippet])
828
+ buf << "urls:\n" unless empty?(d[:urls])
778
829
  d[:urls].each do |url|
779
- s << " - #{url}\n"
780
- end if d[:urls]
830
+ buf << " - #{url}\n"
831
+ end unless empty?(d[:urls])
781
832
  end
782
- return s
833
+ return buf.join()
783
834
  end
784
835
 
785
836
  def do_download_library(cdn_code, library, version, basedir)
786
837
  cdn = find_cdn(cdn_code)
787
- version = _latest_version(cdn, library) if version == 'latest'
838
+ version = cdn.latest_version(library) if version == 'latest'
788
839
  cdn.download(library, version, basedir, quiet: @quiet)
789
840
  return nil
790
841
  end
791
842
 
792
843
  private
793
844
 
794
- def _latest_version(cdn, library)
795
- d = cdn.find(library)
796
- return d[:versions].first
845
+ def empty?(x)
846
+ return x.nil? || x.empty?
797
847
  end
798
848
 
799
849
  end